from __future__ import annotations import hashlib import json import mimetypes import posixpath import re import threading import time from collections import deque from collections.abc import Callable from concurrent.futures import ThreadPoolExecutor, as_completed from dataclasses import asdict, dataclass, field from pathlib import Path from typing import Literal from urllib.parse import urljoin, urlsplit from urllib.robotparser import RobotFileParser import httpx from bs4 import BeautifulSoup from bs4.element import Tag from app.config import Settings from app.models import SUPPORTED_ASSET_TYPES from app.pinned_transport import PinnedAddressBook, PinnedHTTPTransport from app.url_policy import ( NetworkRules, UrlPolicyError, canonical_url, hostname, normalize_url, resolve_and_validate, same_site_host, ) ResourceKind = Literal["document", "asset"] DEFAULT_ASSET_TYPES = SUPPORTED_ASSET_TYPES CSS_URL_RE = re.compile(r"url\(\s*(?P['\"]?)(?P.*?)(?P=quote)\s*\)", re.IGNORECASE) CSS_IMPORT_RE = re.compile( r"@import\s+(?:url\(\s*)?(?P['\"])(?P.*?)(?P=quote)\s*\)?", re.IGNORECASE, ) CSS_CHARSET_RE = re.compile(r"^\s*@charset\s+(['\"])[^'\"]+\1\s*;", re.IGNORECASE) JS_DYNAMIC_IMPORT_RE = re.compile( r"\bimport\s*\(\s*(?P['\"])(?P[^'\"]+)(?P=quote)\s*\)" ) JS_STATIC_IMPORT_RE = re.compile( r"\b(?:import|export)\s+(?:[^'\"]*?\s+from\s+)?" r"(?P['\"])(?P[^'\"]+)(?P=quote)" ) JS_NEW_URL_RE = re.compile( r"\bnew\s+URL\(\s*(?P['\"])(?P[^'\"]+)(?P=quote)\s*,\s*import\.meta\.url\s*\)" ) JS_WORKER_RE = re.compile( r"\bnew\s+(?:Worker|SharedWorker)\s*\(\s*" r"(?P['\"])(?P[^'\"]+)(?P=quote)" ) JS_SOURCE_MAP_RE = re.compile(r"(?://[#@]\s*sourceMappingURL=)(?P\S+)") META_REFRESH_RE = re.compile(r"^(?P\s*\d+(?:\.\d+)?\s*;?\s*url\s*=\s*)(?P.+)$", re.I) MAX_RECORDED_WARNINGS = 100 MAX_RECORDED_ERRORS = 250 WINDOWS_RESERVED_NAMES = { "CON", "PRN", "AUX", "NUL", *(f"COM{number}" for number in range(1, 10)), *(f"LPT{number}" for number in range(1, 10)), } class CaptureError(RuntimeError): """A capture cannot produce a valid archive.""" class CaptureCancelled(CaptureError): """The user cancelled a capture while it was running.""" @dataclass(frozen=True, slots=True) class CaptureOptions: source_url: str include_external_assets: bool max_pages: int max_depth: int max_bytes: int max_duration_seconds: int parallel_connections: int = 6 download_asset_types: tuple[str, ...] = DEFAULT_ASSET_TYPES skip_file_types: tuple[str, ...] = () use_proxy: bool = False @dataclass(frozen=True, slots=True) class CrawlRequest: url: str kind: ResourceKind depth: int referrer: str | None = None @dataclass(slots=True) class FetchResult: request: CrawlRequest outcome: Literal["ok", "skipped", "error", "cancelled"] final_url: str | None = None redirects: list[str] = field(default_factory=list) status_code: int | None = None content_type: str = "" encoding: str | None = None body: bytes | None = None message: str = "" @dataclass(slots=True) class ResourceRecord: requested_url: str final_url: str local_path: str kind: ResourceKind content_type: str status_code: int byte_count: int redirects: list[str] encoding: str | None def as_manifest(self) -> dict[str, object]: return asdict(self) @dataclass(slots=True) class CrawlResult: site_dir: Path entry_point: str report: dict[str, object] stats: dict[str, int] warnings: list[str] class PathMapper: """Map origin URLs to safe, mostly human-readable archive-relative paths.""" def __init__(self, rules: NetworkRules) -> None: self.rules = rules self._paths: dict[str, str] = {} self._used_path_keys: set[str] = set() def assign(self, url: str, content_type: str) -> str: key = canonical_url(url, self.rules) if key in self._paths: return self._paths[key] parsed = urlsplit(key) host = self._safe_segment(parsed.hostname or "unknown-host") segments = [self._safe_segment(segment) for segment in parsed.path.split("/") if segment] extension = self._extension_for(content_type) if not segments or parsed.path.endswith("/"): segments.append(f"index{extension or '.html'}") elif content_type.startswith("text/html") and "." not in segments[-1]: segments.append("index.html") elif extension and "." not in segments[-1]: segments[-1] = f"{segments[-1]}{extension}" if parsed.query: digest = hashlib.sha256(parsed.query.encode("utf-8")).hexdigest()[:12] filename = segments[-1] stem, dot, suffix = filename.rpartition(".") segments[-1] = ( f"{stem or filename}__q-{digest}{dot}{suffix}" if dot else f"{filename}__q-{digest}" ) candidate = "/".join([host, *segments]) if len(candidate) > 220: digest = hashlib.sha256(key.encode("utf-8")).hexdigest()[:16] candidate = f"{host}/_path-{digest}/{segments[-1][:80]}" if candidate.casefold() in self._used_path_keys: digest = hashlib.sha256(key.encode("utf-8")).hexdigest()[:12] parent, filename = posixpath.split(candidate) stem, dot, suffix = filename.rpartition(".") filename = ( f"{stem or filename}__u-{digest}{dot}{suffix}" if dot else f"{filename}__u-{digest}" ) candidate = posixpath.join(parent, filename) self._paths[key] = candidate self._used_path_keys.add(candidate.casefold()) return candidate @staticmethod def _safe_segment(value: str) -> str: safe = re.sub(r"[^A-Za-z0-9._-]+", "-", value.strip()) safe = safe.strip(".-") or "resource" if safe.split(".", 1)[0].upper() in WINDOWS_RESERVED_NAMES: safe = f"_{safe}" return safe[:80] @staticmethod def _extension_for(content_type: str) -> str: media_type = content_type.split(";", 1)[0].strip().lower() known = { "text/html": ".html", "text/css": ".css", "text/javascript": ".js", "application/javascript": ".js", "application/json": ".json", "application/manifest+json": ".webmanifest", "image/svg+xml": ".svg", "font/woff": ".woff", "font/woff2": ".woff2", } if media_type in known: return known[media_type] return mimetypes.guess_extension(media_type) or "" class SiteCrawler: """A bounded static web crawler with archive-ready link rewriting.""" def __init__( self, *, settings: Settings, options: CaptureOptions, cancelled: Callable[[], bool], on_progress: Callable[[str, str, dict[str, int], list[str]], None], ) -> None: self.settings = settings self.options = options self.cancelled = cancelled self.on_progress = on_progress self.rules = NetworkRules( allow_private_networks=settings.allow_private_networks, allow_nonstandard_ports=settings.allow_nonstandard_ports, ) self.source_url = normalize_url(options.source_url, self.rules) self.source_host = hostname(self.source_url) self.parallel_connections = max( 1, min(options.parallel_connections, self.settings.fetch_concurrency) ) self._proxy_url = settings.proxy_url if options.use_proxy else None self.mapper = PathMapper(self.rules) self.pending: deque[CrawlRequest] = deque() self.enqueued: set[str] = set() self.url_to_path: dict[str, str] = {} self.records: list[ResourceRecord] = [] self.errors: list[dict[str, object]] = [] self.warnings: list[str] = [] self._warning_count = 0 self._warnings_omitted = 0 self._error_count = 0 self._errors_omitted = 0 self.stats = { "pages_found": 0, "pages_fetched": 0, "assets_found": 0, "assets_fetched": 0, "bytes_downloaded": 0, "files_written": 0, } self._started_at = 0.0 self._byte_count = 0 self._byte_lock = threading.Lock() self._scope_lock = threading.Lock() self._robots_lock = threading.Lock() self._external_hosts: set[str] = set() self._robots: dict[str, RobotFileParser | bool] = {} self._root_record: ResourceRecord | None = None self._root_error: str | None = None self._limit_reached = False self._site_dir: Path | None = None self._address_book = PinnedAddressBook() limits = httpx.Limits( max_connections=max(4, self.parallel_connections * 2), max_keepalive_connections=max(2, self.parallel_connections), ) self._client = httpx.Client( follow_redirects=False, trust_env=False, timeout=httpx.Timeout(connect=10.0, read=20.0, write=10.0, pool=10.0), limits=limits, transport=PinnedHTTPTransport(self._address_book, limits, proxy=self._proxy_url), headers={"User-Agent": settings.user_agent, "Accept": "*/*"}, ) def run(self, work_dir: Path) -> CrawlResult: self._started_at = time.monotonic() self._site_dir = work_dir / "site" self._site_dir.mkdir(parents=True, exist_ok=True) try: if self._proxy_url: self._pin_proxy() self._enqueue(CrawlRequest(self.source_url, "document", 0)) with ThreadPoolExecutor(max_workers=self.parallel_connections) as executor: while self.pending and not self._limit_reached: self._ensure_not_cancelled() if self._duration_exceeded(): raise CaptureError("The configured capture time limit was reached.") batch = [ self.pending.popleft() for _ in range(min(len(self.pending), self.parallel_connections)) ] futures = [executor.submit(self._fetch, request) for request in batch] for future in as_completed(futures): self._ensure_not_cancelled() self._handle_fetch_result(future.result()) self._emit_progress("Capturing", "Discovering pages and assets") if not self._root_record: raise CaptureError(self._root_error or "The starting page could not be captured.") self._ensure_within_duration() self._emit_progress("Rewriting", "Converting captured links for offline use") self._rewrite_captured_content() self._ensure_within_duration() report = self._build_report() self._write_capture_metadata(report) self._emit_progress("Packaging", "Preparing archive contents") return CrawlResult( site_dir=self._site_dir, entry_point=self._root_record.local_path, report=report, stats=self.stats.copy(), warnings=self._visible_warnings(), ) finally: self._client.close() def _fetch(self, request: CrawlRequest) -> FetchResult: current_url = request.url redirects: list[str] = [] for _ in range(self.settings.max_redirects + 1): if self.cancelled(): return FetchResult(request, "cancelled", message="Capture cancelled") if self._duration_exceeded(): raise CaptureError("The configured capture time limit was reached.") try: self._pin_target(current_url) except UrlPolicyError as error: return FetchResult(request, "skipped", final_url=current_url, message=str(error)) if not self._robots_allow(current_url): return FetchResult( request, "skipped", final_url=current_url, message="robots.txt excludes this resource", ) try: headers = self._referer_header(request.referrer, current_url) with self._client.stream( "GET", current_url, headers=headers, timeout=self._request_timeout(), ) as response: status_code = response.status_code if status_code in {301, 302, 303, 307, 308}: location = response.headers.get("location") if not location: return FetchResult( request, "error", final_url=current_url, status_code=status_code, message="Redirect response had no Location header", ) try: next_url = normalize_url(urljoin(current_url, location), self.rules) except UrlPolicyError as error: return FetchResult( request, "skipped", final_url=current_url, message=str(error) ) if not self._scope_allows(next_url, request.kind): return FetchResult( request, "skipped", final_url=current_url, message="Redirect leaves the configured capture scope", ) redirects.append(current_url) current_url = next_url continue if status_code >= 400: return FetchResult( request, "error", final_url=current_url, status_code=status_code, message=f"Remote server returned HTTP {status_code}", ) body_or_error = self._read_limited_response(response) if isinstance(body_or_error, str): return FetchResult( request, "skipped", final_url=current_url, status_code=status_code, message=body_or_error, ) return FetchResult( request, "ok", final_url=current_url, redirects=redirects, status_code=status_code, content_type=response.headers.get("content-type", "") .split(";", 1)[0] .lower(), encoding=response.encoding, body=body_or_error, ) except httpx.HTTPError as error: return FetchResult( request, "error", final_url=current_url, message=f"Network error: {error}" ) return FetchResult(request, "skipped", final_url=current_url, message="Too many redirects") def _read_limited_response( self, response: httpx.Response, maximum_bytes: int | None = None ) -> bytes | str: chunks: list[bytes] = [] response_bytes = 0 response_limit = maximum_bytes or self.settings.max_response_bytes for chunk in response.iter_bytes(): if self.cancelled(): raise CaptureCancelled("Capture cancelled") if self._duration_exceeded(): raise CaptureError("The configured capture time limit was reached.") response_bytes += len(chunk) if response_bytes > response_limit: return "Resource exceeds the per-file size limit" with self._byte_lock: if self._byte_count + len(chunk) > self.options.max_bytes: self._limit_reached = True return "The configured total download budget was reached" self._byte_count += len(chunk) self.stats["bytes_downloaded"] = self._byte_count chunks.append(chunk) return b"".join(chunks) def _handle_fetch_result(self, result: FetchResult) -> None: request_key = canonical_url(result.request.url, self.rules) is_root = request_key == canonical_url(self.source_url, self.rules) if result.outcome == "cancelled": raise CaptureCancelled("Capture cancelled") if result.outcome != "ok" or not result.final_url or result.body is None: if is_root: self._root_error = result.message or "The starting page could not be fetched" self._record_problem(result) return local_path = self.mapper.assign(result.final_url, result.content_type) try: destination = self._safe_destination(local_path) destination.parent.mkdir(parents=True, exist_ok=True) destination.write_bytes(result.body) except OSError as error: failed_result = FetchResult( result.request, "error", final_url=result.final_url, status_code=result.status_code, message=f"Could not write this resource locally: {error}", ) if is_root: self._root_error = failed_result.message self._record_problem(failed_result) return record = ResourceRecord( requested_url=result.request.url, final_url=result.final_url, local_path=local_path, kind=result.request.kind, content_type=result.content_type, status_code=result.status_code or 200, byte_count=len(result.body), redirects=result.redirects, encoding=result.encoding, ) self.records.append(record) self.url_to_path[request_key] = local_path self.url_to_path[canonical_url(result.final_url, self.rules)] = local_path for redirect in result.redirects: self.url_to_path[canonical_url(redirect, self.rules)] = local_path self.stats["files_written"] += 1 if result.request.kind == "document": self.stats["pages_fetched"] += 1 else: self.stats["assets_fetched"] += 1 if is_root: self._root_record = record text = self._decode_text(result.body, result.encoding) if self._is_html(record, text): self._discover_html(text, record.final_url, result.request.depth) elif self._is_css(record): self._discover_css(text, record.final_url, result.request.depth) elif self._is_javascript(record): self._discover_javascript(text, record.final_url, result.request.depth) def _record_problem(self, result: FetchResult) -> None: if result.outcome == "skipped" and result.message: self._add_warning(f"Skipped {result.request.url}: {result.message}") elif result.message: self._error_count += 1 if len(self.errors) < MAX_RECORDED_ERRORS: self.errors.append( { "url": result.request.url, "kind": result.request.kind, "status_code": result.status_code, "message": result.message, } ) else: self._errors_omitted += 1 def _discover_html(self, text: str, page_url: str, depth: int) -> None: soup = BeautifulSoup(text, "html.parser") base_url = page_url base = soup.find("base", href=True) if isinstance(base, Tag): resolved = self._resolve_reference(page_url, self._tag_value(base, "href") or "") if resolved: base_url = resolved for tag in soup.find_all(True): if not isinstance(tag, Tag): continue name = (tag.name or "").lower() document_attribute = "href" if name in {"a", "area"} else "src" document_url = self._tag_value(tag, document_attribute) if name in {"a", "area", "iframe", "frame"} and document_url: attribute = "href" if name in {"a", "area"} else "src" self._enqueue_reference( self._tag_value(tag, attribute), base_url, "document", depth + 1, page_url ) link_url = self._tag_value(tag, "href") if name == "link" and link_url: rel_value = tag.get("rel") if isinstance(rel_value, list): rel = {str(item).lower() for item in rel_value} elif isinstance(rel_value, str): rel = {rel_value.lower()} else: rel = set() if not rel.intersection({"canonical", "alternate"}): self._enqueue_reference(link_url, base_url, "asset", depth, page_url) for attribute in ("src", "poster", "data", "background"): attribute_url = self._tag_value(tag, attribute) if attribute_url: kind: ResourceKind = ( "document" if name == "object" and attribute == "data" else "asset" ) self._enqueue_reference(attribute_url, base_url, kind, depth, page_url) if name in {"use", "image"}: for attribute in ("href", "xlink:href"): attribute_url = self._tag_value(tag, attribute) if attribute_url: self._enqueue_reference(attribute_url, base_url, "asset", depth, page_url) srcset = self._tag_value(tag, "srcset") if srcset: for candidate in self._srcset_urls(srcset): self._enqueue_reference(candidate, base_url, "asset", depth, page_url) style = self._tag_value(tag, "style") if style: self._discover_css(style, base_url, depth) if name == "style" and tag.string: self._discover_css(str(tag.string), base_url, depth) if name == "script" and tag.string: self._discover_javascript(str(tag.string), base_url, depth) if name == "meta": property_name = str(tag.get("property") or tag.get("name") or "").lower() content = self._tag_value(tag, "content") if content and property_name in {"og:image", "twitter:image", "og:video"}: self._enqueue_reference(content, base_url, "asset", depth, page_url) for meta in soup.find_all("meta"): if not isinstance(meta, Tag): continue content = self._tag_value(meta, "content") if str(meta.get("http-equiv", "")).lower() == "refresh" and content: match = META_REFRESH_RE.match(content) if match: self._enqueue_reference( match.group("url"), base_url, "document", depth + 1, page_url ) def _discover_css(self, text: str, base_url: str, depth: int) -> None: for match in CSS_URL_RE.finditer(text): self._enqueue_reference(match.group("url"), base_url, "asset", depth, base_url) for match in CSS_IMPORT_RE.finditer(text): self._enqueue_reference(match.group("url"), base_url, "asset", depth, base_url) def _discover_javascript(self, text: str, base_url: str, depth: int) -> None: for pattern in ( JS_DYNAMIC_IMPORT_RE, JS_STATIC_IMPORT_RE, JS_NEW_URL_RE, JS_WORKER_RE, JS_SOURCE_MAP_RE, ): for match in pattern.finditer(text): self._enqueue_reference(match.group("url"), base_url, "asset", depth, base_url) def _enqueue_reference( self, raw_url: object, base_url: str, kind: ResourceKind, depth: int, referrer: str | None ) -> None: if not isinstance(raw_url, str): return resolved = self._resolve_reference(base_url, raw_url) if resolved: self._enqueue(CrawlRequest(resolved, kind, depth, referrer)) def _enqueue(self, request: CrawlRequest) -> None: try: key = canonical_url(request.url, self.rules) except UrlPolicyError: return if key in self.enqueued or self._limit_reached: return if request.kind == "document" and request.depth > self.options.max_depth: self._add_warning(f"Skipped {request.url}: page depth limit reached") return if len(self.enqueued) >= self.settings.max_files: self._add_warning("The configured file-count safety limit was reached.") self._limit_reached = True return if request.kind == "document" and self.stats["pages_found"] >= self.options.max_pages: self._add_warning("The configured page limit was reached.") return if not self._scope_allows(key, request.kind): return if request.kind == "asset" and not self._asset_is_allowed(key): return self.enqueued.add(key) self.pending.append(CrawlRequest(key, request.kind, request.depth, request.referrer)) if request.kind == "document": self.stats["pages_found"] += 1 else: self.stats["assets_found"] += 1 def _scope_allows(self, url: str, kind: ResourceKind) -> bool: candidate_host = hostname(url) if same_site_host(candidate_host, self.source_host): return True if kind == "document": return False if not self.options.include_external_assets: return False with self._scope_lock: if candidate_host not in self._external_hosts: if len(self._external_hosts) >= self.settings.max_external_hosts: self._add_warning( "External asset host limit reached; remaining third-party assets " "were skipped." ) return False self._external_hosts.add(candidate_host) return True def _asset_is_allowed(self, url: str) -> bool: extension = posixpath.splitext(urlsplit(url).path.lower())[1].lstrip(".") if extension and extension in self.options.skip_file_types: self._add_warning(f"Skipped .{extension} files because of the configured file rule.") return False asset_type = self._asset_type(url) if asset_type not in self.options.download_asset_types: self._add_warning( f"Skipped {asset_type} assets because they are not selected for this capture." ) return False return True @staticmethod def _asset_type(url: str) -> str: extension = posixpath.splitext(urlsplit(url).path.lower())[1].lstrip(".") if extension == "css": return "styles" if extension in {"js", "mjs", "cjs", "map"}: return "scripts" if extension in {"avif", "bmp", "gif", "ico", "jpeg", "jpg", "png", "svg", "webp"}: return "images" if extension in {"eot", "otf", "ttf", "woff", "woff2"}: return "fonts" if extension in { "aac", "avi", "m4a", "mkv", "mov", "mp3", "mp4", "mpeg", "oga", "ogg", "ogv", "wav", "webm", }: return "media" return "other" def _robots_allow(self, url: str) -> bool: if not self.settings.respect_robots: return True parsed = urlsplit(url) origin = f"{parsed.scheme}://{parsed.netloc}" with self._robots_lock: cached = self._robots.get(origin) known_origin = origin in self._robots if not known_origin: loaded = self._load_robots(origin) with self._robots_lock: cached = self._robots.setdefault(origin, loaded) if isinstance(cached, RobotFileParser): return cached.can_fetch(self.settings.user_agent, url) return True def _load_robots(self, origin: str) -> RobotFileParser | bool: robots_url = f"{origin}/robots.txt" parser = RobotFileParser() parser.set_url(robots_url) try: self._pin_target(robots_url) with self._client.stream( "GET", robots_url, follow_redirects=False, timeout=self._request_timeout(), ) as response: if response.status_code == 200: body = self._read_limited_response(response, maximum_bytes=512 * 1024) if isinstance(body, bytes): parser.parse(self._decode_text(body, response.encoding).splitlines()) return parser return True if response.status_code in {401, 403}: parser.parse(["User-agent: *", "Disallow: /"]) return parser except (httpx.HTTPError, UrlPolicyError): return True return True def _rewrite_captured_content(self) -> None: for record in self.records: self._ensure_not_cancelled() self._ensure_within_duration() destination = self._safe_destination(record.local_path) if self._is_html(record): text = self._decode_text(destination.read_bytes(), record.encoding) destination.write_text(self._rewrite_html(text, record), encoding="utf-8") elif self._is_css(record): text = self._decode_text(destination.read_bytes(), record.encoding) rewritten = self._rewrite_css(text, record.local_path, record.final_url) rewritten = CSS_CHARSET_RE.sub('@charset "UTF-8";', rewritten, count=1) destination.write_text( rewritten, encoding="utf-8", ) elif self._is_javascript(record): text = self._decode_text(destination.read_bytes(), record.encoding) destination.write_text( self._rewrite_javascript(text, record.local_path, record.final_url), encoding="utf-8", ) def _rewrite_html(self, text: str, record: ResourceRecord) -> str: soup = BeautifulSoup(text, "html.parser") base_url = record.final_url base = soup.find("base", href=True) if isinstance(base, Tag): resolved = self._resolve_reference(base_url, self._tag_value(base, "href") or "") if resolved: base_url = resolved base.decompose() for meta in soup.find_all("meta", charset=True): if isinstance(meta, Tag): meta["charset"] = "utf-8" for meta in soup.find_all("meta"): if not isinstance(meta, Tag): continue if str(meta.get("http-equiv", "")).lower() != "content-type": continue content = self._tag_value(meta, "content") if content and "charset=" in content.lower(): meta["content"] = re.sub( r"charset\s*=\s*[^;\s]+", "charset=UTF-8", content, flags=re.I ) for tag in soup.find_all(True): if not isinstance(tag, Tag): continue for attribute in ("href", "src", "poster", "data", "background", "xlink:href"): attribute_url = self._tag_value(tag, attribute) if attribute_url: tag[attribute] = self._local_reference( record.local_path, base_url, attribute_url ) srcset = self._tag_value(tag, "srcset") if srcset: tag["srcset"] = self._rewrite_srcset(record.local_path, base_url, srcset) style = self._tag_value(tag, "style") if style: tag["style"] = self._rewrite_css(style, record.local_path, base_url) name = (tag.name or "").lower() if name == "style" and tag.string: style_text = str(tag.string) tag.clear() tag.append(self._rewrite_css(style_text, record.local_path, base_url)) if name == "script" and tag.string: script_text = str(tag.string) tag.clear() tag.append(self._rewrite_javascript(script_text, record.local_path, base_url)) for meta in soup.find_all("meta"): if not isinstance(meta, Tag): continue content = self._tag_value(meta, "content") if str(meta.get("http-equiv", "")).lower() == "refresh" and content: match = META_REFRESH_RE.match(content) if match: meta["content"] = match.group("delay") + self._local_reference( record.local_path, base_url, match.group("url") ) return str(soup) def _rewrite_css(self, text: str, local_path: str, base_url: str) -> str: def replace(match: re.Match[str]) -> str: quote = match.group("quote") raw = match.group("url") local = self._local_reference(local_path, base_url, raw) return f"url({quote}{local}{quote})" def replace_import(match: re.Match[str]) -> str: quote = match.group("quote") local = self._local_reference(local_path, base_url, match.group("url")) return f"@import {quote}{local}{quote}" return CSS_IMPORT_RE.sub(replace_import, CSS_URL_RE.sub(replace, text)) def _rewrite_javascript(self, text: str, local_path: str, base_url: str) -> str: def replace_quoted(match: re.Match[str]) -> str: local = self._local_reference(local_path, base_url, match.group("url")) prefix = match.group(0)[: match.start("url") - match.start(0)] suffix = match.group(0)[match.end("url") - match.start(0) :] return f"{prefix}{local}{suffix}" def replace_source_map(match: re.Match[str]) -> str: return match.group(0).replace( match.group("url"), self._local_reference(local_path, base_url, match.group("url")) ) text = JS_DYNAMIC_IMPORT_RE.sub(replace_quoted, text) text = JS_STATIC_IMPORT_RE.sub(replace_quoted, text) text = JS_NEW_URL_RE.sub(replace_quoted, text) text = JS_WORKER_RE.sub(replace_quoted, text) return JS_SOURCE_MAP_RE.sub(replace_source_map, text) def _local_reference(self, from_path: str, base_url: str, raw_url: str) -> str: stripped = raw_url.strip() if not stripped or stripped.startswith("#") or self._is_non_fetchable_reference(stripped): return raw_url absolute = urljoin(base_url, stripped) try: target_key = canonical_url(absolute, self.rules) except UrlPolicyError: return raw_url target_path = self.url_to_path.get(target_key) if not target_path: return raw_url relative = posixpath.relpath(target_path, start=posixpath.dirname(from_path) or ".") fragment = urlsplit(absolute).fragment return f"{relative}#{fragment}" if fragment else relative def _rewrite_srcset(self, local_path: str, base_url: str, value: str) -> str: candidates: list[str] = [] for candidate in value.split(","): pieces = candidate.strip().split(maxsplit=1) if not pieces: continue rewritten = self._local_reference(local_path, base_url, pieces[0]) candidates.append(" ".join([rewritten, *pieces[1:]])) return ", ".join(candidates) def _build_report(self) -> dict[str, object]: elapsed_seconds = round(time.monotonic() - self._started_at, 2) quality = "partial" if self._limit_reached or self.errors or self.warnings else "complete" return { "siteharbor_version": "0.1.0", "source_url": self.source_url, "entry_point": self._root_record.local_path if self._root_record else None, "quality": quality, "elapsed_seconds": elapsed_seconds, "options": asdict(self.options), "stats": self.stats, "warnings": self._visible_warnings(), "warning_count": self._warning_count, "errors": self._visible_errors(), "error_count": self._error_count, "resources": [record.as_manifest() for record in self.records], } def _write_capture_metadata(self, report: dict[str, object]) -> None: if not self._site_dir or not self._root_record: raise CaptureError("Capture output was not initialized.") (self._site_dir / "siteharbor-manifest.json").write_text( json.dumps(report, indent=2, ensure_ascii=True), encoding="utf-8" ) readme = "\n".join( [ "SiteHarbor offline capture", "", f"Entry point: {self._root_record.local_path}", f"Source URL: {self.source_url}", f"Quality: {report['quality']}", "", "This is a bounded static capture. Dynamic APIs, authenticated content,", "service workers, and runtime-generated behavior may not work offline.", "See siteharbor-manifest.json for captured, skipped, and failed resources.", "", ] ) (self._site_dir / "README.txt").write_text(readme, encoding="utf-8") def _emit_progress(self, phase: str, message: str) -> None: self.on_progress(phase, message, self.stats.copy(), self._visible_warnings()) def _ensure_not_cancelled(self) -> None: if self.cancelled(): raise CaptureCancelled("Capture cancelled") def _ensure_within_duration(self) -> None: if self._duration_exceeded(): raise CaptureError("The configured capture time limit was reached.") def _duration_exceeded(self) -> bool: return self._remaining_seconds() <= 0 def _remaining_seconds(self) -> float: elapsed = time.monotonic() - self._started_at return self.options.max_duration_seconds - elapsed def _request_timeout(self) -> httpx.Timeout: remaining = max(0.05, self._remaining_seconds()) return httpx.Timeout( connect=min(10.0, remaining), read=min(20.0, remaining), write=min(10.0, remaining), pool=min(10.0, remaining), ) def _pin_target(self, url: str) -> None: addresses = resolve_and_validate(url, self.rules) parsed = urlsplit(url) port = parsed.port or (443 if parsed.scheme == "https" else 80) self._address_book.pin(hostname(url), port, addresses) def _pin_proxy(self) -> None: if not self._proxy_url: return try: proxy_url = normalize_url(self._proxy_url, self.rules) addresses = resolve_and_validate(proxy_url, self.rules) except UrlPolicyError as error: raise CaptureError("The configured proxy endpoint is not allowed.") from error parsed = urlsplit(proxy_url) port = parsed.port or (443 if parsed.scheme == "https" else 80) self._address_book.pin(hostname(proxy_url), port, addresses) @staticmethod def _referer_header(referrer: str | None, target_url: str) -> dict[str, str] | None: if not referrer: return None try: if not same_site_host(hostname(referrer), hostname(target_url)): return None except UrlPolicyError: return None parsed = urlsplit(referrer) return {"Referer": f"{parsed.scheme}://{parsed.netloc}/"} def _add_warning(self, message: str) -> None: self._warning_count += 1 if message not in self.warnings: if len(self.warnings) < MAX_RECORDED_WARNINGS: self.warnings.append(message) else: self._warnings_omitted += 1 def _visible_warnings(self) -> list[str]: visible = self.warnings.copy() if self._warnings_omitted: visible.append( f"{self._warnings_omitted} additional warnings were omitted from this report." ) return visible def _visible_errors(self) -> list[dict[str, object]]: visible = self.errors.copy() if self._errors_omitted: visible.append( { "url": "", "kind": "summary", "status_code": None, "message": f"{self._errors_omitted} additional resource failures were omitted.", } ) return visible def _safe_destination(self, local_path: str) -> Path: if not self._site_dir: raise CaptureError("Capture output was not initialized.") destination = self._site_dir.joinpath(*local_path.split("/")) resolved_root = self._site_dir.resolve() resolved_destination = destination.resolve() if not resolved_destination.is_relative_to(resolved_root): raise CaptureError("A captured path escaped the archive root.") return destination def _resolve_reference(self, base_url: str, raw_url: str) -> str | None: value = raw_url.strip() if not value or value.startswith("#") or self._is_non_fetchable_reference(value): return None try: return normalize_url(urljoin(base_url, value), self.rules) except UrlPolicyError: return None @staticmethod def _is_non_fetchable_reference(value: str) -> bool: return value.lower().startswith( ("data:", "javascript:", "mailto:", "tel:", "blob:", "about:") ) @staticmethod def _decode_text(body: bytes, encoding: str | None) -> str: encodings = [encoding, "utf-8", "utf-8-sig", "latin-1"] for candidate in encodings: if not candidate: continue try: return body.decode(candidate) except (LookupError, UnicodeDecodeError): continue return body.decode("utf-8", errors="replace") @staticmethod def _srcset_urls(value: str) -> list[str]: return [ candidate.strip().split(maxsplit=1)[0] for candidate in value.split(",") if candidate.strip() ] @staticmethod def _tag_value(tag: Tag, attribute: str) -> str | None: value = tag.get(attribute) return value if isinstance(value, str) else None @staticmethod def _is_html(record: ResourceRecord, text: str | None = None) -> bool: if record.content_type in {"text/html", "application/xhtml+xml"}: return True return bool(text and " bool: return record.content_type == "text/css" or record.local_path.lower().endswith(".css") @staticmethod def _is_javascript(record: ResourceRecord) -> bool: return "javascript" in record.content_type or record.local_path.lower().endswith( (".js", ".mjs", ".cjs") )