commit 07b953fed2ff47d981d05951bb65ff2480607836 Author: kstyagi23 Date: Sat Sep 5 06:07:25 2026 +0530 __init__ diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..c0e85f0 --- /dev/null +++ b/.gitignore @@ -0,0 +1,22 @@ +# Python +__pycache__/ +*.py[cod] +*.pyo +.pytest_cache/ +.ruff_cache/ +.coverage +htmlcov/ + +# Environments +.venv/ +venv/ +env/ + +# SiteHarbor runtime state +data/ + +# Editor and operating-system files +.idea/ +.vscode/ +.DS_Store +Thumbs.db diff --git a/README.md b/README.md new file mode 100644 index 0000000..2ae0619 --- /dev/null +++ b/README.md @@ -0,0 +1,103 @@ +# SiteHarbor + +SiteHarbor is a standalone Python web-capture service. It creates a bounded offline archive of a public website, preserves discovered static assets, rewrites supported local links, records a manifest, and keeps completed ZIP files outside the public web root. + +It is intentionally designed as a local or trusted-network application. It uses SQLite for durable job state and does not require Redis, a queue server, or object storage. + +## What It Does + +```text +Browser -> FastAPI -> SQLite job queue -> local capture worker + -> private local ZIP and JSON report -> browser-scoped download route +``` + +The browser creates an opaque local session key on first use. It is stored in local storage and +scopes capture status, cancellation, reports, and downloads to that browser. The worker and SQLite +job continue independently when the page disconnects; reopening the page reconnects to locally +recorded sessions. Clearing browser storage intentionally removes access to those sessions. + +The capture engine: + +- Crawls same-site documents within configured page, depth, size, and time budgets. +- Captures linked CSS, JavaScript, image, font, media, manifest, SVG, iframe, and CSS-import resources. +- Can include linked third-party assets while avoiding third-party document crawling. +- Rewrites supported HTML, CSS, and static JavaScript references to local archive paths. +- Produces `siteharbor-manifest.json` and `README.txt` inside each archive. +- Reports skipped resources, limits, failures, redirects, and static-capture caveats. + +No static crawler can guarantee a fully functioning offline copy of every modern web application. Dynamic API calls, authenticated content, service workers, CAPTCHAs, backend state, and runtime-generated resources need a separately isolated browser-rendered mode in a later release. + +## Quick Start + +Use Python 3.11 or newer. + +```powershell +py -3.11 -m venv .venv +.\.venv\Scripts\Activate.ps1 +python -m pip install --upgrade pip +python -m pip install -e .[dev] +siteharbor +``` + +Open `http://127.0.0.1:8787`. + +To use Uvicorn directly: + +```powershell +python -m uvicorn app.main:app --host 127.0.0.1 --port 8787 +``` + +The default bind address is loopback. Do not expose SiteHarbor directly to the public Internet without authentication, a network egress policy, rate limits, and operational controls. + +## Configuration + +All configuration uses environment variables prefixed with `SITEHARBOR_`. + +| Variable | Default | Purpose | +| --- | --- | --- | +| `SITEHARBOR_HOST` | `127.0.0.1` | Bind address used by `siteharbor`. | +| `SITEHARBOR_PORT` | `8787` | Bind port used by `siteharbor`. | +| `SITEHARBOR_DATA_DIR` | `./data` | SQLite database, work directories, artifacts, and reports. | +| `SITEHARBOR_WORKER_CONCURRENCY` | `1` | Capture workers per application process. | +| `SITEHARBOR_FETCH_CONCURRENCY` | `6` | Maximum parallel fetches a visitor may select for one capture. | +| `SITEHARBOR_LEASE_SECONDS` | `120` | SQLite worker lease duration; values below 15 seconds are rejected. | +| `SITEHARBOR_ALLOW_PRIVATE_NETWORKS` | `false` | Development-only override that permits loopback and private hosts. | +| `SITEHARBOR_ALLOW_NONSTANDARD_PORTS` | `false` | Development-only override for ports other than 80 and 443. | +| `SITEHARBOR_RESPECT_ROBOTS` | `true` | Respect robots rules during capture. | +| `SITEHARBOR_DEFAULT_RETENTION_DAYS` | `7` | Default retention period for completed artifacts. | +| `SITEHARBOR_PROXY_URL` | unset | Operator-approved HTTP(S) proxy route. Visitors can only opt into this configured route; they cannot supply proxy endpoints. | + +For a local fixture site on `localhost:8000`, use both development overrides: + +```powershell +$env:SITEHARBOR_ALLOW_PRIVATE_NETWORKS = "true" +$env:SITEHARBOR_ALLOW_NONSTANDARD_PORTS = "true" +siteharbor +``` + +## SQLite and Uvicorn Workers + +SiteHarbor uses SQLite WAL mode and a lease-based database claim to run jobs without Redis. A single Uvicorn process with `SITEHARBOR_WORKER_CONCURRENCY=1` is the recommended standalone configuration. + +Multiple Uvicorn processes can claim jobs from the same local SQLite database, but SQLite still permits only one writer at a time. Use modest concurrency and shared local storage only. This design is intentionally for standalone or small trusted deployments, not high-scale public crawling. + +## Safety Boundaries + +The application blocks non-HTTP(S) targets, URL credentials, nonstandard ports by default, and private/reserved network destinations. It revalidates and pins each request target to validated numeric addresses during crawling and redirects, avoiding an uncontrolled second DNS lookup at connection time. + +If the service is ever exposed beyond localhost, still route worker traffic through a controlled public-only egress gateway and add authentication, quotas, and rate limiting. The standalone safeguards are intentionally strong, but a network boundary remains defense in depth for a public service. + +## Browser Sessions and Public Metrics + +- No account, login, or signup flow is required. +- The public page exposes aggregate counts only: successful captures, data collected, active crawls, and files preserved. +- Private capture routes require the opaque browser-local session key. A different browser cannot list, inspect, cancel, delete, download, or open events for another browser's captures. +- This local session key provides browser ownership, not public-service abuse protection. A public deployment still needs rate limits, quotas, egress controls, and monitoring. + +## Development Commands + +```powershell +python -m pytest +python -m ruff check . +python -m compileall app +``` diff --git a/app/__init__.py b/app/__init__.py new file mode 100644 index 0000000..0794441 --- /dev/null +++ b/app/__init__.py @@ -0,0 +1 @@ +"""SiteHarbor application package.""" diff --git a/app/capture.py b/app/capture.py new file mode 100644 index 0000000..c519242 --- /dev/null +++ b/app/capture.py @@ -0,0 +1,302 @@ +from __future__ import annotations + +import json +import os +import shutil +import threading +import time +import uuid +import zipfile +from collections.abc import Callable +from pathlib import Path +from typing import Any + +from app.config import Settings +from app.crawler import ( + DEFAULT_ASSET_TYPES, + CaptureCancelled, + CaptureError, + CaptureOptions, + SiteCrawler, +) +from app.db import Database +from app.url_policy import NetworkRules, UrlPolicyError, resolve_and_validate + + +def _remove_tree(path: Path, allowed_root: Path) -> None: + """Delete only a child path beneath the configured work root.""" + root = allowed_root.resolve() + resolved = path.resolve() + if resolved == root or not resolved.is_relative_to(root): + raise RuntimeError("Refusing to remove a path outside SiteHarbor's work directory.") + shutil.rmtree(resolved, ignore_errors=True) + + +def _create_archive( + source_dir: Path, + destination: Path, + archive_root: str, + cancelled: Callable[[], bool], + deadline_exceeded: Callable[[], bool], + heartbeat: Callable[[], None], +) -> int: + partial = destination.with_suffix(".zip.partial") + partial.unlink(missing_ok=True) + last_heartbeat = time.monotonic() + try: + with zipfile.ZipFile( + partial, + mode="w", + compression=zipfile.ZIP_DEFLATED, + compresslevel=6, + strict_timestamps=False, + ) as archive: + for source_path in sorted(source_dir.rglob("*")): + if cancelled(): + raise CaptureCancelled("Capture cancelled while packaging the archive") + if deadline_exceeded(): + raise CaptureError( + "The configured capture time limit was reached while packaging" + ) + if time.monotonic() - last_heartbeat >= 5: + heartbeat() + last_heartbeat = time.monotonic() + if not source_path.is_file() or source_path.is_symlink(): + continue + relative = source_path.relative_to(source_dir).as_posix() + with source_path.open("rb") as source_file: + with archive.open(f"{archive_root}/{relative}", mode="w") as archive_file: + while chunk := source_file.read(1_024 * 1_024): + if cancelled(): + raise CaptureCancelled( + "Capture cancelled while packaging the archive" + ) + if deadline_exceeded(): + raise CaptureError( + "The configured capture time limit was reached while packaging" + ) + if time.monotonic() - last_heartbeat >= 5: + heartbeat() + last_heartbeat = time.monotonic() + archive_file.write(chunk) + os.replace(partial, destination) + return destination.stat().st_size + except Exception: + partial.unlink(missing_ok=True) + destination.unlink(missing_ok=True) + raise + + +class CaptureService: + """Runs one leased capture and persists a terminal SQLite job state.""" + + def __init__(self, database: Database, settings: Settings, worker_id: str) -> None: + self.database = database + self.settings = settings + self.worker_id = worker_id + + def run(self, job: dict[str, Any]) -> None: + job_id = str(job["id"]) + config = dict(job["config"]) + attempt_id = uuid.uuid4().hex[:12] + work_dir = self.settings.work_dir / f"{job_id}-{attempt_id}" + artifact_path = self.settings.artifacts_dir / f"{job_id}-{attempt_id}.zip" + report_path = self.settings.reports_dir / f"{job_id}-{attempt_id}.json" + heartbeat_stop = threading.Event() + lease_lost = threading.Event() + heartbeat_thread: threading.Thread | None = None + capture_deadline = time.monotonic() + int(config["max_duration_seconds"]) + + def capture_cancelled() -> bool: + return ( + lease_lost.is_set() + or self.database.is_cancel_requested(job_id) + or not self.database.owns_lease(job_id, self.worker_id) + ) + + def renew_lease() -> None: + interval = min(20, max(3, self.settings.lease_seconds // 3)) + while not heartbeat_stop.wait(interval): + try: + if not self.database.renew_lease( + job_id, self.worker_id, self.settings.lease_seconds + ): + lease_lost.set() + return + except Exception: + # Ownership checks stop this attempt if the lease eventually expires. + continue + + try: + _remove_tree_if_present(work_dir, self.settings.work_dir) + work_dir.mkdir(parents=True, exist_ok=True) + if capture_cancelled(): + raise CaptureCancelled("Capture cancelled before it started") + + heartbeat_thread = threading.Thread( + target=renew_lease, + name=f"siteharbor-lease-{job_id[-8:]}", + daemon=True, + ) + heartbeat_thread.start() + + rules = NetworkRules( + allow_private_networks=self.settings.allow_private_networks, + allow_nonstandard_ports=self.settings.allow_nonstandard_ports, + ) + resolve_and_validate(str(job["normalized_url"]), rules) + remaining_seconds = int(capture_deadline - time.monotonic()) + if remaining_seconds <= 0: + raise CaptureError( + "The configured capture time limit was reached during validation" + ) + options = CaptureOptions( + source_url=str(job["normalized_url"]), + include_external_assets=bool(config["include_external_assets"]), + max_pages=int(config["max_pages"]), + max_depth=int(config["max_depth"]), + max_bytes=int(config["max_bytes"]), + max_duration_seconds=remaining_seconds, + parallel_connections=int( + config.get("parallel_connections", self.settings.fetch_concurrency) + ), + download_asset_types=tuple( + config.get("download_asset_types", DEFAULT_ASSET_TYPES) + ), + skip_file_types=tuple(config.get("skip_file_types", ())), + use_proxy=bool(config.get("use_proxy", False)), + ) + + def update( + phase: str, message: str, stats: dict[str, int], warnings: list[str] + ) -> None: + renewed = self.database.update_progress( + job_id, + self.worker_id, + self.settings.lease_seconds, + phase=phase, + message=message, + stats=stats, + warnings=warnings, + ) + if not renewed: + raise CaptureCancelled("Capture lease was reassigned") + + if not self.database.update_progress( + job_id, + self.worker_id, + self.settings.lease_seconds, + phase="Validating", + message="Checking capture policy and target reachability", + stats=_job_stats(job), + warnings=[], + ): + raise CaptureCancelled("Capture lease was reassigned") + + crawler = SiteCrawler( + settings=self.settings, + options=options, + cancelled=capture_cancelled, + on_progress=update, + ) + result = crawler.run(work_dir) + + if capture_cancelled(): + raise CaptureCancelled("Capture cancelled before packaging") + if time.monotonic() >= capture_deadline: + raise CaptureError("The configured capture time limit was reached before packaging") + + update("Packaging", "Creating a portable ZIP archive", result.stats, result.warnings) + archive_size = _create_archive( + result.site_dir, + artifact_path, + archive_root=f"siteharbor-{job_id}", + cancelled=capture_cancelled, + deadline_exceeded=lambda: time.monotonic() >= capture_deadline, + heartbeat=lambda: update( + "Packaging", + "Creating a portable ZIP archive", + result.stats, + result.warnings, + ), + ) + + report = { + **result.report, + "job_id": job_id, + "archive": { + "filename": artifact_path.name, + "size_bytes": archive_size, + "archive_root": f"siteharbor-{job_id}", + }, + } + if time.monotonic() >= capture_deadline: + raise CaptureError( + "The configured capture time limit was reached before completion" + ) + report_path.write_text( + json.dumps(report, indent=2, ensure_ascii=True), encoding="utf-8" + ) + marked_ready = self.database.mark_ready( + job_id, + self.worker_id, + artifact_path=str(artifact_path), + report_path=str(report_path), + archive_size=archive_size, + entry_point=result.entry_point, + stats=result.stats, + warnings=result.warnings, + retention_days=int(config["retention_days"]), + ) + if not marked_ready: + artifact_path.unlink(missing_ok=True) + report_path.unlink(missing_ok=True) + if self.database.is_cancel_requested(job_id): + self.database.mark_cancelled( + job_id, + self.worker_id, + "Capture cancelled before the archive became available", + ) + except CaptureCancelled as error: + artifact_path.unlink(missing_ok=True) + report_path.unlink(missing_ok=True) + if self.database.is_cancel_requested(job_id): + self.database.mark_cancelled(job_id, self.worker_id, str(error)) + except UrlPolicyError as error: + artifact_path.unlink(missing_ok=True) + self.database.mark_failed(job_id, self.worker_id, "target_blocked", str(error)) + except CaptureError as error: + artifact_path.unlink(missing_ok=True) + self.database.mark_failed(job_id, self.worker_id, "capture_failed", str(error)) + except Exception as error: + artifact_path.unlink(missing_ok=True) + self.database.mark_failed( + job_id, + self.worker_id, + "internal_error", + f"Capture failed: {error}", + ) + finally: + heartbeat_stop.set() + if heartbeat_thread: + heartbeat_thread.join(timeout=1) + try: + _remove_tree_if_present(work_dir, self.settings.work_dir) + except OSError: + pass + + +def _job_stats(job: dict[str, Any]) -> dict[str, int]: + return { + "pages_found": int(job.get("pages_found", 0)), + "pages_fetched": int(job.get("pages_fetched", 0)), + "assets_found": int(job.get("assets_found", 0)), + "assets_fetched": int(job.get("assets_fetched", 0)), + "bytes_downloaded": int(job.get("bytes_downloaded", 0)), + "files_written": int(job.get("files_written", 0)), + } + + +def _remove_tree_if_present(path: Path, root: Path) -> None: + if path.exists(): + _remove_tree(path, root) diff --git a/app/config.py b/app/config.py new file mode 100644 index 0000000..f05f1c4 --- /dev/null +++ b/app/config.py @@ -0,0 +1,102 @@ +from __future__ import annotations + +import os +from dataclasses import dataclass +from pathlib import Path + + +def _as_bool(name: str, default: bool) -> bool: + value = os.getenv(name) + if value is None: + return default + return value.strip().lower() in {"1", "true", "yes", "on"} + + +def _as_int(name: str, default: int, minimum: int = 1) -> int: + value = os.getenv(name) + if value is None: + return default + try: + return max(minimum, int(value)) + except ValueError: + return default + + +@dataclass(frozen=True, slots=True) +class Settings: + project_root: Path + data_dir: Path + database_path: Path + work_dir: Path + artifacts_dir: Path + reports_dir: Path + host: str + port: int + worker_enabled: bool + worker_concurrency: int + fetch_concurrency: int + queue_poll_seconds: float + lease_seconds: int + allow_private_networks: bool + allow_nonstandard_ports: bool + respect_robots: bool + default_retention_days: int + max_retention_days: int + max_pages_cap: int + max_depth_cap: int + max_bytes_cap: int + max_duration_seconds_cap: int + max_response_bytes: int + max_files: int + max_external_hosts: int + max_redirects: int + user_agent: str + proxy_url: str | None + + @classmethod + def from_environment(cls) -> Settings: + project_root = Path(__file__).resolve().parents[1] + data_dir = Path(os.getenv("SITEHARBOR_DATA_DIR", project_root / "data")).expanduser() + if not data_dir.is_absolute(): + data_dir = project_root / data_dir + + return cls( + project_root=project_root, + data_dir=data_dir, + database_path=data_dir / "siteharbor.sqlite3", + work_dir=data_dir / "work", + artifacts_dir=data_dir / "artifacts", + reports_dir=data_dir / "reports", + host=os.getenv("SITEHARBOR_HOST", "127.0.0.1"), + port=_as_int("SITEHARBOR_PORT", 8787), + worker_enabled=_as_bool("SITEHARBOR_WORKER_ENABLED", True), + worker_concurrency=_as_int("SITEHARBOR_WORKER_CONCURRENCY", 1), + fetch_concurrency=_as_int("SITEHARBOR_FETCH_CONCURRENCY", 6), + queue_poll_seconds=max(0.1, float(os.getenv("SITEHARBOR_QUEUE_POLL_SECONDS", "0.75"))), + lease_seconds=_as_int("SITEHARBOR_LEASE_SECONDS", 120, minimum=15), + allow_private_networks=_as_bool("SITEHARBOR_ALLOW_PRIVATE_NETWORKS", False), + allow_nonstandard_ports=_as_bool("SITEHARBOR_ALLOW_NONSTANDARD_PORTS", False), + respect_robots=_as_bool("SITEHARBOR_RESPECT_ROBOTS", True), + default_retention_days=_as_int("SITEHARBOR_DEFAULT_RETENTION_DAYS", 7), + max_retention_days=_as_int("SITEHARBOR_MAX_RETENTION_DAYS", 30), + max_pages_cap=_as_int("SITEHARBOR_MAX_PAGES", 1_000), + max_depth_cap=_as_int("SITEHARBOR_MAX_DEPTH", 12), + max_bytes_cap=_as_int("SITEHARBOR_MAX_BYTES", 1024 * 1024 * 1024), + max_duration_seconds_cap=_as_int("SITEHARBOR_MAX_DURATION_SECONDS", 3_600), + max_response_bytes=_as_int("SITEHARBOR_MAX_RESPONSE_BYTES", 32 * 1024 * 1024), + max_files=_as_int("SITEHARBOR_MAX_FILES", 5_000), + max_external_hosts=_as_int("SITEHARBOR_MAX_EXTERNAL_HOSTS", 20), + max_redirects=_as_int("SITEHARBOR_MAX_REDIRECTS", 10), + user_agent=os.getenv( + "SITEHARBOR_USER_AGENT", + "SiteHarbor/0.1 (+local standalone capture; respect robots.txt)", + ), + proxy_url=os.getenv("SITEHARBOR_PROXY_URL", "").strip() or None, + ) + + def ensure_directories(self) -> None: + for directory in (self.data_dir, self.work_dir, self.artifacts_dir, self.reports_dir): + directory.mkdir(parents=True, exist_ok=True) + + +settings = Settings.from_environment() diff --git a/app/crawler.py b/app/crawler.py new file mode 100644 index 0000000..c9f14e8 --- /dev/null +++ b/app/crawler.py @@ -0,0 +1,1101 @@ +from __future__ import annotations + +import hashlib +import json +import mimetypes +import posixpath +import re +import threading +import time +from collections import deque +from collections.abc import Callable +from concurrent.futures import ThreadPoolExecutor, as_completed +from dataclasses import asdict, dataclass, field +from pathlib import Path +from typing import Literal +from urllib.parse import urljoin, urlsplit +from urllib.robotparser import RobotFileParser + +import httpx +from bs4 import BeautifulSoup +from bs4.element import Tag + +from app.config import Settings +from app.models import SUPPORTED_ASSET_TYPES +from app.pinned_transport import PinnedAddressBook, PinnedHTTPTransport +from app.url_policy import ( + NetworkRules, + UrlPolicyError, + canonical_url, + hostname, + normalize_url, + resolve_and_validate, + same_site_host, +) + +ResourceKind = Literal["document", "asset"] +DEFAULT_ASSET_TYPES = SUPPORTED_ASSET_TYPES + +CSS_URL_RE = re.compile(r"url\(\s*(?P['\"]?)(?P.*?)(?P=quote)\s*\)", re.IGNORECASE) +CSS_IMPORT_RE = re.compile( + r"@import\s+(?:url\(\s*)?(?P['\"])(?P.*?)(?P=quote)\s*\)?", + re.IGNORECASE, +) +CSS_CHARSET_RE = re.compile(r"^\s*@charset\s+(['\"])[^'\"]+\1\s*;", re.IGNORECASE) +JS_DYNAMIC_IMPORT_RE = re.compile( + r"\bimport\s*\(\s*(?P['\"])(?P[^'\"]+)(?P=quote)\s*\)" +) +JS_STATIC_IMPORT_RE = re.compile( + r"\b(?:import|export)\s+(?:[^'\"]*?\s+from\s+)?" + r"(?P['\"])(?P[^'\"]+)(?P=quote)" +) +JS_NEW_URL_RE = re.compile( + r"\bnew\s+URL\(\s*(?P['\"])(?P[^'\"]+)(?P=quote)\s*,\s*import\.meta\.url\s*\)" +) +JS_WORKER_RE = re.compile( + r"\bnew\s+(?:Worker|SharedWorker)\s*\(\s*" + r"(?P['\"])(?P[^'\"]+)(?P=quote)" +) +JS_SOURCE_MAP_RE = re.compile(r"(?://[#@]\s*sourceMappingURL=)(?P\S+)") +META_REFRESH_RE = re.compile(r"^(?P\s*\d+(?:\.\d+)?\s*;?\s*url\s*=\s*)(?P.+)$", re.I) +MAX_RECORDED_WARNINGS = 100 +MAX_RECORDED_ERRORS = 250 +WINDOWS_RESERVED_NAMES = { + "CON", + "PRN", + "AUX", + "NUL", + *(f"COM{number}" for number in range(1, 10)), + *(f"LPT{number}" for number in range(1, 10)), +} + + +class CaptureError(RuntimeError): + """A capture cannot produce a valid archive.""" + + +class CaptureCancelled(CaptureError): + """The user cancelled a capture while it was running.""" + + +@dataclass(frozen=True, slots=True) +class CaptureOptions: + source_url: str + include_external_assets: bool + max_pages: int + max_depth: int + max_bytes: int + max_duration_seconds: int + parallel_connections: int = 6 + download_asset_types: tuple[str, ...] = DEFAULT_ASSET_TYPES + skip_file_types: tuple[str, ...] = () + use_proxy: bool = False + + +@dataclass(frozen=True, slots=True) +class CrawlRequest: + url: str + kind: ResourceKind + depth: int + referrer: str | None = None + + +@dataclass(slots=True) +class FetchResult: + request: CrawlRequest + outcome: Literal["ok", "skipped", "error", "cancelled"] + final_url: str | None = None + redirects: list[str] = field(default_factory=list) + status_code: int | None = None + content_type: str = "" + encoding: str | None = None + body: bytes | None = None + message: str = "" + + +@dataclass(slots=True) +class ResourceRecord: + requested_url: str + final_url: str + local_path: str + kind: ResourceKind + content_type: str + status_code: int + byte_count: int + redirects: list[str] + encoding: str | None + + def as_manifest(self) -> dict[str, object]: + return asdict(self) + + +@dataclass(slots=True) +class CrawlResult: + site_dir: Path + entry_point: str + report: dict[str, object] + stats: dict[str, int] + warnings: list[str] + + +class PathMapper: + """Map origin URLs to safe, mostly human-readable archive-relative paths.""" + + def __init__(self, rules: NetworkRules) -> None: + self.rules = rules + self._paths: dict[str, str] = {} + self._used_path_keys: set[str] = set() + + def assign(self, url: str, content_type: str) -> str: + key = canonical_url(url, self.rules) + if key in self._paths: + return self._paths[key] + + parsed = urlsplit(key) + host = self._safe_segment(parsed.hostname or "unknown-host") + segments = [self._safe_segment(segment) for segment in parsed.path.split("/") if segment] + extension = self._extension_for(content_type) + + if not segments or parsed.path.endswith("/"): + segments.append(f"index{extension or '.html'}") + elif content_type.startswith("text/html") and "." not in segments[-1]: + segments.append("index.html") + elif extension and "." not in segments[-1]: + segments[-1] = f"{segments[-1]}{extension}" + + if parsed.query: + digest = hashlib.sha256(parsed.query.encode("utf-8")).hexdigest()[:12] + filename = segments[-1] + stem, dot, suffix = filename.rpartition(".") + segments[-1] = ( + f"{stem or filename}__q-{digest}{dot}{suffix}" if dot else f"{filename}__q-{digest}" + ) + + candidate = "/".join([host, *segments]) + if len(candidate) > 220: + digest = hashlib.sha256(key.encode("utf-8")).hexdigest()[:16] + candidate = f"{host}/_path-{digest}/{segments[-1][:80]}" + + if candidate.casefold() in self._used_path_keys: + digest = hashlib.sha256(key.encode("utf-8")).hexdigest()[:12] + parent, filename = posixpath.split(candidate) + stem, dot, suffix = filename.rpartition(".") + filename = ( + f"{stem or filename}__u-{digest}{dot}{suffix}" if dot else f"{filename}__u-{digest}" + ) + candidate = posixpath.join(parent, filename) + + self._paths[key] = candidate + self._used_path_keys.add(candidate.casefold()) + return candidate + + @staticmethod + def _safe_segment(value: str) -> str: + safe = re.sub(r"[^A-Za-z0-9._-]+", "-", value.strip()) + safe = safe.strip(".-") or "resource" + if safe.split(".", 1)[0].upper() in WINDOWS_RESERVED_NAMES: + safe = f"_{safe}" + return safe[:80] + + @staticmethod + def _extension_for(content_type: str) -> str: + media_type = content_type.split(";", 1)[0].strip().lower() + known = { + "text/html": ".html", + "text/css": ".css", + "text/javascript": ".js", + "application/javascript": ".js", + "application/json": ".json", + "application/manifest+json": ".webmanifest", + "image/svg+xml": ".svg", + "font/woff": ".woff", + "font/woff2": ".woff2", + } + if media_type in known: + return known[media_type] + return mimetypes.guess_extension(media_type) or "" + + +class SiteCrawler: + """A bounded static web crawler with archive-ready link rewriting.""" + + def __init__( + self, + *, + settings: Settings, + options: CaptureOptions, + cancelled: Callable[[], bool], + on_progress: Callable[[str, str, dict[str, int], list[str]], None], + ) -> None: + self.settings = settings + self.options = options + self.cancelled = cancelled + self.on_progress = on_progress + self.rules = NetworkRules( + allow_private_networks=settings.allow_private_networks, + allow_nonstandard_ports=settings.allow_nonstandard_ports, + ) + self.source_url = normalize_url(options.source_url, self.rules) + self.source_host = hostname(self.source_url) + self.parallel_connections = max( + 1, min(options.parallel_connections, self.settings.fetch_concurrency) + ) + self._proxy_url = settings.proxy_url if options.use_proxy else None + self.mapper = PathMapper(self.rules) + self.pending: deque[CrawlRequest] = deque() + self.enqueued: set[str] = set() + self.url_to_path: dict[str, str] = {} + self.records: list[ResourceRecord] = [] + self.errors: list[dict[str, object]] = [] + self.warnings: list[str] = [] + self._warning_count = 0 + self._warnings_omitted = 0 + self._error_count = 0 + self._errors_omitted = 0 + self.stats = { + "pages_found": 0, + "pages_fetched": 0, + "assets_found": 0, + "assets_fetched": 0, + "bytes_downloaded": 0, + "files_written": 0, + } + self._started_at = 0.0 + self._byte_count = 0 + self._byte_lock = threading.Lock() + self._scope_lock = threading.Lock() + self._robots_lock = threading.Lock() + self._external_hosts: set[str] = set() + self._robots: dict[str, RobotFileParser | bool] = {} + self._root_record: ResourceRecord | None = None + self._root_error: str | None = None + self._limit_reached = False + self._site_dir: Path | None = None + self._address_book = PinnedAddressBook() + limits = httpx.Limits( + max_connections=max(4, self.parallel_connections * 2), + max_keepalive_connections=max(2, self.parallel_connections), + ) + self._client = httpx.Client( + follow_redirects=False, + trust_env=False, + timeout=httpx.Timeout(connect=10.0, read=20.0, write=10.0, pool=10.0), + limits=limits, + transport=PinnedHTTPTransport(self._address_book, limits, proxy=self._proxy_url), + headers={"User-Agent": settings.user_agent, "Accept": "*/*"}, + ) + + def run(self, work_dir: Path) -> CrawlResult: + self._started_at = time.monotonic() + self._site_dir = work_dir / "site" + self._site_dir.mkdir(parents=True, exist_ok=True) + + try: + if self._proxy_url: + self._pin_proxy() + self._enqueue(CrawlRequest(self.source_url, "document", 0)) + with ThreadPoolExecutor(max_workers=self.parallel_connections) as executor: + while self.pending and not self._limit_reached: + self._ensure_not_cancelled() + if self._duration_exceeded(): + raise CaptureError("The configured capture time limit was reached.") + + batch = [ + self.pending.popleft() + for _ in range(min(len(self.pending), self.parallel_connections)) + ] + futures = [executor.submit(self._fetch, request) for request in batch] + for future in as_completed(futures): + self._ensure_not_cancelled() + self._handle_fetch_result(future.result()) + + self._emit_progress("Capturing", "Discovering pages and assets") + + if not self._root_record: + raise CaptureError(self._root_error or "The starting page could not be captured.") + + self._ensure_within_duration() + self._emit_progress("Rewriting", "Converting captured links for offline use") + self._rewrite_captured_content() + self._ensure_within_duration() + report = self._build_report() + self._write_capture_metadata(report) + self._emit_progress("Packaging", "Preparing archive contents") + return CrawlResult( + site_dir=self._site_dir, + entry_point=self._root_record.local_path, + report=report, + stats=self.stats.copy(), + warnings=self._visible_warnings(), + ) + finally: + self._client.close() + + def _fetch(self, request: CrawlRequest) -> FetchResult: + current_url = request.url + redirects: list[str] = [] + + for _ in range(self.settings.max_redirects + 1): + if self.cancelled(): + return FetchResult(request, "cancelled", message="Capture cancelled") + if self._duration_exceeded(): + raise CaptureError("The configured capture time limit was reached.") + + try: + self._pin_target(current_url) + except UrlPolicyError as error: + return FetchResult(request, "skipped", final_url=current_url, message=str(error)) + + if not self._robots_allow(current_url): + return FetchResult( + request, + "skipped", + final_url=current_url, + message="robots.txt excludes this resource", + ) + + try: + headers = self._referer_header(request.referrer, current_url) + with self._client.stream( + "GET", + current_url, + headers=headers, + timeout=self._request_timeout(), + ) as response: + status_code = response.status_code + if status_code in {301, 302, 303, 307, 308}: + location = response.headers.get("location") + if not location: + return FetchResult( + request, + "error", + final_url=current_url, + status_code=status_code, + message="Redirect response had no Location header", + ) + try: + next_url = normalize_url(urljoin(current_url, location), self.rules) + except UrlPolicyError as error: + return FetchResult( + request, "skipped", final_url=current_url, message=str(error) + ) + if not self._scope_allows(next_url, request.kind): + return FetchResult( + request, + "skipped", + final_url=current_url, + message="Redirect leaves the configured capture scope", + ) + redirects.append(current_url) + current_url = next_url + continue + + if status_code >= 400: + return FetchResult( + request, + "error", + final_url=current_url, + status_code=status_code, + message=f"Remote server returned HTTP {status_code}", + ) + + body_or_error = self._read_limited_response(response) + if isinstance(body_or_error, str): + return FetchResult( + request, + "skipped", + final_url=current_url, + status_code=status_code, + message=body_or_error, + ) + + return FetchResult( + request, + "ok", + final_url=current_url, + redirects=redirects, + status_code=status_code, + content_type=response.headers.get("content-type", "") + .split(";", 1)[0] + .lower(), + encoding=response.encoding, + body=body_or_error, + ) + except httpx.HTTPError as error: + return FetchResult( + request, "error", final_url=current_url, message=f"Network error: {error}" + ) + + return FetchResult(request, "skipped", final_url=current_url, message="Too many redirects") + + def _read_limited_response( + self, response: httpx.Response, maximum_bytes: int | None = None + ) -> bytes | str: + chunks: list[bytes] = [] + response_bytes = 0 + response_limit = maximum_bytes or self.settings.max_response_bytes + for chunk in response.iter_bytes(): + if self.cancelled(): + raise CaptureCancelled("Capture cancelled") + if self._duration_exceeded(): + raise CaptureError("The configured capture time limit was reached.") + response_bytes += len(chunk) + if response_bytes > response_limit: + return "Resource exceeds the per-file size limit" + with self._byte_lock: + if self._byte_count + len(chunk) > self.options.max_bytes: + self._limit_reached = True + return "The configured total download budget was reached" + self._byte_count += len(chunk) + self.stats["bytes_downloaded"] = self._byte_count + chunks.append(chunk) + return b"".join(chunks) + + def _handle_fetch_result(self, result: FetchResult) -> None: + request_key = canonical_url(result.request.url, self.rules) + is_root = request_key == canonical_url(self.source_url, self.rules) + + if result.outcome == "cancelled": + raise CaptureCancelled("Capture cancelled") + if result.outcome != "ok" or not result.final_url or result.body is None: + if is_root: + self._root_error = result.message or "The starting page could not be fetched" + self._record_problem(result) + return + + local_path = self.mapper.assign(result.final_url, result.content_type) + try: + destination = self._safe_destination(local_path) + destination.parent.mkdir(parents=True, exist_ok=True) + destination.write_bytes(result.body) + except OSError as error: + failed_result = FetchResult( + result.request, + "error", + final_url=result.final_url, + status_code=result.status_code, + message=f"Could not write this resource locally: {error}", + ) + if is_root: + self._root_error = failed_result.message + self._record_problem(failed_result) + return + + record = ResourceRecord( + requested_url=result.request.url, + final_url=result.final_url, + local_path=local_path, + kind=result.request.kind, + content_type=result.content_type, + status_code=result.status_code or 200, + byte_count=len(result.body), + redirects=result.redirects, + encoding=result.encoding, + ) + self.records.append(record) + self.url_to_path[request_key] = local_path + self.url_to_path[canonical_url(result.final_url, self.rules)] = local_path + for redirect in result.redirects: + self.url_to_path[canonical_url(redirect, self.rules)] = local_path + + self.stats["files_written"] += 1 + if result.request.kind == "document": + self.stats["pages_fetched"] += 1 + else: + self.stats["assets_fetched"] += 1 + + if is_root: + self._root_record = record + + text = self._decode_text(result.body, result.encoding) + if self._is_html(record, text): + self._discover_html(text, record.final_url, result.request.depth) + elif self._is_css(record): + self._discover_css(text, record.final_url, result.request.depth) + elif self._is_javascript(record): + self._discover_javascript(text, record.final_url, result.request.depth) + + def _record_problem(self, result: FetchResult) -> None: + if result.outcome == "skipped" and result.message: + self._add_warning(f"Skipped {result.request.url}: {result.message}") + elif result.message: + self._error_count += 1 + if len(self.errors) < MAX_RECORDED_ERRORS: + self.errors.append( + { + "url": result.request.url, + "kind": result.request.kind, + "status_code": result.status_code, + "message": result.message, + } + ) + else: + self._errors_omitted += 1 + + def _discover_html(self, text: str, page_url: str, depth: int) -> None: + soup = BeautifulSoup(text, "html.parser") + base_url = page_url + base = soup.find("base", href=True) + if isinstance(base, Tag): + resolved = self._resolve_reference(page_url, self._tag_value(base, "href") or "") + if resolved: + base_url = resolved + + for tag in soup.find_all(True): + if not isinstance(tag, Tag): + continue + name = (tag.name or "").lower() + document_attribute = "href" if name in {"a", "area"} else "src" + document_url = self._tag_value(tag, document_attribute) + if name in {"a", "area", "iframe", "frame"} and document_url: + attribute = "href" if name in {"a", "area"} else "src" + self._enqueue_reference( + self._tag_value(tag, attribute), base_url, "document", depth + 1, page_url + ) + + link_url = self._tag_value(tag, "href") + if name == "link" and link_url: + rel_value = tag.get("rel") + if isinstance(rel_value, list): + rel = {str(item).lower() for item in rel_value} + elif isinstance(rel_value, str): + rel = {rel_value.lower()} + else: + rel = set() + if not rel.intersection({"canonical", "alternate"}): + self._enqueue_reference(link_url, base_url, "asset", depth, page_url) + + for attribute in ("src", "poster", "data", "background"): + attribute_url = self._tag_value(tag, attribute) + if attribute_url: + kind: ResourceKind = ( + "document" if name == "object" and attribute == "data" else "asset" + ) + self._enqueue_reference(attribute_url, base_url, kind, depth, page_url) + + if name in {"use", "image"}: + for attribute in ("href", "xlink:href"): + attribute_url = self._tag_value(tag, attribute) + if attribute_url: + self._enqueue_reference(attribute_url, base_url, "asset", depth, page_url) + + srcset = self._tag_value(tag, "srcset") + if srcset: + for candidate in self._srcset_urls(srcset): + self._enqueue_reference(candidate, base_url, "asset", depth, page_url) + + style = self._tag_value(tag, "style") + if style: + self._discover_css(style, base_url, depth) + + if name == "style" and tag.string: + self._discover_css(str(tag.string), base_url, depth) + if name == "script" and tag.string: + self._discover_javascript(str(tag.string), base_url, depth) + if name == "meta": + property_name = str(tag.get("property") or tag.get("name") or "").lower() + content = self._tag_value(tag, "content") + if content and property_name in {"og:image", "twitter:image", "og:video"}: + self._enqueue_reference(content, base_url, "asset", depth, page_url) + + for meta in soup.find_all("meta"): + if not isinstance(meta, Tag): + continue + content = self._tag_value(meta, "content") + if str(meta.get("http-equiv", "")).lower() == "refresh" and content: + match = META_REFRESH_RE.match(content) + if match: + self._enqueue_reference( + match.group("url"), base_url, "document", depth + 1, page_url + ) + + def _discover_css(self, text: str, base_url: str, depth: int) -> None: + for match in CSS_URL_RE.finditer(text): + self._enqueue_reference(match.group("url"), base_url, "asset", depth, base_url) + for match in CSS_IMPORT_RE.finditer(text): + self._enqueue_reference(match.group("url"), base_url, "asset", depth, base_url) + + def _discover_javascript(self, text: str, base_url: str, depth: int) -> None: + for pattern in ( + JS_DYNAMIC_IMPORT_RE, + JS_STATIC_IMPORT_RE, + JS_NEW_URL_RE, + JS_WORKER_RE, + JS_SOURCE_MAP_RE, + ): + for match in pattern.finditer(text): + self._enqueue_reference(match.group("url"), base_url, "asset", depth, base_url) + + def _enqueue_reference( + self, raw_url: object, base_url: str, kind: ResourceKind, depth: int, referrer: str | None + ) -> None: + if not isinstance(raw_url, str): + return + resolved = self._resolve_reference(base_url, raw_url) + if resolved: + self._enqueue(CrawlRequest(resolved, kind, depth, referrer)) + + def _enqueue(self, request: CrawlRequest) -> None: + try: + key = canonical_url(request.url, self.rules) + except UrlPolicyError: + return + if key in self.enqueued or self._limit_reached: + return + if request.kind == "document" and request.depth > self.options.max_depth: + self._add_warning(f"Skipped {request.url}: page depth limit reached") + return + if len(self.enqueued) >= self.settings.max_files: + self._add_warning("The configured file-count safety limit was reached.") + self._limit_reached = True + return + if request.kind == "document" and self.stats["pages_found"] >= self.options.max_pages: + self._add_warning("The configured page limit was reached.") + return + if not self._scope_allows(key, request.kind): + return + if request.kind == "asset" and not self._asset_is_allowed(key): + return + + self.enqueued.add(key) + self.pending.append(CrawlRequest(key, request.kind, request.depth, request.referrer)) + if request.kind == "document": + self.stats["pages_found"] += 1 + else: + self.stats["assets_found"] += 1 + + def _scope_allows(self, url: str, kind: ResourceKind) -> bool: + candidate_host = hostname(url) + if same_site_host(candidate_host, self.source_host): + return True + if kind == "document": + return False + if not self.options.include_external_assets: + return False + with self._scope_lock: + if candidate_host not in self._external_hosts: + if len(self._external_hosts) >= self.settings.max_external_hosts: + self._add_warning( + "External asset host limit reached; remaining third-party assets " + "were skipped." + ) + return False + self._external_hosts.add(candidate_host) + return True + + def _asset_is_allowed(self, url: str) -> bool: + extension = posixpath.splitext(urlsplit(url).path.lower())[1].lstrip(".") + if extension and extension in self.options.skip_file_types: + self._add_warning(f"Skipped .{extension} files because of the configured file rule.") + return False + asset_type = self._asset_type(url) + if asset_type not in self.options.download_asset_types: + self._add_warning( + f"Skipped {asset_type} assets because they are not selected for this capture." + ) + return False + return True + + @staticmethod + def _asset_type(url: str) -> str: + extension = posixpath.splitext(urlsplit(url).path.lower())[1].lstrip(".") + if extension == "css": + return "styles" + if extension in {"js", "mjs", "cjs", "map"}: + return "scripts" + if extension in {"avif", "bmp", "gif", "ico", "jpeg", "jpg", "png", "svg", "webp"}: + return "images" + if extension in {"eot", "otf", "ttf", "woff", "woff2"}: + return "fonts" + if extension in { + "aac", + "avi", + "m4a", + "mkv", + "mov", + "mp3", + "mp4", + "mpeg", + "oga", + "ogg", + "ogv", + "wav", + "webm", + }: + return "media" + return "other" + + def _robots_allow(self, url: str) -> bool: + if not self.settings.respect_robots: + return True + parsed = urlsplit(url) + origin = f"{parsed.scheme}://{parsed.netloc}" + with self._robots_lock: + cached = self._robots.get(origin) + known_origin = origin in self._robots + if not known_origin: + loaded = self._load_robots(origin) + with self._robots_lock: + cached = self._robots.setdefault(origin, loaded) + if isinstance(cached, RobotFileParser): + return cached.can_fetch(self.settings.user_agent, url) + return True + + def _load_robots(self, origin: str) -> RobotFileParser | bool: + robots_url = f"{origin}/robots.txt" + parser = RobotFileParser() + parser.set_url(robots_url) + try: + self._pin_target(robots_url) + with self._client.stream( + "GET", + robots_url, + follow_redirects=False, + timeout=self._request_timeout(), + ) as response: + if response.status_code == 200: + body = self._read_limited_response(response, maximum_bytes=512 * 1024) + if isinstance(body, bytes): + parser.parse(self._decode_text(body, response.encoding).splitlines()) + return parser + return True + if response.status_code in {401, 403}: + parser.parse(["User-agent: *", "Disallow: /"]) + return parser + except (httpx.HTTPError, UrlPolicyError): + return True + return True + + def _rewrite_captured_content(self) -> None: + for record in self.records: + self._ensure_not_cancelled() + self._ensure_within_duration() + destination = self._safe_destination(record.local_path) + if self._is_html(record): + text = self._decode_text(destination.read_bytes(), record.encoding) + destination.write_text(self._rewrite_html(text, record), encoding="utf-8") + elif self._is_css(record): + text = self._decode_text(destination.read_bytes(), record.encoding) + rewritten = self._rewrite_css(text, record.local_path, record.final_url) + rewritten = CSS_CHARSET_RE.sub('@charset "UTF-8";', rewritten, count=1) + destination.write_text( + rewritten, + encoding="utf-8", + ) + elif self._is_javascript(record): + text = self._decode_text(destination.read_bytes(), record.encoding) + destination.write_text( + self._rewrite_javascript(text, record.local_path, record.final_url), + encoding="utf-8", + ) + + def _rewrite_html(self, text: str, record: ResourceRecord) -> str: + soup = BeautifulSoup(text, "html.parser") + base_url = record.final_url + base = soup.find("base", href=True) + if isinstance(base, Tag): + resolved = self._resolve_reference(base_url, self._tag_value(base, "href") or "") + if resolved: + base_url = resolved + base.decompose() + + for meta in soup.find_all("meta", charset=True): + if isinstance(meta, Tag): + meta["charset"] = "utf-8" + for meta in soup.find_all("meta"): + if not isinstance(meta, Tag): + continue + if str(meta.get("http-equiv", "")).lower() != "content-type": + continue + content = self._tag_value(meta, "content") + if content and "charset=" in content.lower(): + meta["content"] = re.sub( + r"charset\s*=\s*[^;\s]+", "charset=UTF-8", content, flags=re.I + ) + + for tag in soup.find_all(True): + if not isinstance(tag, Tag): + continue + for attribute in ("href", "src", "poster", "data", "background", "xlink:href"): + attribute_url = self._tag_value(tag, attribute) + if attribute_url: + tag[attribute] = self._local_reference( + record.local_path, base_url, attribute_url + ) + srcset = self._tag_value(tag, "srcset") + if srcset: + tag["srcset"] = self._rewrite_srcset(record.local_path, base_url, srcset) + style = self._tag_value(tag, "style") + if style: + tag["style"] = self._rewrite_css(style, record.local_path, base_url) + name = (tag.name or "").lower() + if name == "style" and tag.string: + style_text = str(tag.string) + tag.clear() + tag.append(self._rewrite_css(style_text, record.local_path, base_url)) + if name == "script" and tag.string: + script_text = str(tag.string) + tag.clear() + tag.append(self._rewrite_javascript(script_text, record.local_path, base_url)) + + for meta in soup.find_all("meta"): + if not isinstance(meta, Tag): + continue + content = self._tag_value(meta, "content") + if str(meta.get("http-equiv", "")).lower() == "refresh" and content: + match = META_REFRESH_RE.match(content) + if match: + meta["content"] = match.group("delay") + self._local_reference( + record.local_path, base_url, match.group("url") + ) + return str(soup) + + def _rewrite_css(self, text: str, local_path: str, base_url: str) -> str: + def replace(match: re.Match[str]) -> str: + quote = match.group("quote") + raw = match.group("url") + local = self._local_reference(local_path, base_url, raw) + return f"url({quote}{local}{quote})" + + def replace_import(match: re.Match[str]) -> str: + quote = match.group("quote") + local = self._local_reference(local_path, base_url, match.group("url")) + return f"@import {quote}{local}{quote}" + + return CSS_IMPORT_RE.sub(replace_import, CSS_URL_RE.sub(replace, text)) + + def _rewrite_javascript(self, text: str, local_path: str, base_url: str) -> str: + def replace_quoted(match: re.Match[str]) -> str: + local = self._local_reference(local_path, base_url, match.group("url")) + prefix = match.group(0)[: match.start("url") - match.start(0)] + suffix = match.group(0)[match.end("url") - match.start(0) :] + return f"{prefix}{local}{suffix}" + + def replace_source_map(match: re.Match[str]) -> str: + return match.group(0).replace( + match.group("url"), self._local_reference(local_path, base_url, match.group("url")) + ) + + text = JS_DYNAMIC_IMPORT_RE.sub(replace_quoted, text) + text = JS_STATIC_IMPORT_RE.sub(replace_quoted, text) + text = JS_NEW_URL_RE.sub(replace_quoted, text) + text = JS_WORKER_RE.sub(replace_quoted, text) + return JS_SOURCE_MAP_RE.sub(replace_source_map, text) + + def _local_reference(self, from_path: str, base_url: str, raw_url: str) -> str: + stripped = raw_url.strip() + if not stripped or stripped.startswith("#") or self._is_non_fetchable_reference(stripped): + return raw_url + absolute = urljoin(base_url, stripped) + try: + target_key = canonical_url(absolute, self.rules) + except UrlPolicyError: + return raw_url + target_path = self.url_to_path.get(target_key) + if not target_path: + return raw_url + relative = posixpath.relpath(target_path, start=posixpath.dirname(from_path) or ".") + fragment = urlsplit(absolute).fragment + return f"{relative}#{fragment}" if fragment else relative + + def _rewrite_srcset(self, local_path: str, base_url: str, value: str) -> str: + candidates: list[str] = [] + for candidate in value.split(","): + pieces = candidate.strip().split(maxsplit=1) + if not pieces: + continue + rewritten = self._local_reference(local_path, base_url, pieces[0]) + candidates.append(" ".join([rewritten, *pieces[1:]])) + return ", ".join(candidates) + + def _build_report(self) -> dict[str, object]: + elapsed_seconds = round(time.monotonic() - self._started_at, 2) + quality = "partial" if self._limit_reached or self.errors or self.warnings else "complete" + return { + "siteharbor_version": "0.1.0", + "source_url": self.source_url, + "entry_point": self._root_record.local_path if self._root_record else None, + "quality": quality, + "elapsed_seconds": elapsed_seconds, + "options": asdict(self.options), + "stats": self.stats, + "warnings": self._visible_warnings(), + "warning_count": self._warning_count, + "errors": self._visible_errors(), + "error_count": self._error_count, + "resources": [record.as_manifest() for record in self.records], + } + + def _write_capture_metadata(self, report: dict[str, object]) -> None: + if not self._site_dir or not self._root_record: + raise CaptureError("Capture output was not initialized.") + (self._site_dir / "siteharbor-manifest.json").write_text( + json.dumps(report, indent=2, ensure_ascii=True), encoding="utf-8" + ) + readme = "\n".join( + [ + "SiteHarbor offline capture", + "", + f"Entry point: {self._root_record.local_path}", + f"Source URL: {self.source_url}", + f"Quality: {report['quality']}", + "", + "This is a bounded static capture. Dynamic APIs, authenticated content,", + "service workers, and runtime-generated behavior may not work offline.", + "See siteharbor-manifest.json for captured, skipped, and failed resources.", + "", + ] + ) + (self._site_dir / "README.txt").write_text(readme, encoding="utf-8") + + def _emit_progress(self, phase: str, message: str) -> None: + self.on_progress(phase, message, self.stats.copy(), self._visible_warnings()) + + def _ensure_not_cancelled(self) -> None: + if self.cancelled(): + raise CaptureCancelled("Capture cancelled") + + def _ensure_within_duration(self) -> None: + if self._duration_exceeded(): + raise CaptureError("The configured capture time limit was reached.") + + def _duration_exceeded(self) -> bool: + return self._remaining_seconds() <= 0 + + def _remaining_seconds(self) -> float: + elapsed = time.monotonic() - self._started_at + return self.options.max_duration_seconds - elapsed + + def _request_timeout(self) -> httpx.Timeout: + remaining = max(0.05, self._remaining_seconds()) + return httpx.Timeout( + connect=min(10.0, remaining), + read=min(20.0, remaining), + write=min(10.0, remaining), + pool=min(10.0, remaining), + ) + + def _pin_target(self, url: str) -> None: + addresses = resolve_and_validate(url, self.rules) + parsed = urlsplit(url) + port = parsed.port or (443 if parsed.scheme == "https" else 80) + self._address_book.pin(hostname(url), port, addresses) + + def _pin_proxy(self) -> None: + if not self._proxy_url: + return + try: + proxy_url = normalize_url(self._proxy_url, self.rules) + addresses = resolve_and_validate(proxy_url, self.rules) + except UrlPolicyError as error: + raise CaptureError("The configured proxy endpoint is not allowed.") from error + parsed = urlsplit(proxy_url) + port = parsed.port or (443 if parsed.scheme == "https" else 80) + self._address_book.pin(hostname(proxy_url), port, addresses) + + @staticmethod + def _referer_header(referrer: str | None, target_url: str) -> dict[str, str] | None: + if not referrer: + return None + try: + if not same_site_host(hostname(referrer), hostname(target_url)): + return None + except UrlPolicyError: + return None + parsed = urlsplit(referrer) + return {"Referer": f"{parsed.scheme}://{parsed.netloc}/"} + + def _add_warning(self, message: str) -> None: + self._warning_count += 1 + if message not in self.warnings: + if len(self.warnings) < MAX_RECORDED_WARNINGS: + self.warnings.append(message) + else: + self._warnings_omitted += 1 + + def _visible_warnings(self) -> list[str]: + visible = self.warnings.copy() + if self._warnings_omitted: + visible.append( + f"{self._warnings_omitted} additional warnings were omitted from this report." + ) + return visible + + def _visible_errors(self) -> list[dict[str, object]]: + visible = self.errors.copy() + if self._errors_omitted: + visible.append( + { + "url": "", + "kind": "summary", + "status_code": None, + "message": f"{self._errors_omitted} additional resource failures were omitted.", + } + ) + return visible + + def _safe_destination(self, local_path: str) -> Path: + if not self._site_dir: + raise CaptureError("Capture output was not initialized.") + destination = self._site_dir.joinpath(*local_path.split("/")) + resolved_root = self._site_dir.resolve() + resolved_destination = destination.resolve() + if not resolved_destination.is_relative_to(resolved_root): + raise CaptureError("A captured path escaped the archive root.") + return destination + + def _resolve_reference(self, base_url: str, raw_url: str) -> str | None: + value = raw_url.strip() + if not value or value.startswith("#") or self._is_non_fetchable_reference(value): + return None + try: + return normalize_url(urljoin(base_url, value), self.rules) + except UrlPolicyError: + return None + + @staticmethod + def _is_non_fetchable_reference(value: str) -> bool: + return value.lower().startswith( + ("data:", "javascript:", "mailto:", "tel:", "blob:", "about:") + ) + + @staticmethod + def _decode_text(body: bytes, encoding: str | None) -> str: + encodings = [encoding, "utf-8", "utf-8-sig", "latin-1"] + for candidate in encodings: + if not candidate: + continue + try: + return body.decode(candidate) + except (LookupError, UnicodeDecodeError): + continue + return body.decode("utf-8", errors="replace") + + @staticmethod + def _srcset_urls(value: str) -> list[str]: + return [ + candidate.strip().split(maxsplit=1)[0] + for candidate in value.split(",") + if candidate.strip() + ] + + @staticmethod + def _tag_value(tag: Tag, attribute: str) -> str | None: + value = tag.get(attribute) + return value if isinstance(value, str) else None + + @staticmethod + def _is_html(record: ResourceRecord, text: str | None = None) -> bool: + if record.content_type in {"text/html", "application/xhtml+xml"}: + return True + return bool(text and " bool: + return record.content_type == "text/css" or record.local_path.lower().endswith(".css") + + @staticmethod + def _is_javascript(record: ResourceRecord) -> bool: + return "javascript" in record.content_type or record.local_path.lower().endswith( + (".js", ".mjs", ".cjs") + ) diff --git a/app/db.py b/app/db.py new file mode 100644 index 0000000..0ae8562 --- /dev/null +++ b/app/db.py @@ -0,0 +1,684 @@ +from __future__ import annotations + +import json +import sqlite3 +import uuid +from collections.abc import Iterator +from contextlib import contextmanager +from datetime import UTC, datetime, timedelta +from pathlib import Path +from typing import Any + + +def utcnow() -> datetime: + return datetime.now(UTC) + + +def timestamp(value: datetime | None = None) -> str: + return (value or utcnow()).isoformat() + + +class Database: + """Small SQLite persistence layer with lease-based job claiming.""" + + def __init__(self, path: Path) -> None: + self.path = path + + @contextmanager + def connection(self) -> Iterator[sqlite3.Connection]: + connection = sqlite3.connect(self.path, timeout=15, isolation_level=None) + connection.row_factory = sqlite3.Row + connection.execute("PRAGMA foreign_keys = ON") + connection.execute("PRAGMA busy_timeout = 15000") + try: + yield connection + finally: + connection.close() + + def initialize(self) -> None: + with self.connection() as connection: + connection.execute("PRAGMA journal_mode = WAL") + connection.execute("PRAGMA synchronous = NORMAL") + connection.executescript( + """ + CREATE TABLE IF NOT EXISTS jobs ( + id TEXT PRIMARY KEY, + source_url TEXT NOT NULL, + normalized_url TEXT NOT NULL, + owner_token TEXT NOT NULL DEFAULT '', + config_json TEXT NOT NULL, + state TEXT NOT NULL, + phase TEXT NOT NULL, + message TEXT NOT NULL, + error_code TEXT, + error_message TEXT, + warnings_json TEXT NOT NULL DEFAULT '[]', + pages_found INTEGER NOT NULL DEFAULT 0, + pages_fetched INTEGER NOT NULL DEFAULT 0, + assets_found INTEGER NOT NULL DEFAULT 0, + assets_fetched INTEGER NOT NULL DEFAULT 0, + bytes_downloaded INTEGER NOT NULL DEFAULT 0, + files_written INTEGER NOT NULL DEFAULT 0, + artifact_path TEXT, + report_path TEXT, + archive_size INTEGER, + entry_point TEXT, + cancel_requested INTEGER NOT NULL DEFAULT 0, + worker_id TEXT, + lease_expires_at TEXT, + attempt_count INTEGER NOT NULL DEFAULT 0, + created_at TEXT NOT NULL, + updated_at TEXT NOT NULL, + started_at TEXT, + completed_at TEXT, + expires_at TEXT + ); + + CREATE INDEX IF NOT EXISTS jobs_state_created_idx + ON jobs(state, created_at); + + CREATE INDEX IF NOT EXISTS jobs_lease_idx + ON jobs(lease_expires_at); + + CREATE TABLE IF NOT EXISTS job_events ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + job_id TEXT NOT NULL REFERENCES jobs(id) ON DELETE CASCADE, + kind TEXT NOT NULL, + payload_json TEXT NOT NULL, + created_at TEXT NOT NULL + ); + + CREATE INDEX IF NOT EXISTS job_events_job_id_idx + ON job_events(job_id, id); + """ + ) + columns = { + str(row["name"]) + for row in connection.execute("PRAGMA table_info(jobs)").fetchall() + } + if "owner_token" not in columns: + connection.execute( + "ALTER TABLE jobs ADD COLUMN owner_token TEXT NOT NULL DEFAULT ''" + ) + connection.execute( + "CREATE INDEX IF NOT EXISTS jobs_owner_created_idx " + "ON jobs(owner_token, created_at DESC)" + ) + + def create_job( + self, + source_url: str, + normalized_url: str, + config: dict[str, Any], + owner_token: str = "", + ) -> dict[str, Any]: + job_id = f"cap_{uuid.uuid4().hex}" + now = timestamp() + payload = json.dumps(config, separators=(",", ":")) + with self.connection() as connection: + connection.execute( + """ + INSERT INTO jobs ( + id, source_url, normalized_url, owner_token, config_json, state, phase, message, + created_at, updated_at, expires_at + ) VALUES ( + ?, ?, ?, ?, ?, 'queued', 'Queued', 'Waiting for a local worker', ?, ?, NULL + ) + """, + ( + job_id, + source_url, + normalized_url, + owner_token, + payload, + now, + now, + ), + ) + self._append_event( + connection, + job_id, + "state", + {"state": "queued", "phase": "Queued", "message": "Waiting for a local worker"}, + ) + return self.get_job(job_id) or {} + + def get_job(self, job_id: str) -> dict[str, Any] | None: + with self.connection() as connection: + row = connection.execute("SELECT * FROM jobs WHERE id = ?", (job_id,)).fetchone() + return self._row_to_job(row) if row else None + + def get_owned_job(self, job_id: str, owner_token: str) -> dict[str, Any] | None: + with self.connection() as connection: + row = connection.execute( + "SELECT * FROM jobs WHERE id = ? AND owner_token = ?", (job_id, owner_token) + ).fetchone() + return self._row_to_job(row) if row else None + + def list_jobs(self, limit: int = 30) -> list[dict[str, Any]]: + with self.connection() as connection: + rows = connection.execute( + "SELECT * FROM jobs ORDER BY created_at DESC LIMIT ?", (limit,) + ).fetchall() + return [self._row_to_job(row) for row in rows] + + def list_owned_jobs(self, owner_token: str, limit: int = 30) -> list[dict[str, Any]]: + with self.connection() as connection: + rows = connection.execute( + """ + SELECT * FROM jobs + WHERE owner_token = ? + ORDER BY created_at DESC + LIMIT ? + """, + (owner_token, limit), + ).fetchall() + return [self._row_to_job(row) for row in rows] + + def public_stats(self) -> dict[str, int]: + with self.connection() as connection: + row = connection.execute( + """ + SELECT + COUNT(CASE WHEN state = 'ready' THEN 1 END) AS websites_cloned, + COALESCE(SUM(bytes_downloaded), 0) AS bytes_scraped, + COUNT(CASE WHEN state IN ('queued', 'running') THEN 1 END) AS active_crawls, + COALESCE(SUM(files_written), 0) AS files_archived + FROM jobs + """ + ).fetchone() + return { + "websites_cloned": int(row["websites_cloned"] or 0), + "bytes_scraped": int(row["bytes_scraped"] or 0), + "active_crawls": int(row["active_crawls"] or 0), + "files_archived": int(row["files_archived"] or 0), + } + + def get_events(self, job_id: str, after_id: int = 0) -> list[dict[str, Any]]: + with self.connection() as connection: + rows = connection.execute( + """ + SELECT id, kind, payload_json, created_at + FROM job_events + WHERE job_id = ? AND id > ? + ORDER BY id ASC + LIMIT 100 + """, + (job_id, after_id), + ).fetchall() + return [ + { + "id": row["id"], + "kind": row["kind"], + "payload": json.loads(row["payload_json"]), + "created_at": row["created_at"], + } + for row in rows + ] + + def claim_next(self, worker_id: str, lease_seconds: int) -> dict[str, Any] | None: + now = utcnow() + now_text = timestamp(now) + lease = timestamp(now + timedelta(seconds=lease_seconds)) + with self.connection() as connection: + try: + connection.execute("BEGIN IMMEDIATE") + cancelled_rows = connection.execute( + """ + SELECT id FROM jobs + WHERE cancel_requested = 1 + AND ( + state = 'queued' + OR ( + state = 'running' + AND lease_expires_at IS NOT NULL + AND lease_expires_at < ? + ) + ) + """, + (now_text,), + ).fetchall() + connection.execute( + """ + UPDATE jobs + SET state = 'cancelled', phase = 'Cancelled', + message = 'Cancelled before starting', + completed_at = ?, updated_at = ? + WHERE cancel_requested = 1 + AND ( + state = 'queued' + OR ( + state = 'running' + AND lease_expires_at IS NOT NULL + AND lease_expires_at < ? + ) + ) + """, + (now_text, now_text, now_text), + ) + for cancelled_row in cancelled_rows: + self._append_event( + connection, + cancelled_row["id"], + "state", + { + "state": "cancelled", + "phase": "Cancelled", + "message": "Cancelled before starting", + }, + ) + row = connection.execute( + """ + SELECT * FROM jobs + WHERE cancel_requested = 0 + AND ( + state = 'queued' + OR ( + state = 'running' + AND lease_expires_at IS NOT NULL + AND lease_expires_at < ? + ) + ) + ORDER BY created_at ASC + LIMIT 1 + """, + (now_text,), + ).fetchone() + if not row: + connection.execute("COMMIT") + return None + + job_id = row["id"] + message = ( + "Recovering a leased capture" + if row["state"] == "running" + else "Preparing capture" + ) + connection.execute( + """ + UPDATE jobs + SET state = 'running', phase = 'Preparing', message = ?, worker_id = ?, + lease_expires_at = ?, attempt_count = attempt_count + 1, + started_at = COALESCE(started_at, ?), updated_at = ? + WHERE id = ? + """, + (message, worker_id, lease, now_text, now_text, job_id), + ) + self._append_event( + connection, + job_id, + "state", + {"state": "running", "phase": "Preparing", "message": message}, + ) + claimed = connection.execute( + "SELECT * FROM jobs WHERE id = ?", (job_id,) + ).fetchone() + connection.execute("COMMIT") + return self._row_to_job(claimed) + except Exception: + connection.execute("ROLLBACK") + raise + + def update_progress( + self, + job_id: str, + worker_id: str, + lease_seconds: int, + *, + phase: str, + message: str, + stats: dict[str, int], + warnings: list[str] | None = None, + ) -> bool: + now = utcnow() + values = { + "phase": phase, + "message": message, + "pages_found": stats.get("pages_found", 0), + "pages_fetched": stats.get("pages_fetched", 0), + "assets_found": stats.get("assets_found", 0), + "assets_fetched": stats.get("assets_fetched", 0), + "bytes_downloaded": stats.get("bytes_downloaded", 0), + "files_written": stats.get("files_written", 0), + "warnings_json": json.dumps(warnings or []), + "worker_id": worker_id, + "lease_expires_at": timestamp(now + timedelta(seconds=lease_seconds)), + "updated_at": timestamp(now), + "current_time": timestamp(now), + } + with self.connection() as connection: + cursor = connection.execute( + """ + UPDATE jobs SET + phase = :phase, message = :message, + pages_found = :pages_found, pages_fetched = :pages_fetched, + assets_found = :assets_found, assets_fetched = :assets_fetched, + bytes_downloaded = :bytes_downloaded, files_written = :files_written, + warnings_json = :warnings_json, worker_id = :worker_id, + lease_expires_at = :lease_expires_at, updated_at = :updated_at + WHERE id = :job_id + AND state = 'running' + AND worker_id = :worker_id + AND lease_expires_at >= :current_time + """, + {**values, "job_id": job_id}, + ) + if cursor.rowcount: + self._append_event( + connection, + job_id, + "progress", + { + "phase": phase, + "message": message, + "stats": stats, + "warnings": warnings or [], + }, + ) + return True + return False + + def renew_lease(self, job_id: str, worker_id: str, lease_seconds: int) -> bool: + now = utcnow() + now_text = timestamp(now) + with self.connection() as connection: + cursor = connection.execute( + """ + UPDATE jobs + SET lease_expires_at = ?, updated_at = ? + WHERE id = ? + AND state = 'running' + AND worker_id = ? + AND lease_expires_at >= ? + """, + ( + timestamp(now + timedelta(seconds=lease_seconds)), + now_text, + job_id, + worker_id, + now_text, + ), + ) + return bool(cursor.rowcount) + + def is_cancel_requested(self, job_id: str) -> bool: + with self.connection() as connection: + row = connection.execute( + "SELECT cancel_requested FROM jobs WHERE id = ?", (job_id,) + ).fetchone() + return bool(row and row["cancel_requested"]) + + def owns_lease(self, job_id: str, worker_id: str) -> bool: + now = timestamp() + with self.connection() as connection: + row = connection.execute( + """ + SELECT state, worker_id, lease_expires_at + FROM jobs + WHERE id = ? + """, + (job_id,), + ).fetchone() + return bool( + row + and row["state"] == "running" + and row["worker_id"] == worker_id + and row["lease_expires_at"] + and row["lease_expires_at"] >= now + ) + + def request_cancel(self, job_id: str) -> dict[str, Any] | None: + now = timestamp() + with self.connection() as connection: + try: + connection.execute("BEGIN IMMEDIATE") + row = connection.execute( + "SELECT state FROM jobs WHERE id = ?", (job_id,) + ).fetchone() + if not row: + connection.execute("COMMIT") + return None + if row["state"] in {"ready", "failed", "cancelled", "expired"}: + connection.execute("COMMIT") + return self.get_job(job_id) + + if row["state"] == "queued": + cursor = connection.execute( + """ + UPDATE jobs + SET cancel_requested = 1, state = 'cancelled', phase = 'Cancelled', + message = 'Cancelled before starting', completed_at = ?, updated_at = ? + WHERE id = ? AND state = 'queued' + """, + (now, now, job_id), + ) + payload = { + "state": "cancelled", + "phase": "Cancelled", + "message": "Cancelled before starting", + } + else: + cursor = connection.execute( + """ + UPDATE jobs + SET cancel_requested = 1, message = 'Cancellation requested', updated_at = ? + WHERE id = ? AND state = 'running' AND cancel_requested = 0 + """, + (now, job_id), + ) + payload = { + "state": "running", + "phase": "Cancelling", + "message": "Cancellation requested", + } + if cursor.rowcount: + self._append_event(connection, job_id, "state", payload) + connection.execute("COMMIT") + except Exception: + connection.execute("ROLLBACK") + raise + return self.get_job(job_id) + + def mark_ready( + self, + job_id: str, + worker_id: str, + *, + artifact_path: str, + report_path: str, + archive_size: int, + entry_point: str, + stats: dict[str, int], + warnings: list[str], + retention_days: int, + ) -> bool: + now_datetime = utcnow() + now = timestamp(now_datetime) + with self.connection() as connection: + cursor = connection.execute( + """ + UPDATE jobs + SET state = 'ready', phase = 'Ready', message = 'Archive is ready to download', + artifact_path = ?, report_path = ?, archive_size = ?, entry_point = ?, + pages_found = ?, pages_fetched = ?, assets_found = ?, assets_fetched = ?, + bytes_downloaded = ?, files_written = ?, warnings_json = ?, + worker_id = NULL, lease_expires_at = NULL, expires_at = ?, + completed_at = ?, updated_at = ? + WHERE id = ? + AND state = 'running' + AND worker_id = ? + AND cancel_requested = 0 + AND lease_expires_at >= ? + """, + ( + artifact_path, + report_path, + archive_size, + entry_point, + stats.get("pages_found", 0), + stats.get("pages_fetched", 0), + stats.get("assets_found", 0), + stats.get("assets_fetched", 0), + stats.get("bytes_downloaded", 0), + stats.get("files_written", 0), + json.dumps(warnings), + timestamp(now_datetime + timedelta(days=retention_days)), + now, + now, + job_id, + worker_id, + now, + ), + ) + if cursor.rowcount: + self._append_event( + connection, + job_id, + "state", + {"state": "ready", "phase": "Ready", "message": "Archive is ready to download"}, + ) + return True + return False + + def mark_cancelled( + self, job_id: str, worker_id: str, message: str = "Capture cancelled" + ) -> bool: + now = timestamp() + with self.connection() as connection: + cursor = connection.execute( + """ + UPDATE jobs + SET state = 'cancelled', phase = 'Cancelled', message = ?, worker_id = NULL, + lease_expires_at = NULL, completed_at = ?, updated_at = ? + WHERE id = ? + AND state = 'running' + AND worker_id = ? + AND cancel_requested = 1 + AND lease_expires_at >= ? + """, + (message, now, now, job_id, worker_id, now), + ) + if cursor.rowcount: + self._append_event( + connection, + job_id, + "state", + {"state": "cancelled", "phase": "Cancelled", "message": message}, + ) + return True + return False + + def mark_failed(self, job_id: str, worker_id: str, code: str, message: str) -> bool: + now = timestamp() + with self.connection() as connection: + cursor = connection.execute( + """ + UPDATE jobs + SET state = 'failed', phase = 'Failed', message = ?, error_code = ?, + error_message = ?, worker_id = NULL, lease_expires_at = NULL, + completed_at = ?, updated_at = ? + WHERE id = ? + AND state = 'running' + AND worker_id = ? + AND cancel_requested = 0 + AND lease_expires_at >= ? + """, + (message, code, message, now, now, job_id, worker_id, now), + ) + if cursor.rowcount: + self._append_event( + connection, + job_id, + "state", + {"state": "failed", "phase": "Failed", "message": message, "code": code}, + ) + return True + return False + + def mark_expired(self, job_id: str) -> None: + now = timestamp() + with self.connection() as connection: + connection.execute( + """ + UPDATE jobs + SET state = 'expired', phase = 'Expired', + message = 'Artifact retention period ended', + artifact_path = NULL, report_path = NULL, updated_at = ? + WHERE id = ? + """, + (now, job_id), + ) + self._append_event( + connection, + job_id, + "state", + { + "state": "expired", + "phase": "Expired", + "message": "Artifact retention period ended", + }, + ) + + def mark_deleted(self, job_id: str) -> None: + now = timestamp() + with self.connection() as connection: + cursor = connection.execute( + """ + UPDATE jobs + SET state = 'expired', phase = 'Deleted', message = 'Artifact deleted by user', + artifact_path = NULL, report_path = NULL, updated_at = ? + WHERE id = ? AND state = 'ready' + """, + (now, job_id), + ) + if cursor.rowcount: + self._append_event( + connection, + job_id, + "state", + { + "state": "expired", + "phase": "Deleted", + "message": "Artifact deleted by user", + }, + ) + + def expired_ready_jobs(self) -> list[dict[str, Any]]: + now = timestamp() + with self.connection() as connection: + rows = connection.execute( + "SELECT * FROM jobs WHERE state = 'ready' AND expires_at <= ?", (now,) + ).fetchall() + return [self._row_to_job(row) for row in rows] + + def referenced_data_paths(self) -> set[str]: + with self.connection() as connection: + rows = connection.execute( + """ + SELECT artifact_path, report_path + FROM jobs + WHERE state = 'ready' + """ + ).fetchall() + return { + path + for row in rows + for path in (row["artifact_path"], row["report_path"]) + if isinstance(path, str) and path + } + + def _append_event( + self, connection: sqlite3.Connection, job_id: str, kind: str, payload: dict[str, Any] + ) -> None: + connection.execute( + "INSERT INTO job_events (job_id, kind, payload_json, created_at) VALUES (?, ?, ?, ?)", + (job_id, kind, json.dumps(payload, separators=(",", ":")), timestamp()), + ) + + @staticmethod + def _row_to_job(row: sqlite3.Row) -> dict[str, Any]: + data = dict(row) + data["config"] = json.loads(data.pop("config_json")) + data["warnings"] = json.loads(data.pop("warnings_json")) + data["cancel_requested"] = bool(data["cancel_requested"]) + return data diff --git a/app/main.py b/app/main.py new file mode 100644 index 0000000..a0f2dfc --- /dev/null +++ b/app/main.py @@ -0,0 +1,395 @@ +from __future__ import annotations + +import asyncio +import json +import re +from collections.abc import AsyncIterator +from contextlib import asynccontextmanager +from pathlib import Path +from typing import Any + +import uvicorn +from fastapi import FastAPI, HTTPException, Request, Response, status +from fastapi.responses import FileResponse, HTMLResponse, JSONResponse, StreamingResponse +from fastapi.staticfiles import StaticFiles +from fastapi.templating import Jinja2Templates + +from app.config import settings +from app.db import Database +from app.models import SUPPORTED_ASSET_TYPES, CaptureRequest +from app.scheduler import JobRunner +from app.url_policy import NetworkRules, UrlPolicyError, normalize_url + +settings.ensure_directories() +database = Database(settings.database_path) +runner = JobRunner(database, settings) +templates = Jinja2Templates(directory=str(settings.project_root / "app" / "templates")) +OWNER_TOKEN_HEADER = "X-SiteHarbor-Session" +OWNER_TOKEN_RE = re.compile(r"[A-Za-z0-9_-]{24,200}") + + +@asynccontextmanager +async def lifespan(_: FastAPI) -> AsyncIterator[None]: + database.initialize() + runner.start() + try: + yield + finally: + runner.stop() + + +app = FastAPI( + title="SiteHarbor", + version="0.1.0", + description="Standalone, bounded website capture for offline archives.", + lifespan=lifespan, +) +app.mount( + "/static", StaticFiles(directory=str(settings.project_root / "app" / "static")), name="static" +) + + +@app.middleware("http") +async def secure_headers(_: Request, call_next: Any) -> Response: + response = await call_next(_) + response.headers.setdefault("X-Content-Type-Options", "nosniff") + response.headers.setdefault("Referrer-Policy", "same-origin") + response.headers.setdefault("X-Frame-Options", "DENY") + response.headers.setdefault( + "Content-Security-Policy", + "default-src 'self'; base-uri 'none'; object-src 'none'; frame-ancestors 'none'; " + "form-action 'self'; connect-src 'self'; img-src 'self' data:; style-src 'self'; " + "script-src 'self'", + ) + return response + + +@app.get("/", response_class=HTMLResponse) +async def index(request: Request) -> Response: + return templates.TemplateResponse( + request=request, + name="index.html", + context={"app_name": "SiteHarbor"}, + ) + + +@app.get("/api/health") +async def health() -> dict[str, Any]: + return { + "status": "ok", + "worker_enabled": settings.worker_enabled, + "worker_concurrency": settings.worker_concurrency, + "storage": "local-sqlite", + } + + +@app.get("/api/capabilities") +async def capabilities() -> dict[str, Any]: + return { + "defaults": { + "max_pages": min(150, settings.max_pages_cap), + "max_depth": min(5, settings.max_depth_cap), + "max_bytes_mb": min(100, max(1, settings.max_bytes_cap // (1024 * 1024))), + "max_duration_seconds": min(300, settings.max_duration_seconds_cap), + "retention_days": min(settings.default_retention_days, settings.max_retention_days), + "include_external_assets": True, + "parallel_connections": min(6, settings.fetch_concurrency), + "download_asset_types": list(SUPPORTED_ASSET_TYPES), + "skip_file_types": [], + "use_proxy": False, + }, + "limits": { + "max_pages": settings.max_pages_cap, + "max_depth": settings.max_depth_cap, + "max_bytes_mb": settings.max_bytes_cap // (1024 * 1024), + "max_duration_seconds": settings.max_duration_seconds_cap, + "max_retention_days": settings.max_retention_days, + "max_parallel_connections": settings.fetch_concurrency, + }, + "proxy_available": bool(settings.proxy_url), + "local_only_warning": not settings.allow_private_networks, + } + + +@app.get("/api/stats") +async def public_stats() -> dict[str, int]: + return database.public_stats() + + +@app.post("/api/captures", status_code=status.HTTP_202_ACCEPTED) +async def create_capture(payload: CaptureRequest, request: Request) -> dict[str, Any]: + owner_token = _owner_token(request) + config = _validated_config(payload) + rules = NetworkRules( + allow_private_networks=settings.allow_private_networks, + allow_nonstandard_ports=settings.allow_nonstandard_ports, + ) + try: + normalized_url = normalize_url(payload.url, rules) + except UrlPolicyError as error: + raise HTTPException(status_code=422, detail=str(error)) from error + + job = database.create_job(payload.url.strip(), normalized_url, config, owner_token) + return _public_job(job) + + +@app.get("/api/captures") +async def list_captures(request: Request) -> dict[str, list[dict[str, Any]]]: + owner_token = _owner_token(request) + return {"items": [_public_job(job) for job in database.list_owned_jobs(owner_token)]} + + +@app.get("/api/captures/{job_id}") +async def get_capture(job_id: str, request: Request) -> dict[str, Any]: + return _require_owned_job(job_id, _owner_token(request)) + + +@app.post("/api/captures/{job_id}/cancel") +async def cancel_capture(job_id: str, request: Request) -> dict[str, Any]: + _require_owned_job(job_id, _owner_token(request)) + job = database.request_cancel(job_id) + if not job: + raise HTTPException(status_code=404, detail="Capture not found.") + return _public_job(job) + + +@app.delete("/api/captures/{job_id}") +async def delete_capture(job_id: str, request: Request) -> dict[str, Any]: + job = _owned_job(job_id, _owner_token(request)) + if job["state"] in {"queued", "running"}: + cancelled = database.request_cancel(job_id) + return _public_job(cancelled or job) + if job["state"] == "ready": + artifact_path = _safe_data_file(str(job.get("artifact_path") or ""), settings.artifacts_dir) + report_path = _safe_data_file(str(job.get("report_path") or ""), settings.reports_dir) + if artifact_path: + artifact_path.unlink(missing_ok=True) + if report_path: + report_path.unlink(missing_ok=True) + database.mark_deleted(job_id) + updated = database.get_owned_job(job_id, _owner_token(request)) + return _public_job(updated or job) + return _public_job(job) + + +@app.get("/api/captures/{job_id}/report") +async def capture_report(job_id: str, request: Request) -> JSONResponse: + job = _owned_job(job_id, _owner_token(request)) + if job["state"] != "ready" or not job.get("report_path"): + raise HTTPException(status_code=409, detail="A capture report is not available yet.") + report_path = _safe_data_file(str(job["report_path"]), settings.reports_dir) + if not report_path or not report_path.is_file(): + raise HTTPException(status_code=410, detail="The capture report has expired.") + return JSONResponse(json.loads(report_path.read_text(encoding="utf-8"))) + + +@app.get("/api/captures/{job_id}/download") +async def download_capture(job_id: str, request: Request) -> FileResponse: + job = _owned_job(job_id, _owner_token(request)) + if job["state"] != "ready" or not job.get("artifact_path"): + raise HTTPException(status_code=409, detail="The archive is not available yet.") + archive_path = _safe_data_file(str(job["artifact_path"]), settings.artifacts_dir) + if not archive_path or not archive_path.is_file(): + raise HTTPException(status_code=410, detail="The archive has expired.") + return FileResponse( + archive_path, + media_type="application/zip", + filename=f"siteharbor-{job_id}.zip", + headers={"Cache-Control": "no-store", "X-Content-Type-Options": "nosniff"}, + ) + + +@app.get("/api/captures/{job_id}/events") +async def capture_events(request: Request, job_id: str) -> StreamingResponse: + _owned_job(job_id, _owner_token(request, allow_query=True)) + after_header = request.headers.get("last-event-id", "0") + try: + after_id = max(0, int(after_header)) + except ValueError: + after_id = 0 + return StreamingResponse( + _event_stream(request, job_id, after_id), + media_type="text/event-stream", + headers={"Cache-Control": "no-cache", "X-Accel-Buffering": "no"}, + ) + + +async def _event_stream(request: Request, job_id: str, after_id: int) -> AsyncIterator[str]: + terminal_states = {"ready", "failed", "cancelled", "expired"} + last_id = after_id + while True: + if await request.is_disconnected(): + return + events = database.get_events(job_id, last_id) + for event in events: + last_id = int(event["id"]) + payload = json.dumps(event["payload"], separators=(",", ":")) + yield f"id: {last_id}\nevent: {event['kind']}\ndata: {payload}\n\n" + + job = database.get_job(job_id) + if not job or job["state"] in terminal_states: + terminal = json.dumps({"state": job["state"] if job else "missing"}) + yield f"event: terminal\ndata: {terminal}\n\n" + return + yield ": keepalive\n\n" + await asyncio.sleep(0.8) + + +def _validated_config(payload: CaptureRequest) -> dict[str, Any]: + max_bytes = payload.max_bytes_mb * 1024 * 1024 + limits = { + "max_pages": (payload.max_pages, settings.max_pages_cap, "page"), + "max_depth": (payload.max_depth, settings.max_depth_cap, "depth"), + "max_bytes": (max_bytes, settings.max_bytes_cap, "download size"), + "max_duration_seconds": ( + payload.max_duration_seconds, + settings.max_duration_seconds_cap, + "capture duration", + ), + "retention_days": (payload.retention_days, settings.max_retention_days, "retention"), + } + for _, (value, maximum, label) in limits.items(): + if value > maximum: + raise HTTPException( + status_code=422, detail=f"Requested {label} exceeds this instance's limit." + ) + if payload.parallel_connections > settings.fetch_concurrency: + raise HTTPException( + status_code=422, + detail="Requested parallel connections exceed this instance's limit.", + ) + if payload.use_proxy and not settings.proxy_url: + raise HTTPException( + status_code=422, + detail="This service does not have an approved proxy route configured.", + ) + skip_file_types = _normalized_file_types(payload.skip_file_types) + return { + "include_external_assets": payload.include_external_assets, + "parallel_connections": payload.parallel_connections, + "download_asset_types": list(dict.fromkeys(payload.download_asset_types)), + "skip_file_types": skip_file_types, + "use_proxy": payload.use_proxy, + "max_pages": payload.max_pages, + "max_depth": payload.max_depth, + "max_bytes": max_bytes, + "max_duration_seconds": payload.max_duration_seconds, + "retention_days": payload.retention_days, + } + + +def _normalized_file_types(values: list[str]) -> list[str]: + normalized: list[str] = [] + for value in values: + for item in value.split(","): + extension = item.strip().lower().lstrip(".") + if not extension: + continue + if not re.fullmatch(r"[a-z0-9]{1,16}", extension): + raise HTTPException( + status_code=422, + detail=( + "File rules must be comma-separated extensions such as pdf, zip, or mp4." + ), + ) + if extension not in normalized: + normalized.append(extension) + if len(normalized) > 24: + raise HTTPException(status_code=422, detail="At most 24 file types can be skipped.") + return normalized + + +def _owner_token(request: Request, *, allow_query: bool = False) -> str: + value = request.headers.get(OWNER_TOKEN_HEADER) + if not value and allow_query: + value = request.query_params.get("session") + token = (value or "").strip() + if not OWNER_TOKEN_RE.fullmatch(token): + raise HTTPException( + status_code=status.HTTP_401_UNAUTHORIZED, + detail="A browser-local session is required to access captures.", + ) + return token + + +def _owned_job(job_id: str, owner_token: str) -> dict[str, Any]: + job = database.get_owned_job(job_id, owner_token) + if not job: + raise HTTPException(status_code=404, detail="Capture not found.") + return job + + +def _require_owned_job(job_id: str, owner_token: str) -> dict[str, Any]: + return _public_job(_owned_job(job_id, owner_token)) + + +def _public_job(job: dict[str, Any]) -> dict[str, Any]: + config = job["config"] + return { + "id": job["id"], + "url": job["normalized_url"], + "state": job["state"], + "phase": job["phase"], + "message": job["message"], + "error_code": job["error_code"], + "error_message": job["error_message"], + "warnings": job["warnings"], + "created_at": job["created_at"], + "updated_at": job["updated_at"], + "started_at": job["started_at"], + "completed_at": job["completed_at"], + "expires_at": job["expires_at"], + "cancel_requested": job["cancel_requested"], + "entry_point": job["entry_point"], + "stats": { + "pages_found": job["pages_found"], + "pages_fetched": job["pages_fetched"], + "assets_found": job["assets_found"], + "assets_fetched": job["assets_fetched"], + "bytes_downloaded": job["bytes_downloaded"], + "files_written": job["files_written"], + }, + "options": { + "include_external_assets": config.get("include_external_assets", True), + "parallel_connections": config.get( + "parallel_connections", settings.fetch_concurrency + ), + "download_asset_types": config.get( + "download_asset_types", list(SUPPORTED_ASSET_TYPES) + ), + "skip_file_types": config.get("skip_file_types", []), + "use_proxy": config.get("use_proxy", False), + "max_pages": config["max_pages"], + "max_depth": config["max_depth"], + "max_bytes": config["max_bytes"], + "max_duration_seconds": config["max_duration_seconds"], + "retention_days": config["retention_days"], + }, + "archive": { + "available": job["state"] == "ready" and bool(job["artifact_path"]), + "size_bytes": job["archive_size"], + "download_url": f"/api/captures/{job['id']}/download" + if job["state"] == "ready" + else None, + "report_url": f"/api/captures/{job['id']}/report" if job["state"] == "ready" else None, + }, + } + + +def _safe_data_file(path_value: str, root: Path) -> Path | None: + path = Path(path_value) + try: + resolved = path.resolve() + if resolved.is_relative_to(root.resolve()): + return resolved + except OSError: + return None + return None + + +def run() -> None: + uvicorn.run("app.main:app", host=settings.host, port=settings.port, reload=False) + + +if __name__ == "__main__": + run() diff --git a/app/models.py b/app/models.py new file mode 100644 index 0000000..366fa42 --- /dev/null +++ b/app/models.py @@ -0,0 +1,31 @@ +from __future__ import annotations + +from typing import Literal + +from pydantic import BaseModel, Field + +AssetType = Literal["styles", "scripts", "images", "fonts", "media", "other"] +SUPPORTED_ASSET_TYPES: tuple[AssetType, ...] = ( + "styles", + "scripts", + "images", + "fonts", + "media", + "other", +) + + +class CaptureRequest(BaseModel): + url: str = Field(min_length=1, max_length=2_048) + include_external_assets: bool = True + parallel_connections: int = Field(default=6, ge=1) + download_asset_types: list[AssetType] = Field( + default_factory=lambda: list(SUPPORTED_ASSET_TYPES), max_length=len(SUPPORTED_ASSET_TYPES) + ) + skip_file_types: list[str] = Field(default_factory=list, max_length=24) + use_proxy: bool = False + max_pages: int = Field(default=150, ge=1) + max_depth: int = Field(default=5, ge=0) + max_bytes_mb: int = Field(default=100, ge=1) + max_duration_seconds: int = Field(default=300, ge=10) + retention_days: int = Field(default=7, ge=1) diff --git a/app/pinned_transport.py b/app/pinned_transport.py new file mode 100644 index 0000000..9dee592 --- /dev/null +++ b/app/pinned_transport.py @@ -0,0 +1,96 @@ +from __future__ import annotations + +import threading +from collections.abc import Iterable + +import httpcore +import httpx + + +class PinnedAddressBook: + """Thread-safe, short-lived host-to-validated-IP mapping for direct HTTP connections.""" + + def __init__(self) -> None: + self._addresses: dict[tuple[str, int], tuple[str, ...]] = {} + self._lock = threading.Lock() + + def pin(self, host: str, port: int, addresses: Iterable[str]) -> None: + values = tuple(addresses) + if not values: + raise ValueError("At least one validated address is required.") + with self._lock: + self._addresses[(host.rstrip(".").lower(), port)] = values + + def addresses_for(self, host: str, port: int) -> tuple[str, ...]: + with self._lock: + addresses = self._addresses.get((host.rstrip(".").lower(), port)) + if not addresses: + raise httpcore.ConnectError(f"No validated address is available for {host}:{port}") + return addresses + + +class PinnedNetworkBackend(httpcore.NetworkBackend): + """Dials only validated numeric addresses while httpcore retains the logical hostname.""" + + def __init__(self, address_book: PinnedAddressBook) -> None: + self._address_book = address_book + self._backend = httpcore.SyncBackend() + + def connect_tcp( + self, + host: str, + port: int, + timeout: float | None = None, + local_address: str | None = None, + socket_options: Iterable[httpcore.SOCKET_OPTION] | None = None, + ) -> httpcore.NetworkStream: + last_error: Exception | None = None + for address in self._address_book.addresses_for(host, port): + try: + return self._backend.connect_tcp( + address, + port, + timeout=timeout, + local_address=local_address, + socket_options=socket_options, + ) + except (httpcore.ConnectError, httpcore.ConnectTimeout) as error: + last_error = error + if last_error: + raise last_error + raise httpcore.ConnectError(f"Could not connect to a validated address for {host}:{port}") + + def connect_unix_socket( + self, + path: str, + timeout: float | None = None, + socket_options: Iterable[httpcore.SOCKET_OPTION] | None = None, + ) -> httpcore.NetworkStream: + return self._backend.connect_unix_socket( + path, timeout=timeout, socket_options=socket_options + ) + + def sleep(self, seconds: float) -> None: + self._backend.sleep(seconds) + + +class PinnedHTTPTransport(httpx.HTTPTransport): + """HTTPX transport that prevents a second, uncontrolled DNS lookup at connect time.""" + + def __init__( + self, + address_book: PinnedAddressBook, + limits: httpx.Limits, + proxy: str | None = None, + ) -> None: + super().__init__( + verify=True, + trust_env=False, + http1=True, + http2=False, + limits=limits, + proxy=proxy, + retries=0, + ) + # HTTPX constructs a direct httpcore ConnectionPool for this transport. + self._pool._network_backend = PinnedNetworkBackend(address_book) diff --git a/app/scheduler.py b/app/scheduler.py new file mode 100644 index 0000000..9641054 --- /dev/null +++ b/app/scheduler.py @@ -0,0 +1,121 @@ +from __future__ import annotations + +import logging +import os +import shutil +import socket +import threading +import time +import uuid +from pathlib import Path + +from app.capture import CaptureService +from app.config import Settings +from app.db import Database + +logger = logging.getLogger(__name__) + + +class JobRunner: + """SQLite-backed local workers. Every process safely claims jobs through a database lease.""" + + def __init__(self, database: Database, settings: Settings) -> None: + self.database = database + self.settings = settings + self.stop_event = threading.Event() + self.threads: list[threading.Thread] = [] + self.maintenance_thread: threading.Thread | None = None + self._last_expiry_sweep = 0.0 + + def start(self) -> None: + if not self.maintenance_thread: + self.maintenance_thread = threading.Thread( + target=self._run_maintenance, + name="siteharbor-maintenance", + daemon=True, + ) + self.maintenance_thread.start() + if self.settings.worker_enabled and not self.threads: + for index in range(self.settings.worker_concurrency): + worker_id = f"{socket.gethostname()}-{os.getpid()}-{index}-{uuid.uuid4().hex[:8]}" + thread = threading.Thread( + target=self._run_worker, + args=(worker_id,), + name=f"siteharbor-worker-{index}", + daemon=True, + ) + thread.start() + self.threads.append(thread) + + def stop(self) -> None: + self.stop_event.set() + for thread in self.threads: + thread.join(timeout=5) + if self.maintenance_thread: + self.maintenance_thread.join(timeout=2) + + def _run_worker(self, worker_id: str) -> None: + service = CaptureService(self.database, self.settings, worker_id) + while not self.stop_event.is_set(): + try: + job = self.database.claim_next(worker_id, self.settings.lease_seconds) + if not job: + self.stop_event.wait(self.settings.queue_poll_seconds) + continue + service.run(job) + except Exception: + logger.exception("SiteHarbor worker loop failed") + self.stop_event.wait(self.settings.queue_poll_seconds) + + def _run_maintenance(self) -> None: + while not self.stop_event.is_set(): + try: + self._expire_artifacts_if_due() + self._sweep_orphaned_attempts() + except Exception: + logger.exception("SiteHarbor maintenance loop failed") + self.stop_event.wait(10) + + def _expire_artifacts_if_due(self) -> None: + now = time.monotonic() + if now - self._last_expiry_sweep < 60: + return + self._last_expiry_sweep = now + for job in self.database.expired_ready_jobs(): + self._unlink_under_root(job.get("artifact_path"), self.settings.artifacts_dir) + self._unlink_under_root(job.get("report_path"), self.settings.reports_dir) + self.database.mark_expired(str(job["id"])) + + def _sweep_orphaned_attempts(self) -> None: + cutoff = time.time() - max(7_200, self.settings.max_duration_seconds_cap + 3_600) + referenced_paths = {Path(path).resolve() for path in self.database.referenced_data_paths()} + + for path in self.settings.work_dir.iterdir(): + try: + if path.stat().st_mtime >= cutoff: + continue + if path.is_dir(): + shutil.rmtree(path, ignore_errors=True) + except OSError: + logger.warning("Could not remove orphaned SiteHarbor work directory: %s", path) + + for root in (self.settings.artifacts_dir, self.settings.reports_dir): + for path in root.iterdir(): + try: + if path.resolve() in referenced_paths or path.stat().st_mtime >= cutoff: + continue + if path.is_file(): + path.unlink(missing_ok=True) + except OSError: + logger.warning("Could not remove orphaned SiteHarbor output: %s", path) + + @staticmethod + def _unlink_under_root(path_value: object, root: Path) -> None: + if not isinstance(path_value, str) or not path_value: + return + path = Path(path_value) + try: + if path.resolve().is_relative_to(root.resolve()): + path.unlink(missing_ok=True) + except OSError: + logger.warning("Could not remove expired SiteHarbor artifact: %s", path) diff --git a/app/static/app.css b/app/static/app.css new file mode 100644 index 0000000..ddbf301 --- /dev/null +++ b/app/static/app.css @@ -0,0 +1,1382 @@ +:root { + color-scheme: dark; + --paper: #f4f4f0; + --ink: #f4f6f1; + --muted: #9b9f98; + --quiet: #666b65; + --page: #1b1c1b; + --black: #050605; + --black-soft: #0a0c0a; + --forest: #063b2c; + --forest-deep: #02271e; + --mint: #8eeab8; + --mint-bright: #c3f8d8; + --line: rgba(238, 247, 239, 0.13); + --line-strong: rgba(238, 247, 239, 0.22); + --danger: #f0978e; + --amber: #e7c77b; + --blue: #9ac4f7; + --radius: 18px; + --shadow: 0 36px 95px rgba(0, 0, 0, 0.32); + font-family: Inter, ui-sans-serif, system-ui, -apple-system, BlinkMacSystemFont, "Segoe UI", sans-serif; +} + +* { + box-sizing: border-box; +} + +html { + min-width: 320px; + background: var(--page); +} + +body { + min-width: 320px; + min-height: 100vh; + margin: 0; + color: var(--ink); + background: + radial-gradient(circle at 86% -8%, rgba(122, 255, 181, 0.035), transparent 24rem), + radial-gradient(circle at 10% 100%, rgba(0, 0, 0, 0.24), transparent 32rem), + var(--page); +} + +body::before { + position: fixed; + z-index: -1; + inset: 0; + content: ""; + pointer-events: none; + opacity: 0.23; + background-image: + linear-gradient(rgba(255, 255, 255, 0.025) 1px, transparent 1px), + linear-gradient(90deg, rgba(255, 255, 255, 0.018) 1px, transparent 1px); + background-size: 48px 48px; + mask-image: linear-gradient(to bottom, black, transparent 71%); +} + +button, +input { + font: inherit; +} + +button { + cursor: pointer; +} + +button:focus-visible, +input:focus-visible, +summary:focus-visible, +a:focus-visible { + outline: 2px solid var(--mint-bright); + outline-offset: 3px; +} + +.site-shell { + width: min(1220px, calc(100% - 40px)); + margin: 0 auto; +} + +.topbar { + display: flex; + align-items: center; + justify-content: space-between; + min-height: 92px; + border-bottom: 1px solid rgba(255, 255, 255, 0.1); +} + +.brand { + display: inline-flex; + align-items: center; + gap: 11px; + color: var(--ink); + font-size: 1.06rem; + font-weight: 780; + letter-spacing: -0.045em; + text-decoration: none; +} + +.brand-mark { + position: relative; + display: block; + width: 24px; + height: 24px; + transform: rotate(45deg); + border: 1px solid rgba(152, 242, 187, 0.74); + border-radius: 7px; + background: rgba(79, 212, 138, 0.1); +} + +.brand-mark i { + position: absolute; + display: block; + width: 5px; + height: 5px; + border-radius: 50%; + background: var(--mint); +} + +.brand-mark i:nth-child(1) { + top: 4px; + left: 4px; +} + +.brand-mark i:nth-child(2) { + right: 4px; + bottom: 4px; + opacity: 0.75; +} + +.brand-mark i:nth-child(3) { + top: 4px; + right: 4px; + width: 3px; + height: 3px; + opacity: 0.55; +} + +.topbar-meta, +.service-status { + display: flex; + align-items: center; +} + +.topbar-meta { + gap: 20px; +} + +.public-label, +.eyebrow { + color: var(--mint); + font-size: 0.65rem; + font-weight: 780; + letter-spacing: 0.16em; +} + +.public-label { + color: var(--quiet); +} + +.service-status { + gap: 8px; + color: #b7bdb7; + font-size: 0.72rem; +} + +.service-status > i, +.pulse-dot { + display: inline-block; + flex: 0 0 auto; + width: 7px; + height: 7px; + border-radius: 50%; + background: var(--mint); + box-shadow: 0 0 0 4px rgba(142, 234, 184, 0.08), 0 0 13px rgba(142, 234, 184, 0.72); +} + +main { + padding: 53px 0 84px; +} + +.workbench { + display: grid; + grid-template-columns: minmax(0, 1.06fr) minmax(420px, 0.94fr); + overflow: hidden; + min-height: 630px; + border: 1px solid rgba(255, 255, 255, 0.14); + background: var(--black); + box-shadow: var(--shadow); +} + +.observatory, +.capture-console { + position: relative; + min-width: 0; +} + +.observatory { + display: flex; + flex-direction: column; + justify-content: space-between; + overflow: hidden; + padding: clamp(34px, 5.2vw, 68px); + background: + radial-gradient(ellipse 92% 88% at 22% 90%, rgba(25, 205, 137, 0.45), transparent 61%), + radial-gradient(ellipse 80% 78% at 86% 19%, rgba(7, 91, 67, 0.78), transparent 66%), + linear-gradient(135deg, #064632 0%, var(--forest-deep) 58%, #04271e 100%); +} + +.observatory::before { + position: absolute; + top: -38%; + left: -31%; + width: 106%; + aspect-ratio: 1; + border: 1px solid rgba(204, 255, 222, 0.12); + border-radius: 50%; + content: ""; + box-shadow: + 0 0 0 59px rgba(189, 255, 216, 0.025), + 0 0 0 120px rgba(189, 255, 216, 0.018); +} + +.observatory::after { + position: absolute; + right: -28%; + bottom: -38%; + width: 82%; + aspect-ratio: 1; + border: 1px solid rgba(202, 255, 222, 0.11); + border-radius: 50%; + content: ""; + box-shadow: 0 0 0 48px rgba(202, 255, 222, 0.025); +} + +.observatory-copy, +.metric-grid, +.observatory-foot { + position: relative; + z-index: 1; +} + +.eyebrow { + margin: 0 0 12px; +} + +h1, +h2, +h3, +p { + margin-top: 0; +} + +h1, +h2, +h3 { + color: var(--paper); + letter-spacing: -0.055em; +} + +h1 { + max-width: 525px; + margin-bottom: 21px; + font-size: clamp(3.05rem, 5.25vw, 5.35rem); + font-weight: 710; + line-height: 0.91; +} + +h1 em { + color: var(--mint-bright); + font-style: normal; +} + +h2 { + margin-bottom: 0; + font-size: clamp(1.8rem, 2.9vw, 2.55rem); + font-weight: 710; + line-height: 0.98; +} + +h3 { + margin-bottom: 7px; + font-size: 0.98rem; + font-weight: 700; +} + +.lede { + max-width: 480px; + margin-bottom: 0; + color: rgba(233, 250, 239, 0.75); + font-size: 0.96rem; + line-height: 1.67; +} + +.metric-grid { + display: grid; + grid-template-columns: repeat(2, minmax(0, 1fr)); + gap: 10px; + margin: 37px 0; +} + +.metric-card { + min-height: 102px; + padding: 17px; + border: 1px solid rgba(231, 255, 239, 0.2); + border-radius: 10px; + background: rgba(2, 27, 20, 0.22); + backdrop-filter: blur(7px); +} + +.metric-card.metric-primary { + border-color: rgba(249, 255, 249, 0.76); + color: #0f261b; + background: var(--paper); +} + +.metric-value, +.metric-label { + display: block; +} + +.metric-value { + overflow: hidden; + color: #eafff0; + font-size: clamp(1.34rem, 2vw, 1.72rem); + font-weight: 740; + letter-spacing: -0.06em; + text-overflow: ellipsis; + white-space: nowrap; +} + +.metric-primary .metric-value { + color: #11261a; +} + +.metric-label { + margin-top: 6px; + color: rgba(226, 251, 233, 0.7); + font-size: 0.65rem; + font-weight: 670; + letter-spacing: 0.06em; + text-transform: uppercase; +} + +.metric-primary .metric-label { + color: #547062; +} + +.observatory-foot { + display: flex; + align-items: flex-start; + gap: 12px; + max-width: 455px; + padding-top: 17px; + border-top: 1px solid rgba(234, 255, 241, 0.2); +} + +.observatory-foot .pulse-dot { + width: 6px; + height: 6px; + margin-top: 5px; +} + +.observatory-foot p { + margin: 0; + color: rgba(230, 249, 236, 0.66); + font-size: 0.73rem; + line-height: 1.55; +} + +.capture-console { + padding: clamp(31px, 4.2vw, 53px); + background: + linear-gradient(135deg, rgba(255, 255, 255, 0.025), transparent 28%), + var(--black); +} + +.console-head { + display: flex; + align-items: flex-start; + justify-content: space-between; + gap: 18px; + margin-bottom: 31px; +} + +.console-head .eyebrow { + color: #a6ad9f; +} + +.console-step { + padding-top: 4px; + color: #747a72; + font-family: ui-monospace, SFMono-Regular, Menlo, Consolas, monospace; + font-size: 0.67rem; + letter-spacing: 0.08em; +} + +.field-label, +.limit-grid label > span, +.text-rule > span { + display: block; + margin-bottom: 9px; + color: #c5cbc3; + font-size: 0.73rem; + font-weight: 690; +} + +.url-input-wrap { + display: flex; + align-items: center; + border: 1px solid #41463f; + border-radius: 9px; + background: #101210; + transition: border-color 160ms ease, box-shadow 160ms ease, background 160ms ease; +} + +.url-input-wrap:focus-within { + border-color: var(--mint); + background: #121611; + box-shadow: 0 0 0 4px rgba(142, 234, 184, 0.1); +} + +.url-input-wrap > span { + padding-left: 15px; + color: var(--mint); + font-family: ui-monospace, SFMono-Regular, Menlo, Consolas, monospace; + font-size: 1rem; +} + +.url-input-wrap input { + width: 100%; + min-width: 0; + padding: 15px; + border: 0; + outline: 0; + color: var(--ink); + background: transparent; + font-size: 0.96rem; +} + +.url-input-wrap input::placeholder, +.text-rule input::placeholder { + color: #656a64; +} + +.quick-settings { + display: grid; + gap: 10px; + margin-top: 18px; +} + +.switch-row, +.connection-control { + border: 1px solid var(--line); + border-radius: 10px; + background: rgba(255, 255, 255, 0.022); +} + +.switch-row { + display: flex; + align-items: center; + justify-content: space-between; + gap: 17px; + min-height: 75px; + padding: 14px 15px; +} + +.switch-row strong, +.switch-row small, +.connection-control strong, +.connection-control small { + display: block; +} + +.switch-row strong, +.connection-control strong { + margin-bottom: 4px; + color: #e3e7e1; + font-size: 0.82rem; + font-weight: 680; +} + +.switch-row small, +.connection-control small { + max-width: 320px; + color: #7e847d; + font-size: 0.7rem; + line-height: 1.4; +} + +.switch { + flex: 0 0 auto; +} + +.switch input { + position: absolute; + width: 1px; + height: 1px; + opacity: 0; +} + +.switch > span { + position: relative; + display: block; + width: 42px; + height: 23px; + border: 1px solid #525850; + border-radius: 999px; + background: #222622; + transition: border-color 160ms ease, background 160ms ease; +} + +.switch > span::after { + position: absolute; + top: 3px; + left: 3px; + width: 15px; + height: 15px; + border-radius: 50%; + background: #acb3ab; + content: ""; + transition: transform 160ms ease, background 160ms ease; +} + +.switch input:checked + span { + border-color: rgba(142, 234, 184, 0.82); + background: rgba(55, 146, 90, 0.57); +} + +.switch input:checked + span::after { + transform: translateX(17px); + background: var(--mint-bright); +} + +.switch input:disabled + span { + cursor: not-allowed; + opacity: 0.38; +} + +.connection-control { + position: relative; + display: grid; + grid-template-columns: minmax(0, 1fr) auto; + gap: 11px 16px; + padding: 14px 15px 13px; +} + +.connection-value { + align-self: center; + color: #a8afa6; + font-family: ui-monospace, SFMono-Regular, Menlo, Consolas, monospace; + font-size: 0.68rem; +} + +.connection-value output { + color: var(--mint); + font-weight: 700; +} + +.connection-control input[type="range"] { + width: 100%; + height: 4px; + grid-column: 1 / -1; + margin: 2px 0 0; + accent-color: var(--mint); + cursor: pointer; +} + +.advanced-settings { + display: grid; + gap: 8px; + margin-top: 10px; +} + +.setting-panel { + border: 1px solid var(--line); + border-radius: 10px; + background: rgba(255, 255, 255, 0.012); +} + +.setting-panel summary { + display: flex; + align-items: center; + justify-content: space-between; + gap: 15px; + min-height: 52px; + padding: 0 15px; + color: #d9ddd7; + cursor: pointer; + font-size: 0.79rem; + font-weight: 660; + list-style: none; +} + +.setting-panel summary::-webkit-details-marker { + display: none; +} + +.setting-panel summary::after { + order: 3; + margin-left: 2px; + color: #8f978d; + content: "+"; + font-size: 1rem; + font-weight: 400; +} + +.setting-panel[open] summary { + border-bottom: 1px solid var(--line); +} + +.setting-panel[open] summary::after { + content: "-"; +} + +.summary-value { + overflow: hidden; + color: #727970; + font-size: 0.66rem; + font-weight: 500; + text-align: right; + text-overflow: ellipsis; + white-space: nowrap; +} + +.limit-grid { + display: grid; + grid-template-columns: repeat(5, minmax(0, 1fr)); + gap: 8px; + padding: 15px; +} + +.limit-grid label > span { + margin-bottom: 6px; + color: #8f968d; + font-size: 0.63rem; +} + +.limit-grid input, +.text-rule input { + width: 100%; + min-width: 0; + border: 1px solid #3b403a; + border-radius: 7px; + outline: 0; + color: #e9ede7; + background: #111310; + transition: border-color 160ms ease, box-shadow 160ms ease; +} + +.limit-grid input { + padding: 9px 7px; + font-size: 0.77rem; +} + +.limit-grid input:focus, +.text-rule input:focus { + border-color: var(--mint); + box-shadow: 0 0 0 3px rgba(142, 234, 184, 0.09); +} + +.file-rules { + padding: 15px; +} + +.file-rules > p { + margin-bottom: 12px; + color: #858b83; + font-size: 0.7rem; + line-height: 1.45; +} + +.asset-choices { + display: grid; + grid-template-columns: repeat(3, minmax(0, 1fr)); + gap: 7px; +} + +.asset-choices label { + display: flex; + align-items: center; + gap: 7px; + min-height: 33px; + padding: 0 8px; + border: 1px solid #373d36; + border-radius: 7px; + color: #bfc5bd; + background: #0f110f; + font-size: 0.69rem; +} + +.asset-choices input { + width: 13px; + height: 13px; + margin: 0; + accent-color: var(--mint); +} + +.text-rule { + display: block; + margin-top: 13px; +} + +.text-rule > span { + margin-bottom: 6px; + color: #929890; + font-size: 0.66rem; +} + +.text-rule input { + padding: 10px 11px; + font-size: 0.76rem; +} + +.proxy-row { + min-height: 74px; + margin: 15px; +} + +.proxy-row.is-unavailable { + opacity: 0.6; +} + +.form-message { + min-height: 1.25em; + margin: 13px 0 0; + color: var(--danger); + font-size: 0.75rem; + line-height: 1.4; +} + +.form-message.success { + color: var(--mint); +} + +.console-actions { + display: flex; + align-items: center; + justify-content: space-between; + gap: 20px; + margin-top: 13px; +} + +.capture-button { + display: inline-flex; + align-items: center; + justify-content: space-between; + gap: 20px; + flex: 0 0 auto; + min-height: 45px; + padding: 0 15px 0 17px; + border: 1px solid var(--paper); + border-radius: 7px; + color: #112118; + background: var(--paper); + font-size: 0.79rem; + font-weight: 780; + transition: transform 160ms ease, background 160ms ease, box-shadow 160ms ease; +} + +.capture-button:hover:not(:disabled) { + transform: translateY(-1px); + background: var(--mint-bright); + box-shadow: 0 13px 26px rgba(142, 234, 184, 0.14); +} + +.capture-button:disabled { + cursor: wait; + opacity: 0.58; +} + +.capture-button span:last-child { + color: #315540; + font-family: ui-monospace, SFMono-Regular, Menlo, Consolas, monospace; + font-size: 1rem; +} + +.console-actions p { + max-width: 260px; + margin: 0; + color: #777d75; + font-size: 0.66rem; + line-height: 1.45; + text-align: right; +} + +.activity-section { + padding-top: 77px; +} + +.section-heading { + display: flex; + align-items: flex-end; + justify-content: space-between; + gap: 22px; + margin-bottom: 21px; +} + +.section-heading .eyebrow { + margin-bottom: 10px; + color: #8d968b; +} + +.section-heading h2 { + margin-bottom: 9px; + font-size: clamp(1.75rem, 2.4vw, 2.2rem); +} + +.section-heading p:not(.eyebrow) { + max-width: 580px; + margin: 0; + color: #969b95; + font-size: 0.79rem; + line-height: 1.52; +} + +.quiet-button, +.job-action, +.icon-button { + border: 1px solid var(--line-strong); + border-radius: 7px; + color: #ced4cc; + background: rgba(255, 255, 255, 0.035); + font-size: 0.72rem; + font-weight: 670; + transition: border-color 160ms ease, color 160ms ease, background 160ms ease; +} + +.quiet-button { + min-height: 34px; + padding: 0 12px; + white-space: nowrap; +} + +.quiet-button:hover, +.job-action:hover, +.icon-button:hover { + border-color: rgba(142, 234, 184, 0.66); + color: var(--mint-bright); + background: rgba(69, 136, 94, 0.17); +} + +.empty-state { + display: flex; + align-items: center; + gap: 16px; + min-height: 124px; + padding: 23px; + border: 1px dashed rgba(231, 238, 230, 0.22); + border-radius: var(--radius); + color: var(--muted); +} + +.empty-state[hidden] { + display: none; +} + +.empty-state p { + max-width: 580px; + margin: 0; + color: #969d95; + font-size: 0.79rem; + line-height: 1.52; +} + +.empty-mark { + display: grid; + flex: 0 0 auto; + width: 38px; + height: 38px; + place-items: center; + border: 1px solid rgba(142, 234, 184, 0.52); + border-radius: 50%; + color: var(--mint); + font-size: 1.3rem; +} + +.job-grid { + display: grid; + grid-template-columns: repeat(2, minmax(0, 1fr)); + gap: 13px; +} + +.job-card { + position: relative; + overflow: hidden; + min-height: 259px; + padding: 19px; + border: 1px solid var(--line); + border-radius: 12px; + background: + linear-gradient(135deg, rgba(255, 255, 255, 0.035), transparent 42%), + rgba(8, 10, 8, 0.58); +} + +.job-card::before { + position: absolute; + top: 0; + left: 0; + width: 100%; + height: 2px; + content: ""; + background: #5c635c; +} + +.job-card[data-state="running"]::before, +.job-card[data-state="queued"]::before { + background: var(--blue); +} + +.job-card[data-state="ready"]::before { + background: var(--mint); +} + +.job-card[data-state="failed"]::before, +.job-card[data-state="cancelled"]::before { + background: var(--danger); +} + +.job-topline, +.job-actions, +.job-stats, +.job-meta, +.job-tags { + display: flex; +} + +.job-topline { + align-items: center; + justify-content: space-between; + gap: 14px; +} + +.job-host { + overflow: hidden; + color: #f0f2ef; + font-size: 0.94rem; + font-weight: 700; + letter-spacing: -0.025em; + text-overflow: ellipsis; + white-space: nowrap; +} + +.state-pill { + flex: 0 0 auto; + padding: 5px 7px; + border: 1px solid rgba(202, 211, 200, 0.25); + border-radius: 999px; + color: #aab1a8; + font-size: 0.61rem; + font-weight: 720; + letter-spacing: 0.07em; + text-transform: uppercase; +} + +.state-pill.ready { + border-color: rgba(142, 234, 184, 0.45); + color: var(--mint); +} + +.state-pill.failed, +.state-pill.cancelled, +.state-pill.expired { + border-color: rgba(240, 151, 142, 0.45); + color: var(--danger); +} + +.state-pill.running, +.state-pill.queued { + border-color: rgba(154, 196, 247, 0.48); + color: var(--blue); +} + +.job-url { + overflow: hidden; + margin: 5px 0 16px; + color: #7e857c; + font-family: ui-monospace, SFMono-Regular, Menlo, Consolas, monospace; + font-size: 0.67rem; + text-overflow: ellipsis; + white-space: nowrap; +} + +.job-phase { + min-height: 37px; + margin: 0 0 12px; + color: #c9cec7; + font-size: 0.77rem; + line-height: 1.45; +} + +.job-progress { + overflow: hidden; + height: 3px; + margin: 0 0 15px; + border-radius: 999px; + background: rgba(235, 243, 235, 0.1); +} + +.job-progress > span { + display: block; + width: var(--progress, 0%); + height: 100%; + border-radius: inherit; + background: var(--blue); + transition: width 260ms ease; +} + +.job-card[data-state="ready"] .job-progress > span { + background: var(--mint); +} + +.job-card[data-state="failed"] .job-progress > span, +.job-card[data-state="cancelled"] .job-progress > span { + background: var(--danger); +} + +.job-stats { + margin-bottom: 13px; + border-top: 1px solid var(--line); + border-bottom: 1px solid var(--line); +} + +.job-stats div { + flex: 1; + min-width: 0; + padding: 10px 7px; +} + +.job-stats div + div { + border-left: 1px solid var(--line); +} + +.job-stats strong, +.job-stats span, +.job-meta span { + display: block; +} + +.job-stats strong { + overflow: hidden; + color: #e8ece6; + font-size: 0.82rem; + text-overflow: ellipsis; + white-space: nowrap; +} + +.job-stats span, +.job-meta span:first-child { + margin-top: 3px; + color: #777e75; + font-size: 0.59rem; + letter-spacing: 0.05em; + text-transform: uppercase; +} + +.job-meta { + justify-content: space-between; + gap: 10px; + min-height: 18px; + margin-bottom: 9px; + color: #929991; + font-size: 0.67rem; +} + +.job-meta span:last-child { + max-width: 55%; + overflow: hidden; + text-align: right; + text-overflow: ellipsis; + white-space: nowrap; +} + +.job-tags { + flex-wrap: wrap; + gap: 5px; + min-height: 21px; + margin-bottom: 10px; +} + +.job-tags span { + padding: 3px 6px; + border: 1px solid rgba(207, 218, 206, 0.15); + border-radius: 999px; + color: #879087; + font-size: 0.58rem; +} + +.job-warning { + overflow: hidden; + margin: 0 0 10px; + color: var(--amber); + font-size: 0.67rem; + line-height: 1.35; + text-overflow: ellipsis; + white-space: nowrap; +} + +.job-actions { + align-items: center; + flex-wrap: wrap; + gap: 7px; +} + +.job-action { + display: inline-flex; + align-items: center; + min-height: 30px; + padding: 0 9px; +} + +.job-action.primary { + border-color: rgba(142, 234, 184, 0.7); + color: #10251a; + background: var(--mint); +} + +.job-action.primary:hover { + color: #092015; + background: var(--mint-bright); +} + +.job-action.danger { + border-color: rgba(240, 151, 142, 0.36); + color: #efaaa3; +} + +.job-action:disabled { + cursor: wait; + opacity: 0.56; +} + +.footer-note { + display: flex; + justify-content: space-between; + gap: 22px; + padding: 24px 0 31px; + border-top: 1px solid rgba(255, 255, 255, 0.1); + color: #7c827b; + font-size: 0.68rem; + line-height: 1.48; +} + +.footer-note span:last-child { + max-width: 535px; + text-align: right; +} + +.report-dialog { + width: min(760px, calc(100% - 30px)); + max-height: min(760px, calc(100vh - 44px)); + padding: 0; + border: 1px solid var(--line-strong); + border-radius: 15px; + color: var(--ink); + background: #111310; + box-shadow: 0 30px 100px rgba(0, 0, 0, 0.67); +} + +.report-dialog::backdrop { + background: rgba(0, 0, 0, 0.72); + backdrop-filter: blur(5px); +} + +.dialog-head { + display: flex; + align-items: flex-start; + justify-content: space-between; + gap: 20px; + padding: 24px 25px 18px; + border-bottom: 1px solid var(--line); +} + +.dialog-head .eyebrow { + color: var(--mint); +} + +.dialog-head h2 { + margin: 0; + font-size: 1.7rem; +} + +.icon-button { + width: 31px; + height: 31px; + font-size: 1rem; +} + +.report-content { + overflow: auto; + max-height: calc(min(760px, 100vh - 44px) - 102px); + padding: 23px 25px 28px; +} + +.report-summary { + display: grid; + grid-template-columns: repeat(4, minmax(0, 1fr)); + gap: 9px; + margin-bottom: 21px; +} + +.report-summary div { + padding: 12px; + border: 1px solid var(--line); + border-radius: 8px; + background: rgba(255, 255, 255, 0.025); +} + +.report-summary strong, +.report-summary span { + display: block; +} + +.report-summary strong { + overflow: hidden; + color: var(--mint); + font-size: 1rem; + text-overflow: ellipsis; + white-space: nowrap; +} + +.report-summary span { + margin-top: 4px; + color: #7f867d; + font-size: 0.61rem; + letter-spacing: 0.04em; + text-transform: uppercase; +} + +.report-list { + margin: 0; + padding-left: 18px; + color: #c6ccc4; + font-size: 0.78rem; + line-height: 1.55; +} + +.report-list li + li { + margin-top: 6px; +} + +.report-subhead { + margin: 21px 0 9px; + color: var(--mint); + font-size: 0.7rem; + letter-spacing: 0.1em; + text-transform: uppercase; +} + +@media (prefers-reduced-motion: no-preference) { + .pulse-dot, + .service-status > i { + animation: pulse 2.2s ease-in-out infinite; + } +} + +@keyframes pulse { + 50% { + opacity: 0.45; + transform: scale(0.76); + } +} + +@media (max-width: 930px) { + .workbench { + grid-template-columns: 1fr; + } + + .observatory { + min-height: 490px; + } + + .capture-console { + min-height: 0; + } +} + +@media (max-width: 680px) { + .site-shell { + width: min(100% - 28px, 1220px); + } + + .topbar { + min-height: 72px; + } + + .public-label { + display: none; + } + + .topbar-meta { + gap: 0; + } + + .service-status { + font-size: 0.66rem; + } + + main { + padding: 31px 0 61px; + } + + .workbench { + min-height: 0; + } + + .observatory, + .capture-console { + padding: 28px 22px; + } + + .observatory { + min-height: 495px; + } + + h1 { + font-size: clamp(2.85rem, 14vw, 4.1rem); + } + + .metric-grid { + margin: 28px 0; + } + + .metric-card { + min-height: 94px; + padding: 14px; + } + + .console-head { + margin-bottom: 25px; + } + + .limit-grid { + grid-template-columns: repeat(2, minmax(0, 1fr)); + } + + .asset-choices { + grid-template-columns: repeat(2, minmax(0, 1fr)); + } + + .console-actions, + .section-heading, + .footer-note { + align-items: flex-start; + flex-direction: column; + } + + .console-actions p, + .footer-note span:last-child { + max-width: none; + text-align: left; + } + + .activity-section { + padding-top: 57px; + } + + .section-heading { + gap: 16px; + } + + .job-grid { + grid-template-columns: 1fr; + } + + .report-summary { + grid-template-columns: repeat(2, minmax(0, 1fr)); + } +} + +@media (max-width: 390px) { + .metric-grid { + grid-template-columns: 1fr; + } + + .observatory { + min-height: 0; + } + + .summary-value { + max-width: 125px; + } +} diff --git a/app/static/app.js b/app/static/app.js new file mode 100644 index 0000000..1d96e12 --- /dev/null +++ b/app/static/app.js @@ -0,0 +1,679 @@ +const form = document.querySelector("#capture-form"); +const urlInput = document.querySelector("#capture-url"); +const submitButton = document.querySelector("#capture-submit"); +const formMessage = document.querySelector("#form-message"); +const jobGrid = document.querySelector("#job-grid"); +const emptyState = document.querySelector("#empty-state"); +const refreshButton = document.querySelector("#refresh-jobs"); +const serviceStatus = document.querySelector("#service-status"); +const reportDialog = document.querySelector("#report-dialog"); +const reportContent = document.querySelector("#report-content"); +const closeReportButton = document.querySelector("#close-report"); + +const controls = { + externalAssets: document.querySelector("#external-assets"), + parallelConnections: document.querySelector("#parallel-connections"), + parallelValue: document.querySelector("#parallel-value"), + maxPages: document.querySelector("#max-pages"), + maxDepth: document.querySelector("#max-depth"), + maxBytes: document.querySelector("#max-bytes"), + maxDuration: document.querySelector("#max-duration"), + retentionDays: document.querySelector("#retention-days"), + budgetSummary: document.querySelector("#budget-summary"), + assetTypes: [...document.querySelectorAll("[data-asset-type]")], + skipFileTypes: document.querySelector("#skip-file-types"), + useProxy: document.querySelector("#use-proxy"), + proxySummary: document.querySelector("#proxy-summary"), + proxyDetail: document.querySelector("#proxy-detail"), + proxyRow: document.querySelector(".proxy-row"), +}; + +const publicStats = { + websites: document.querySelector("#stat-websites"), + data: document.querySelector("#stat-data"), + active: document.querySelector("#stat-active"), + files: document.querySelector("#stat-files"), +}; + +const OWNER_STORAGE_KEY = "siteharbor.owner-token.v1"; +const SESSION_STORAGE_KEY = "siteharbor.sessions.v1"; +const OWNER_HEADER = "X-SiteHarbor-Session"; +const MAX_STORED_SESSIONS = 40; +const TOKEN_PATTERN = /^[A-Za-z0-9_-]{24,200}$/; + +const jobs = new Map(); +const streams = new Map(); +let storageAvailable = true; +const ownerToken = getOwnerToken(); + +function createOwnerToken() { + if (window.crypto?.randomUUID) return window.crypto.randomUUID(); + if (window.crypto?.getRandomValues) { + const bytes = new Uint8Array(24); + window.crypto.getRandomValues(bytes); + return [...bytes].map((byte) => byte.toString(16).padStart(2, "0")).join(""); + } + return `browser-${Date.now()}-${Math.random().toString(36).slice(2).padEnd(16, "0")}`; +} + +function getOwnerToken() { + try { + const existing = window.localStorage.getItem(OWNER_STORAGE_KEY); + if (existing && TOKEN_PATTERN.test(existing)) return existing; + const token = createOwnerToken(); + window.localStorage.setItem(OWNER_STORAGE_KEY, token); + return token; + } catch { + storageAvailable = false; + return createOwnerToken(); + } +} + +function readStoredSessions() { + if (!storageAvailable) return []; + try { + const raw = window.localStorage.getItem(SESSION_STORAGE_KEY); + if (!raw) return []; + const parsed = JSON.parse(raw); + const sessions = Array.isArray(parsed) ? parsed : parsed.sessions; + if (!Array.isArray(sessions)) return []; + return sessions.filter((session) => typeof session?.id === "string" && session.id.length > 0); + } catch { + storageAvailable = false; + return []; + } +} + +function writeStoredSessions(sessions) { + if (!storageAvailable) return; + try { + window.localStorage.setItem( + SESSION_STORAGE_KEY, + JSON.stringify({ version: 1, sessions: sessions.slice(0, MAX_STORED_SESSIONS) }), + ); + } catch { + storageAvailable = false; + } +} + +function snapshotJob(job) { + return { + id: job.id, + url: job.url, + state: job.state, + phase: job.phase, + message: job.message, + error_message: job.error_message, + warnings: Array.isArray(job.warnings) ? job.warnings.slice(0, 5) : [], + created_at: job.created_at, + updated_at: job.updated_at, + started_at: job.started_at, + completed_at: job.completed_at, + expires_at: job.expires_at, + cancel_requested: job.cancel_requested, + stats: job.stats || {}, + options: job.options || {}, + archive: job.archive || {}, + }; +} + +function rememberJob(job) { + const sessions = readStoredSessions(); + const record = { id: job.id, created_at: job.created_at, job: snapshotJob(job) }; + const existingIndex = sessions.findIndex((session) => session.id === job.id); + if (existingIndex >= 0) sessions[existingIndex] = { ...sessions[existingIndex], ...record }; + else sessions.unshift(record); + sessions.sort((left, right) => sortByCreated(right.job || right, left.job || left)); + writeStoredSessions(sessions); +} + +function forgetJob(jobId) { + jobs.delete(jobId); + closeEvents(jobId); + writeStoredSessions(readStoredSessions().filter((session) => session.id !== jobId)); + renderJobs(); +} + +function sortByCreated(left, right) { + return String(left.created_at || "").localeCompare(String(right.created_at || "")); +} + +function formatBytes(value) { + const amount = Number(value) || 0; + if (!amount) return "0 B"; + const units = ["B", "KB", "MB", "GB", "TB"]; + const index = Math.min(Math.floor(Math.log(amount) / Math.log(1024)), units.length - 1); + const rounded = amount / (1024 ** index); + return `${rounded >= 10 || index === 0 ? rounded.toFixed(0) : rounded.toFixed(1)} ${units[index]}`; +} + +function formatCount(value) { + return new Intl.NumberFormat(undefined, { notation: "compact", maximumFractionDigits: 1 }).format( + Number(value) || 0, + ); +} + +function formatTime(value) { + if (!value) return "--"; + const date = new Date(value); + return Number.isNaN(date.getTime()) + ? "--" + : date.toLocaleString([], { dateStyle: "medium", timeStyle: "short" }); +} + +function hostname(url) { + try { + return new URL(url).hostname; + } catch { + return url; + } +} + +function setMessage(message = "", kind = "") { + formMessage.textContent = message; + formMessage.className = `form-message ${kind}`.trim(); +} + +function setServiceStatus(message, online = false) { + serviceStatus.querySelector("span").textContent = message; + serviceStatus.dataset.online = String(online); +} + +function readNumber(control, fallback) { + const value = Number.parseInt(control.value, 10); + return Number.isFinite(value) ? value : fallback; +} + +function formatDuration(seconds) { + const value = Number(seconds) || 0; + if (value >= 60 && value % 60 === 0) return `${value / 60} min`; + return `${value} sec`; +} + +function updateBudgetSummary() { + controls.budgetSummary.textContent = [ + `${readNumber(controls.maxPages, 150)} pages`, + `${readNumber(controls.maxBytes, 100)} MB`, + formatDuration(readNumber(controls.maxDuration, 300)), + ].join(" / "); +} + +function updateParallelValue() { + controls.parallelValue.value = controls.parallelConnections.value; + controls.parallelValue.textContent = controls.parallelConnections.value; +} + +function updateProxySummary() { + if (controls.useProxy.disabled) { + controls.proxySummary.textContent = "Not configured"; + } else if (controls.useProxy.checked) { + controls.proxySummary.textContent = "Approved proxy"; + } else { + controls.proxySummary.textContent = "Direct connection"; + } +} + +function setControlLimits(capabilities) { + const defaults = capabilities.defaults || {}; + const limits = capabilities.limits || {}; + const bindings = [ + [controls.maxPages, "max_pages"], + [controls.maxDepth, "max_depth"], + [controls.maxBytes, "max_bytes_mb"], + [controls.maxDuration, "max_duration_seconds"], + [controls.retentionDays, "max_retention_days"], + ]; + bindings.forEach(([control, key]) => { + if (limits[key] !== undefined) control.max = limits[key]; + }); + if (limits.max_parallel_connections !== undefined) { + controls.parallelConnections.max = limits.max_parallel_connections; + } + + controls.maxPages.value = defaults.max_pages ?? controls.maxPages.value; + controls.maxDepth.value = defaults.max_depth ?? controls.maxDepth.value; + controls.maxBytes.value = defaults.max_bytes_mb ?? controls.maxBytes.value; + controls.maxDuration.value = defaults.max_duration_seconds ?? controls.maxDuration.value; + controls.retentionDays.value = defaults.retention_days ?? controls.retentionDays.value; + controls.externalAssets.checked = defaults.include_external_assets ?? true; + controls.parallelConnections.value = defaults.parallel_connections ?? controls.parallelConnections.value; + + const selectedTypes = new Set(defaults.download_asset_types || []); + controls.assetTypes.forEach((control) => { + control.checked = selectedTypes.has(control.dataset.assetType); + }); + controls.skipFileTypes.value = (defaults.skip_file_types || []).join(", "); + controls.useProxy.checked = Boolean(defaults.use_proxy); + controls.useProxy.disabled = !capabilities.proxy_available; + controls.proxyRow.classList.toggle("is-unavailable", !capabilities.proxy_available); + controls.proxyDetail.textContent = capabilities.proxy_available + ? "Use the operator-approved route for this capture." + : "No operator-approved proxy route is available on this service."; + + updateBudgetSummary(); + updateParallelValue(); + updateProxySummary(); +} + +async function request(path, options = {}, { session = true } = {}) { + const headers = new Headers(options.headers || {}); + if (session) headers.set(OWNER_HEADER, ownerToken); + if (options.body && !headers.has("Content-Type")) headers.set("Content-Type", "application/json"); + + const response = await fetch(path, { cache: "no-store", ...options, headers }); + const body = response.status === 204 ? null : await response.json().catch(() => null); + if (!response.ok) { + const error = new Error(body?.detail || "SiteHarbor could not complete that request."); + error.status = response.status; + throw error; + } + return body; +} + +function isActive(job) { + return ["queued", "running"].includes(job.state); +} + +function upsertJob(job, { persist = true, connect = true } = {}) { + jobs.set(job.id, job); + if (persist) rememberJob(job); + renderJobs(); + if (connect && isActive(job)) connectEvents(job.id); + else if (!isActive(job)) closeEvents(job.id); +} + +function renderJobs() { + const orderedJobs = [...jobs.values()].sort((left, right) => sortByCreated(right, left)); + emptyState.hidden = orderedJobs.length > 0; + jobGrid.replaceChildren(...orderedJobs.map(createJobCard)); +} + +function createElement(tag, className, text) { + const element = document.createElement(tag); + if (className) element.className = className; + if (text !== undefined) element.textContent = text; + return element; +} + +function jobProgress(job) { + if (job.state === "ready") return 100; + if (["failed", "cancelled", "expired"].includes(job.state)) return 100; + const phase = String(job.phase || "").toLowerCase(); + if (phase.includes("packag")) return 94; + if (phase.includes("rewrit")) return 88; + if (phase.includes("validat") || phase.includes("prepar")) return 6; + const maximum = Number(job.options?.max_pages) || 0; + const fetched = Number(job.stats?.pages_fetched) || 0; + if (!maximum) return 16; + return Math.max(10, Math.min(82, Math.round((fetched / maximum) * 82))); +} + +function createJobCard(job) { + const card = createElement("article", "job-card"); + card.dataset.state = job.state; + + const topLine = createElement("div", "job-topline"); + topLine.append( + createElement("div", "job-host", hostname(job.url)), + createElement("span", `state-pill ${job.state}`, job.phase || job.state), + ); + card.append(topLine); + card.append(createElement("p", "job-url", job.url)); + card.append(createElement("p", "job-phase", job.error_message || job.message || "Waiting for status")); + + const progress = createElement("div", "job-progress"); + progress.append(createElement("span")); + progress.firstElementChild.style.setProperty("--progress", `${jobProgress(job)}%`); + card.append(progress); + + const stats = createElement("div", "job-stats"); + const statItems = [ + [job.stats?.pages_fetched || 0, "pages"], + [job.stats?.files_written || 0, "files"], + [formatBytes(job.stats?.bytes_downloaded || 0), "collected"], + ]; + statItems.forEach(([value, label]) => { + const item = document.createElement("div"); + item.append(createElement("strong", "", String(value)), createElement("span", "", label)); + stats.append(item); + }); + card.append(stats); + + const meta = createElement("div", "job-meta"); + meta.append( + createElement("span", "", `Created ${formatTime(job.created_at)}`), + createElement( + "span", + "", + job.archive?.available + ? `Archive ${formatBytes(job.archive.size_bytes)}` + : `${job.options?.max_pages || "--"} page budget`, + ), + ); + card.append(meta); + + const tags = createElement("div", "job-tags"); + const parallel = job.options?.parallel_connections; + if (parallel) tags.append(createElement("span", "", `${parallel} connections`)); + tags.append( + createElement( + "span", + "", + job.options?.include_external_assets ? "external assets" : "first-party only", + ), + ); + if (job.options?.skip_file_types?.length) { + tags.append(createElement("span", "", `skip .${job.options.skip_file_types[0]}`)); + } + card.append(tags); + + if (job.warnings?.length) card.append(createElement("p", "job-warning", job.warnings[0])); + + const actions = createElement("div", "job-actions"); + if (job.archive?.available) { + const download = createElement("button", "job-action primary", "Download ZIP"); + download.type = "button"; + download.addEventListener("click", () => downloadArchive(job, download)); + actions.append(download); + + const report = createElement("button", "job-action", "View report"); + report.type = "button"; + report.addEventListener("click", () => showReport(job.id)); + actions.append(report); + + const remove = createElement("button", "job-action danger", "Delete archive"); + remove.type = "button"; + remove.addEventListener("click", () => deleteJob(job.id)); + actions.append(remove); + } + if (isActive(job)) { + const cancel = createElement("button", "job-action danger", job.cancel_requested ? "Cancelling" : "Cancel"); + cancel.type = "button"; + cancel.disabled = Boolean(job.cancel_requested); + cancel.addEventListener("click", () => cancelJob(job.id)); + actions.append(cancel); + } + if (!isActive(job) && !job.archive?.available) { + const forget = createElement("button", "job-action", "Forget session"); + forget.type = "button"; + forget.addEventListener("click", () => forgetJob(job.id)); + actions.append(forget); + } + card.append(actions); + return card; +} + +function connectEvents(jobId) { + if (streams.has(jobId)) return; + const eventUrl = `/api/captures/${encodeURIComponent(jobId)}/events?session=${encodeURIComponent(ownerToken)}`; + const stream = new EventSource(eventUrl); + streams.set(jobId, stream); + ["state", "progress"].forEach((eventName) => { + stream.addEventListener(eventName, () => { + void refreshJob(jobId); + }); + }); + stream.addEventListener("terminal", () => { + closeEvents(jobId); + void refreshJob(jobId); + void loadPublicStats(); + }); + stream.onerror = () => { + window.setTimeout(() => { + if (streams.get(jobId) === stream) void refreshJob(jobId); + }, 1_000); + }; +} + +function closeEvents(jobId) { + const stream = streams.get(jobId); + if (stream) stream.close(); + streams.delete(jobId); +} + +async function refreshJob(jobId) { + try { + upsertJob(await request(`/api/captures/${encodeURIComponent(jobId)}`)); + } catch (error) { + closeEvents(jobId); + if (error.status === 404) forgetJob(jobId); + } +} + +async function refreshKnownJobs() { + const storedIds = readStoredSessions().map((session) => session.id); + const knownIds = [...new Set([...storedIds, ...jobs.keys()])]; + await Promise.all(knownIds.map((jobId) => refreshJob(jobId))); +} + +async function loadStoredJobs() { + const sessions = readStoredSessions(); + sessions.forEach((session) => { + if (session.job?.id && !jobs.has(session.id)) jobs.set(session.id, session.job); + }); + renderJobs(); + await refreshKnownJobs(); +} + +async function loadPublicStats() { + try { + const stats = await request("/api/stats", {}, { session: false }); + publicStats.websites.textContent = formatCount(stats.websites_cloned); + publicStats.data.textContent = formatBytes(stats.bytes_scraped); + publicStats.active.textContent = formatCount(stats.active_crawls); + publicStats.files.textContent = formatCount(stats.files_archived); + setServiceStatus("Capture service online", true); + } catch { + setServiceStatus("Service unavailable", false); + } +} + +async function cancelJob(jobId) { + try { + upsertJob(await request(`/api/captures/${encodeURIComponent(jobId)}/cancel`, { method: "POST" })); + void loadPublicStats(); + } catch (error) { + setMessage(error.message); + } +} + +async function deleteJob(jobId) { + try { + const updated = await request(`/api/captures/${encodeURIComponent(jobId)}`, { method: "DELETE" }); + if (updated.state === "expired") forgetJob(jobId); + else upsertJob(updated); + void loadPublicStats(); + } catch (error) { + setMessage(error.message); + } +} + +async function downloadArchive(job, button) { + if (!job.archive?.download_url) return; + const originalText = button.textContent; + button.disabled = true; + button.textContent = "Preparing download"; + try { + const response = await fetch(job.archive.download_url, { + headers: { [OWNER_HEADER]: ownerToken }, + }); + if (!response.ok) { + const body = await response.json().catch(() => null); + const error = new Error(body?.detail || "The archive could not be downloaded."); + error.status = response.status; + throw error; + } + const objectUrl = URL.createObjectURL(await response.blob()); + const anchor = document.createElement("a"); + anchor.href = objectUrl; + anchor.download = `siteharbor-${job.id}.zip`; + document.body.append(anchor); + anchor.click(); + anchor.remove(); + window.setTimeout(() => URL.revokeObjectURL(objectUrl), 1_000); + } catch (error) { + setMessage(error.message); + } finally { + button.disabled = false; + button.textContent = originalText; + } +} + +async function showReport(jobId) { + try { + const report = await request(`/api/captures/${encodeURIComponent(jobId)}/report`); + renderReport(report); + reportDialog.showModal(); + } catch (error) { + setMessage(error.message); + } +} + +function renderReport(report) { + reportContent.replaceChildren(); + const summary = createElement("div", "report-summary"); + const stats = report.stats || {}; + [ + [stats.pages_fetched || 0, "pages"], + [stats.assets_fetched || 0, "assets"], + [formatBytes(stats.bytes_downloaded || 0), "downloaded"], + [report.quality || "unknown", "capture quality"], + ].forEach(([value, label]) => { + const item = document.createElement("div"); + item.append(createElement("strong", "", String(value)), createElement("span", "", label)); + summary.append(item); + }); + reportContent.append(summary); + + const details = createElement("ul", "report-list"); + [ + `Entry point: ${report.entry_point || "not available"}`, + `Capture duration: ${report.elapsed_seconds || 0} seconds`, + `Captured resources: ${(report.resources || []).length}`, + ].forEach((line) => details.append(createElement("li", "", line))); + reportContent.append(details); + + if (report.warnings?.length) { + reportContent.append(createElement("h3", "report-subhead", "Warnings")); + const warnings = createElement("ul", "report-list"); + report.warnings.forEach((warning) => warnings.append(createElement("li", "", warning))); + reportContent.append(warnings); + } + if (report.errors?.length) { + reportContent.append(createElement("h3", "report-subhead", "Resource failures")); + const errors = createElement("ul", "report-list"); + report.errors.slice(0, 20).forEach((error) => { + errors.append(createElement("li", "", `${error.url}: ${error.message}`)); + }); + reportContent.append(errors); + } +} + +function selectedAssetTypes() { + return controls.assetTypes + .filter((control) => control.checked) + .map((control) => control.dataset.assetType); +} + +function skippedFileTypes() { + return controls.skipFileTypes.value + .split(",") + .map((item) => item.trim()) + .filter(Boolean); +} + +form.addEventListener("submit", async (event) => { + event.preventDefault(); + const url = urlInput.value.trim(); + if (!url) { + setMessage("Enter a public website URL to begin."); + urlInput.focus(); + return; + } + + setMessage(""); + submitButton.disabled = true; + submitButton.querySelector("span").textContent = "Creating crawl"; + try { + const job = await request("/api/captures", { + method: "POST", + body: JSON.stringify({ + url, + include_external_assets: controls.externalAssets.checked, + parallel_connections: readNumber(controls.parallelConnections, 6), + download_asset_types: selectedAssetTypes(), + skip_file_types: skippedFileTypes(), + use_proxy: controls.useProxy.checked && !controls.useProxy.disabled, + max_pages: readNumber(controls.maxPages, 150), + max_depth: readNumber(controls.maxDepth, 5), + max_bytes_mb: readNumber(controls.maxBytes, 100), + max_duration_seconds: readNumber(controls.maxDuration, 300), + retention_days: readNumber(controls.retentionDays, 7), + }), + }); + upsertJob(job); + setMessage("Crawl queued. This browser will reconnect to it automatically.", "success"); + urlInput.value = ""; + void loadPublicStats(); + } catch (error) { + setMessage(error.message); + } finally { + submitButton.disabled = false; + submitButton.querySelector("span").textContent = "Start private crawl"; + } +}); + +[ + controls.maxPages, + controls.maxDepth, + controls.maxBytes, + controls.maxDuration, + controls.retentionDays, +].forEach((control) => control.addEventListener("input", updateBudgetSummary)); +controls.parallelConnections.addEventListener("input", updateParallelValue); +controls.useProxy.addEventListener("change", updateProxySummary); + +refreshButton.addEventListener("click", async () => { + refreshButton.disabled = true; + try { + await Promise.all([refreshKnownJobs(), loadPublicStats()]); + } finally { + refreshButton.disabled = false; + } +}); +closeReportButton.addEventListener("click", () => reportDialog.close()); +reportDialog.addEventListener("click", (event) => { + if (event.target === reportDialog) reportDialog.close(); +}); + +document.addEventListener("visibilitychange", () => { + if (!document.hidden) { + void refreshKnownJobs(); + void loadPublicStats(); + } +}); +window.addEventListener("online", () => { + void refreshKnownJobs(); + void loadPublicStats(); +}); +window.addEventListener("beforeunload", () => { + streams.forEach((stream) => stream.close()); +}); + +async function initialize() { + if (!storageAvailable) { + setMessage("Browser storage is unavailable, so this session cannot be restored after this tab closes."); + } + try { + setControlLimits(await request("/api/capabilities", {}, { session: false })); + } catch (error) { + setMessage(error.message); + } + await Promise.all([loadStoredJobs(), loadPublicStats()]); + window.setInterval(() => { + void refreshKnownJobs(); + void loadPublicStats(); + }, 15_000); +} + +void initialize(); diff --git a/app/templates/index.html b/app/templates/index.html new file mode 100644 index 0000000..5264e5e --- /dev/null +++ b/app/templates/index.html @@ -0,0 +1,238 @@ + + + + + + + + {{ app_name }} - Public web capture + + + + +
+
+ + + siteharbor + +
+ PUBLIC WEB CAPTURE + + Checking service + +
+
+ +
+
+
+
+

LIVE NETWORK SIGNAL

+

Copy the web.
Keep it close.

+

+ Build an offline-ready snapshot of a public site. The service runs the crawl; + this browser keeps the key to its own sessions. +

+
+ +
+
+ -- + Websites cloned +
+
+ -- + Data collected +
+
+ -- + Crawls running +
+
+ -- + Files preserved +
+
+ +
+ +

Metrics are shared totals. Crawl URLs and archives stay private to the browser session that created them.

+
+
+ +
+
+
+

NEW SESSION

+

Set your crawl.

+
+ 01 / 01 +
+ + + +
+ + + +
+ +
+
+ + Crawl limits + 150 pages / 100 MB / 5 min + +
+ + + + + +
+
+ +
+ + File rules + Choose what comes aboard + +
+

Pages are always copied. Select the asset groups that should be downloaded.

+
+ + + + + + +
+ +
+
+ +
+ + Network route + Direct connection + + +
+
+ +

+
+ +

Only crawl sites you are allowed to archive. Private networks and unsafe ports are blocked.

+
+
+
+ +
+
+
+

THIS BROWSER

+

Your local sessions

+

Session records stay in this browser's local storage. Reopen this page to reconnect to an active crawl.

+
+ +
+ +
+ +
+

No local sessions yet

+

Start a crawl above. Its status and archive will remain available from this browser after a reconnect.

+
+
+
+
+
+ +
+ SiteHarbor captures browser-delivered static content. + Dynamic APIs, logins, service workers, and server behavior may not work offline. +
+
+ + +
+
+

CAPTURE REPORT

+

Archive details

+
+ +
+
+ + + diff --git a/app/url_policy.py b/app/url_policy.py new file mode 100644 index 0000000..c5d71ea --- /dev/null +++ b/app/url_policy.py @@ -0,0 +1,109 @@ +from __future__ import annotations + +import ipaddress +import re +import socket +from dataclasses import dataclass +from urllib.parse import quote, urlsplit, urlunsplit + + +class UrlPolicyError(ValueError): + """Raised when a URL is structurally unsafe or points at a blocked network.""" + + +@dataclass(frozen=True, slots=True) +class NetworkRules: + allow_private_networks: bool = False + allow_nonstandard_ports: bool = False + + +def normalize_url(value: str, rules: NetworkRules) -> str: + raw = value.strip() + if not raw: + raise UrlPolicyError("Enter a website address.") + + if "://" not in raw: + raw = f"https://{raw}" + + try: + parsed = urlsplit(raw) + port = parsed.port + except ValueError as error: + raise UrlPolicyError("The website address has an invalid port.") from error + + if parsed.scheme.lower() not in {"http", "https"}: + raise UrlPolicyError("Only HTTP and HTTPS websites can be captured.") + if not parsed.hostname: + raise UrlPolicyError("The website address needs a hostname.") + if parsed.username or parsed.password: + raise UrlPolicyError("Website addresses with embedded credentials are not allowed.") + + host = parsed.hostname.rstrip(".").lower() + try: + host = host.encode("idna").decode("ascii") + except UnicodeError as error: + raise UrlPolicyError("The website hostname is not valid.") from error + + default_port = 443 if parsed.scheme.lower() == "https" else 80 + if port and port != default_port and not rules.allow_nonstandard_ports: + raise UrlPolicyError("Only ports 80 and 443 are allowed by this SiteHarbor instance.") + + host_for_netloc = f"[{host}]" if ":" in host else host + netloc = host_for_netloc if not port or port == default_port else f"{host_for_netloc}:{port}" + path_value = parsed.path or "/" + if re.search(r"%(?![0-9A-Fa-f]{2})", path_value): + raise UrlPolicyError("The website address has an invalid percent escape.") + # Keep encoded delimiters such as %2F intact: decoding them changes resource identity. + path = quote(path_value, safe="/%:@!$&'()*+,;=-._~") + return urlunsplit((parsed.scheme.lower(), netloc, path, parsed.query, "")) + + +def canonical_url(value: str, rules: NetworkRules) -> str: + """Return a stable URL key with its fragment removed.""" + return normalize_url(value, rules) + + +def hostname(value: str) -> str: + host = urlsplit(value).hostname + if not host: + raise UrlPolicyError("The URL has no hostname.") + return host.rstrip(".").lower() + + +def is_public_address(address: ipaddress.IPv4Address | ipaddress.IPv6Address) -> bool: + return bool( + address.is_global + and not address.is_loopback + and not address.is_link_local + and not address.is_multicast + and not address.is_unspecified + and not address.is_reserved + and not address.is_private + ) + + +def resolve_and_validate(value: str, rules: NetworkRules) -> tuple[str, ...]: + """Resolve a host and reject non-public destinations unless explicitly in dev mode.""" + host = hostname(value) + try: + results = socket.getaddrinfo(host, None, type=socket.SOCK_STREAM) + except socket.gaierror as error: + raise UrlPolicyError(f"Could not resolve {host}.") from error + + addresses: set[str] = set() + for _, _, _, _, sockaddr in results: + address = ipaddress.ip_address(sockaddr[0]) + addresses.add(str(address)) + if not rules.allow_private_networks and not is_public_address(address): + raise UrlPolicyError("Private, local, and reserved network targets are blocked.") + + if not addresses: + raise UrlPolicyError(f"Could not resolve {host}.") + return tuple(sorted(addresses)) + + +def same_site_host(left: str, right: str) -> bool: + """Allow an exact host or the common example.com/www.example.com redirect pair.""" + left = left.lower().removeprefix("www.") + right = right.lower().removeprefix("www.") + return left == right diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 0000000..b603149 --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,44 @@ +[build-system] +requires = ["hatchling>=1.25"] +build-backend = "hatchling.build" + +[project] +name = "siteharbor" +version = "0.1.0" +description = "A standalone, bounded web capture service for offline archives." +readme = "README.md" +requires-python = ">=3.11" +license = { text = "MIT" } +authors = [{ name = "SiteHarbor" }] +dependencies = [ + "beautifulsoup4>=4.12,<5", + "fastapi>=0.115,<1", + "httpcore>=1,<2", + "httpx>=0.27,<1", + "jinja2>=3.1,<4", + "uvicorn[standard]>=0.30,<1", +] + +[project.optional-dependencies] +dev = [ + "pytest>=8.3,<9", + "pytest-cov>=5,<7", + "ruff>=0.8,<1", +] + +[project.scripts] +siteharbor = "app.main:run" + +[tool.hatch.build.targets.wheel] +packages = ["app"] + +[tool.pytest.ini_options] +testpaths = ["tests"] +addopts = "-q" + +[tool.ruff] +line-length = 100 +target-version = "py311" + +[tool.ruff.lint] +select = ["E", "F", "I", "UP", "B"] diff --git a/tests/test_capture_service.py b/tests/test_capture_service.py new file mode 100644 index 0000000..26ac78d --- /dev/null +++ b/tests/test_capture_service.py @@ -0,0 +1,91 @@ +from __future__ import annotations + +import threading +import zipfile +from dataclasses import replace +from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer + +from app.capture import CaptureService +from app.config import settings +from app.db import Database + + +class MinimalSiteHandler(BaseHTTPRequestHandler): + def do_GET(self) -> None: # noqa: N802 + if self.path == "/": + body = ( + b"" + b"Home" + ) + content_type = "text/html" + status = 200 + elif self.path == "/site.css": + body = b"body { color: #123456; }" + content_type = "text/css" + status = 200 + else: + body = b"not found" + content_type = "text/plain" + status = 404 + self.send_response(status) + self.send_header("Content-Type", content_type) + self.send_header("Content-Length", str(len(body))) + self.end_headers() + self.wfile.write(body) + + def log_message(self, format: str, *args) -> None: # type: ignore[override] + return + + +def test_capture_service_creates_private_archive(tmp_path) -> None: + server = ThreadingHTTPServer(("127.0.0.1", 0), MinimalSiteHandler) + thread = threading.Thread(target=server.serve_forever, daemon=True) + thread.start() + port = server.server_address[1] + + test_settings = replace( + settings, + data_dir=tmp_path, + database_path=tmp_path / "siteharbor.sqlite3", + work_dir=tmp_path / "work", + artifacts_dir=tmp_path / "artifacts", + reports_dir=tmp_path / "reports", + allow_private_networks=True, + allow_nonstandard_ports=True, + respect_robots=False, + ) + test_settings.ensure_directories() + database = Database(test_settings.database_path) + database.initialize() + source_url = f"http://127.0.0.1:{port}/" + job = database.create_job( + source_url, + source_url, + { + "include_external_assets": False, + "max_pages": 10, + "max_depth": 2, + "max_bytes": 2_000_000, + "max_duration_seconds": 30, + "retention_days": 1, + }, + ) + claimed = database.claim_next("test-worker", lease_seconds=120) + assert claimed is not None + + try: + CaptureService(database, test_settings, "test-worker").run(claimed) + finally: + server.shutdown() + server.server_close() + + completed = database.get_job(job["id"]) + assert completed is not None + assert completed["state"] == "ready" + assert completed["artifact_path"] + assert completed["report_path"] + + with zipfile.ZipFile(completed["artifact_path"]) as archive: + names = archive.namelist() + assert any(name.endswith("siteharbor-manifest.json") for name in names) + assert any(name.endswith("site.css") for name in names) diff --git a/tests/test_crawler.py b/tests/test_crawler.py new file mode 100644 index 0000000..aa7e535 --- /dev/null +++ b/tests/test_crawler.py @@ -0,0 +1,101 @@ +from __future__ import annotations + +import threading +from dataclasses import replace +from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer + +from app.config import settings +from app.crawler import CaptureOptions, SiteCrawler + + +class FixtureHandler(BaseHTTPRequestHandler): + fixtures = { + "/robots.txt": ("text/plain", b"User-agent: *\nAllow: /\n"), + "/": ( + "text/html", + b""" + + + + + About + """, + ), + "/about/": ("text/html", b"Home"), + "/assets/site.css": ( + "text/css", + b"@import '/assets/nested.css'; .hero { background: url('/assets/hero.png'); }", + ), + "/assets/nested.css": ("text/css", b".logo { background: url('/assets/logo.svg'); }"), + "/assets/app.js": ("application/javascript", b"import('/assets/chunk.js');"), + "/assets/chunk.js": ("application/javascript", b"export const loaded = true;"), + "/assets/logo.svg": ("image/svg+xml", b""), + "/assets/hero.png": ("image/png", b"not-a-real-png-but-a-static-asset"), + } + + def do_GET(self) -> None: # noqa: N802 + content_type, body = self.fixtures.get(self.path, ("text/plain", b"not found")) + status = 200 if self.path in self.fixtures else 404 + self.send_response(status) + self.send_header("Content-Type", content_type) + self.send_header("Content-Length", str(len(body))) + self.end_headers() + self.wfile.write(body) + + def log_message(self, format: str, *args) -> None: # type: ignore[override] + return + + +def test_crawler_captures_and_rewrites_static_fixture(tmp_path) -> None: + server = ThreadingHTTPServer(("127.0.0.1", 0), FixtureHandler) + thread = threading.Thread(target=server.serve_forever, daemon=True) + thread.start() + port = server.server_address[1] + + test_settings = replace( + settings, + data_dir=tmp_path, + work_dir=tmp_path / "work", + artifacts_dir=tmp_path / "artifacts", + reports_dir=tmp_path / "reports", + allow_private_networks=True, + allow_nonstandard_ports=True, + respect_robots=True, + fetch_concurrency=3, + ) + events: list[tuple[str, str]] = [] + crawler = SiteCrawler( + settings=test_settings, + options=CaptureOptions( + source_url=f"http://127.0.0.1:{port}/", + include_external_assets=False, + max_pages=10, + max_depth=3, + max_bytes=2_000_000, + max_duration_seconds=30, + ), + cancelled=lambda: False, + on_progress=lambda phase, message, stats, warnings: events.append((phase, message)), + ) + + try: + result = crawler.run(tmp_path / "work") + finally: + server.shutdown() + server.server_close() + + host_dir = result.site_dir / "127.0.0.1" + root_html = (host_dir / "index.html").read_text(encoding="utf-8") + css = (host_dir / "assets" / "site.css").read_text(encoding="utf-8") + javascript = (host_dir / "assets" / "app.js").read_text(encoding="utf-8") + + assert result.entry_point == "127.0.0.1/index.html" + assert result.stats["pages_fetched"] == 2 + assert result.stats["assets_fetched"] >= 6 + assert "about/index.html" in root_html + assert "assets/site.css" in root_html + assert "hero.png" in css + assert "import('chunk.js')" in javascript + assert (host_dir / "assets" / "chunk.js").is_file() + assert (result.site_dir / "siteharbor-manifest.json").is_file() + assert events diff --git a/tests/test_database.py b/tests/test_database.py new file mode 100644 index 0000000..9dc8550 --- /dev/null +++ b/tests/test_database.py @@ -0,0 +1,161 @@ +from __future__ import annotations + +from app.db import Database + + +def capture_config() -> dict[str, int | bool]: + return { + "include_external_assets": True, + "max_pages": 20, + "max_depth": 3, + "max_bytes": 1_000_000, + "max_duration_seconds": 60, + "retention_days": 1, + } + + +def test_sqlite_lease_claims_one_queued_job(tmp_path) -> None: + database = Database(tmp_path / "siteharbor.sqlite3") + database.initialize() + job = database.create_job( + "https://example.com", + "https://example.com/", + capture_config(), + ) + + claimed = database.claim_next("worker-a", lease_seconds=30) + + assert claimed is not None + assert claimed["id"] == job["id"] + assert claimed["state"] == "running" + assert database.claim_next("worker-b", lease_seconds=30) is None + + assert database.update_progress( + job["id"], + "worker-a", + 30, + phase="Packaging", + message="Creating archive", + stats={ + "pages_found": 1, + "pages_fetched": 1, + "assets_found": 0, + "assets_fetched": 0, + "bytes_downloaded": 10, + "files_written": 1, + }, + warnings=[], + ) + assert database.mark_ready( + job["id"], + "worker-a", + artifact_path=str(tmp_path / "capture.zip"), + report_path=str(tmp_path / "capture.json"), + archive_size=10, + entry_point="example.com/index.html", + stats={ + "pages_found": 1, + "pages_fetched": 1, + "assets_found": 0, + "assets_fetched": 0, + "bytes_downloaded": 10, + "files_written": 1, + }, + warnings=[], + retention_days=1, + ) + ready = database.get_job(job["id"]) + assert ready is not None + assert ready["state"] == "ready" + assert ready["expires_at"] > ready["completed_at"] + + +def test_worker_cannot_cancel_without_a_user_cancellation_request(tmp_path) -> None: + database = Database(tmp_path / "siteharbor.sqlite3") + database.initialize() + job = database.create_job( + "https://example.com", + "https://example.com/", + { + **capture_config(), + "include_external_assets": False, + "max_pages": 1, + "max_depth": 0, + "max_bytes": 1_000, + "max_duration_seconds": 30, + }, + ) + assert database.claim_next("worker-a", lease_seconds=30) + + assert not database.mark_cancelled(job["id"], "worker-a", "Lease was lost") + running = database.get_job(job["id"]) + assert running is not None + assert running["state"] == "running" + + database.request_cancel(job["id"]) + assert database.mark_cancelled(job["id"], "worker-a", "Cancelled by user") + cancelled = database.get_job(job["id"]) + assert cancelled is not None + assert cancelled["state"] == "cancelled" + + +def test_expired_cancel_requested_job_is_terminalized_before_reclaim(tmp_path) -> None: + database = Database(tmp_path / "siteharbor.sqlite3") + database.initialize() + job = database.create_job( + "https://example.com", + "https://example.com/", + { + **capture_config(), + "include_external_assets": False, + "max_pages": 1, + "max_depth": 0, + "max_bytes": 1_000, + "max_duration_seconds": 30, + }, + ) + assert database.claim_next("worker-a", lease_seconds=30) + with database.connection() as connection: + connection.execute( + """ + UPDATE jobs + SET cancel_requested = 1, lease_expires_at = '2000-01-01T00:00:00+00:00' + WHERE id = ? + """, + (job["id"],), + ) + + assert database.claim_next("worker-b", lease_seconds=30) is None + cancelled = database.get_job(job["id"]) + assert cancelled is not None + assert cancelled["state"] == "cancelled" + + +def test_owner_token_scopes_jobs_and_public_stats(tmp_path) -> None: + database = Database(tmp_path / "siteharbor.sqlite3") + database.initialize() + first_owner = "a" * 32 + second_owner = "b" * 32 + first = database.create_job( + "https://one.example", + "https://one.example/", + capture_config(), + first_owner, + ) + second = database.create_job( + "https://two.example", + "https://two.example/", + capture_config(), + second_owner, + ) + + assert database.get_owned_job(first["id"], first_owner) is not None + assert database.get_owned_job(first["id"], second_owner) is None + assert [job["id"] for job in database.list_owned_jobs(first_owner)] == [first["id"]] + assert [job["id"] for job in database.list_owned_jobs(second_owner)] == [second["id"]] + assert database.public_stats() == { + "websites_cloned": 0, + "bytes_scraped": 0, + "active_crawls": 2, + "files_archived": 0, + } diff --git a/tests/test_main.py b/tests/test_main.py new file mode 100644 index 0000000..17d319f --- /dev/null +++ b/tests/test_main.py @@ -0,0 +1,67 @@ +from __future__ import annotations + +from dataclasses import replace + +from fastapi.testclient import TestClient + +import app.main as main +from app.config import settings +from app.db import Database +from app.scheduler import JobRunner + + +def test_capture_routes_are_scoped_to_the_browser_session(tmp_path, monkeypatch) -> None: + test_settings = replace( + settings, + data_dir=tmp_path, + database_path=tmp_path / "siteharbor.sqlite3", + work_dir=tmp_path / "work", + artifacts_dir=tmp_path / "artifacts", + reports_dir=tmp_path / "reports", + worker_enabled=False, + ) + test_settings.ensure_directories() + test_database = Database(test_settings.database_path) + monkeypatch.setattr(main, "settings", test_settings) + monkeypatch.setattr(main, "database", test_database) + monkeypatch.setattr(main, "runner", JobRunner(test_database, test_settings)) + + owner_a = "a" * 32 + owner_b = "b" * 32 + with TestClient(main.app) as client: + homepage = client.get("/") + assert homepage.status_code == 200 + assert "Copy the web." in homepage.text + assert client.get("/api/captures").status_code == 401 + + created = client.post( + "/api/captures", + headers={"X-SiteHarbor-Session": owner_a}, + json={ + "url": "https://example.com", + "parallel_connections": 2, + "download_asset_types": ["styles", "images"], + "skip_file_types": ["pdf, zip"], + }, + ) + assert created.status_code == 202 + created_job = created.json() + job_id = created_job["id"] + assert created_job["options"]["parallel_connections"] == 2 + assert created_job["options"]["download_asset_types"] == ["styles", "images"] + assert created_job["options"]["skip_file_types"] == ["pdf", "zip"] + + assert client.get( + f"/api/captures/{job_id}", headers={"X-SiteHarbor-Session": owner_b} + ).status_code == 404 + assert client.post( + f"/api/captures/{job_id}/cancel", headers={"X-SiteHarbor-Session": owner_b} + ).status_code == 404 + + owned_jobs = client.get("/api/captures", headers={"X-SiteHarbor-Session": owner_a}) + assert owned_jobs.status_code == 200 + assert [job["id"] for job in owned_jobs.json()["items"]] == [job_id] + + stats = client.get("/api/stats") + assert stats.status_code == 200 + assert stats.json()["active_crawls"] == 1 diff --git a/tests/test_pinned_transport.py b/tests/test_pinned_transport.py new file mode 100644 index 0000000..51e6605 --- /dev/null +++ b/tests/test_pinned_transport.py @@ -0,0 +1,31 @@ +from __future__ import annotations + +import pytest + +from app.pinned_transport import PinnedAddressBook, PinnedNetworkBackend + + +def test_pinned_backend_dials_the_validated_address_not_hostname(monkeypatch) -> None: + address_book = PinnedAddressBook() + address_book.pin("example.com", 443, ["93.184.216.34"]) + backend = PinnedNetworkBackend(address_book) + called: dict[str, object] = {} + + class StubBackend: + def connect_tcp(self, host, port, **kwargs): + called["host"] = host + called["port"] = port + return object() + + monkeypatch.setattr(backend, "_backend", StubBackend()) + + backend.connect_tcp("example.com", 443) + + assert called == {"host": "93.184.216.34", "port": 443} + + +def test_pinned_backend_rejects_unvalidated_host() -> None: + backend = PinnedNetworkBackend(PinnedAddressBook()) + + with pytest.raises(Exception, match="No validated address"): + backend.connect_tcp("example.com", 443) diff --git a/tests/test_url_policy.py b/tests/test_url_policy.py new file mode 100644 index 0000000..36815bf --- /dev/null +++ b/tests/test_url_policy.py @@ -0,0 +1,38 @@ +from __future__ import annotations + +import socket + +import pytest + +from app.url_policy import NetworkRules, UrlPolicyError, normalize_url, resolve_and_validate + + +def test_normalize_url_adds_https_and_removes_fragment() -> None: + result = normalize_url("example.com/docs#section", NetworkRules()) + + assert result == "https://example.com/docs" + + +def test_normalize_url_preserves_encoded_path_delimiters() -> None: + result = normalize_url("https://example.com/a%2Fb", NetworkRules()) + + assert result == "https://example.com/a%2Fb" + + +def test_normalize_url_rejects_credentials_and_unsafe_ports() -> None: + with pytest.raises(UrlPolicyError, match="credentials"): + normalize_url("https://user:secret@example.com", NetworkRules()) + + with pytest.raises(UrlPolicyError, match="ports 80 and 443"): + normalize_url("https://example.com:8443", NetworkRules()) + + +def test_resolve_blocks_loopback_by_default(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setattr( + socket, + "getaddrinfo", + lambda *args, **kwargs: [(socket.AF_INET, socket.SOCK_STREAM, 6, "", ("127.0.0.1", 0))], + ) + + with pytest.raises(UrlPolicyError, match="Private, local, and reserved"): + resolve_and_validate("https://example.com", NetworkRules())