__init__

app/__init__.py

@@ -0,0 +1 @@
+"""SiteHarbor application package."""

app/capture.py

@@ -0,0 +1,302 @@
+from __future__ import annotations
+
+import json
+import os
+import shutil
+import threading
+import time
+import uuid
+import zipfile
+from collections.abc import Callable
+from pathlib import Path
+from typing import Any
+
+from app.config import Settings
+from app.crawler import (
+    DEFAULT_ASSET_TYPES,
+    CaptureCancelled,
+    CaptureError,
+    CaptureOptions,
+    SiteCrawler,
+)
+from app.db import Database
+from app.url_policy import NetworkRules, UrlPolicyError, resolve_and_validate
+
+
+def _remove_tree(path: Path, allowed_root: Path) -> None:
+    """Delete only a child path beneath the configured work root."""
+    root = allowed_root.resolve()
+    resolved = path.resolve()
+    if resolved == root or not resolved.is_relative_to(root):
+        raise RuntimeError("Refusing to remove a path outside SiteHarbor's work directory.")
+    shutil.rmtree(resolved, ignore_errors=True)
+
+
+def _create_archive(
+    source_dir: Path,
+    destination: Path,
+    archive_root: str,
+    cancelled: Callable[[], bool],
+    deadline_exceeded: Callable[[], bool],
+    heartbeat: Callable[[], None],
+) -> int:
+    partial = destination.with_suffix(".zip.partial")
+    partial.unlink(missing_ok=True)
+    last_heartbeat = time.monotonic()
+    try:
+        with zipfile.ZipFile(
+            partial,
+            mode="w",
+            compression=zipfile.ZIP_DEFLATED,
+            compresslevel=6,
+            strict_timestamps=False,
+        ) as archive:
+            for source_path in sorted(source_dir.rglob("*")):
+                if cancelled():
+                    raise CaptureCancelled("Capture cancelled while packaging the archive")
+                if deadline_exceeded():
+                    raise CaptureError(
+                        "The configured capture time limit was reached while packaging"
+                    )
+                if time.monotonic() - last_heartbeat >= 5:
+                    heartbeat()
+                    last_heartbeat = time.monotonic()
+                if not source_path.is_file() or source_path.is_symlink():
+                    continue
+                relative = source_path.relative_to(source_dir).as_posix()
+                with source_path.open("rb") as source_file:
+                    with archive.open(f"{archive_root}/{relative}", mode="w") as archive_file:
+                        while chunk := source_file.read(1_024 * 1_024):
+                            if cancelled():
+                                raise CaptureCancelled(
+                                    "Capture cancelled while packaging the archive"
+                                )
+                            if deadline_exceeded():
+                                raise CaptureError(
+                                    "The configured capture time limit was reached while packaging"
+                                )
+                            if time.monotonic() - last_heartbeat >= 5:
+                                heartbeat()
+                                last_heartbeat = time.monotonic()
+                            archive_file.write(chunk)
+        os.replace(partial, destination)
+        return destination.stat().st_size
+    except Exception:
+        partial.unlink(missing_ok=True)
+        destination.unlink(missing_ok=True)
+        raise
+
+
+class CaptureService:
+    """Runs one leased capture and persists a terminal SQLite job state."""
+
+    def __init__(self, database: Database, settings: Settings, worker_id: str) -> None:
+        self.database = database
+        self.settings = settings
+        self.worker_id = worker_id
+
+    def run(self, job: dict[str, Any]) -> None:
+        job_id = str(job["id"])
+        config = dict(job["config"])
+        attempt_id = uuid.uuid4().hex[:12]
+        work_dir = self.settings.work_dir / f"{job_id}-{attempt_id}"
+        artifact_path = self.settings.artifacts_dir / f"{job_id}-{attempt_id}.zip"
+        report_path = self.settings.reports_dir / f"{job_id}-{attempt_id}.json"
+        heartbeat_stop = threading.Event()
+        lease_lost = threading.Event()
+        heartbeat_thread: threading.Thread | None = None
+        capture_deadline = time.monotonic() + int(config["max_duration_seconds"])
+
+        def capture_cancelled() -> bool:
+            return (
+                lease_lost.is_set()
+                or self.database.is_cancel_requested(job_id)
+                or not self.database.owns_lease(job_id, self.worker_id)
+            )
+
+        def renew_lease() -> None:
+            interval = min(20, max(3, self.settings.lease_seconds // 3))
+            while not heartbeat_stop.wait(interval):
+                try:
+                    if not self.database.renew_lease(
+                        job_id, self.worker_id, self.settings.lease_seconds
+                    ):
+                        lease_lost.set()
+                        return
+                except Exception:
+                    # Ownership checks stop this attempt if the lease eventually expires.
+                    continue
+
+        try:
+            _remove_tree_if_present(work_dir, self.settings.work_dir)
+            work_dir.mkdir(parents=True, exist_ok=True)
+            if capture_cancelled():
+                raise CaptureCancelled("Capture cancelled before it started")
+
+            heartbeat_thread = threading.Thread(
+                target=renew_lease,
+                name=f"siteharbor-lease-{job_id[-8:]}",
+                daemon=True,
+            )
+            heartbeat_thread.start()
+
+            rules = NetworkRules(
+                allow_private_networks=self.settings.allow_private_networks,
+                allow_nonstandard_ports=self.settings.allow_nonstandard_ports,
+            )
+            resolve_and_validate(str(job["normalized_url"]), rules)
+            remaining_seconds = int(capture_deadline - time.monotonic())
+            if remaining_seconds <= 0:
+                raise CaptureError(
+                    "The configured capture time limit was reached during validation"
+                )
+            options = CaptureOptions(
+                source_url=str(job["normalized_url"]),
+                include_external_assets=bool(config["include_external_assets"]),
+                max_pages=int(config["max_pages"]),
+                max_depth=int(config["max_depth"]),
+                max_bytes=int(config["max_bytes"]),
+                max_duration_seconds=remaining_seconds,
+                parallel_connections=int(
+                    config.get("parallel_connections", self.settings.fetch_concurrency)
+                ),
+                download_asset_types=tuple(
+                    config.get("download_asset_types", DEFAULT_ASSET_TYPES)
+                ),
+                skip_file_types=tuple(config.get("skip_file_types", ())),
+                use_proxy=bool(config.get("use_proxy", False)),
+            )
+
+            def update(
+                phase: str, message: str, stats: dict[str, int], warnings: list[str]
+            ) -> None:
+                renewed = self.database.update_progress(
+                    job_id,
+                    self.worker_id,
+                    self.settings.lease_seconds,
+                    phase=phase,
+                    message=message,
+                    stats=stats,
+                    warnings=warnings,
+                )
+                if not renewed:
+                    raise CaptureCancelled("Capture lease was reassigned")
+
+            if not self.database.update_progress(
+                job_id,
+                self.worker_id,
+                self.settings.lease_seconds,
+                phase="Validating",
+                message="Checking capture policy and target reachability",
+                stats=_job_stats(job),
+                warnings=[],
+            ):
+                raise CaptureCancelled("Capture lease was reassigned")
+
+            crawler = SiteCrawler(
+                settings=self.settings,
+                options=options,
+                cancelled=capture_cancelled,
+                on_progress=update,
+            )
+            result = crawler.run(work_dir)
+
+            if capture_cancelled():
+                raise CaptureCancelled("Capture cancelled before packaging")
+            if time.monotonic() >= capture_deadline:
+                raise CaptureError("The configured capture time limit was reached before packaging")
+
+            update("Packaging", "Creating a portable ZIP archive", result.stats, result.warnings)
+            archive_size = _create_archive(
+                result.site_dir,
+                artifact_path,
+                archive_root=f"siteharbor-{job_id}",
+                cancelled=capture_cancelled,
+                deadline_exceeded=lambda: time.monotonic() >= capture_deadline,
+                heartbeat=lambda: update(
+                    "Packaging",
+                    "Creating a portable ZIP archive",
+                    result.stats,
+                    result.warnings,
+                ),
+            )
+
+            report = {
+                **result.report,
+                "job_id": job_id,
+                "archive": {
+                    "filename": artifact_path.name,
+                    "size_bytes": archive_size,
+                    "archive_root": f"siteharbor-{job_id}",
+                },
+            }
+            if time.monotonic() >= capture_deadline:
+                raise CaptureError(
+                    "The configured capture time limit was reached before completion"
+                )
+            report_path.write_text(
+                json.dumps(report, indent=2, ensure_ascii=True), encoding="utf-8"
+            )
+            marked_ready = self.database.mark_ready(
+                job_id,
+                self.worker_id,
+                artifact_path=str(artifact_path),
+                report_path=str(report_path),
+                archive_size=archive_size,
+                entry_point=result.entry_point,
+                stats=result.stats,
+                warnings=result.warnings,
+                retention_days=int(config["retention_days"]),
+            )
+            if not marked_ready:
+                artifact_path.unlink(missing_ok=True)
+                report_path.unlink(missing_ok=True)
+                if self.database.is_cancel_requested(job_id):
+                    self.database.mark_cancelled(
+                        job_id,
+                        self.worker_id,
+                        "Capture cancelled before the archive became available",
+                    )
+        except CaptureCancelled as error:
+            artifact_path.unlink(missing_ok=True)
+            report_path.unlink(missing_ok=True)
+            if self.database.is_cancel_requested(job_id):
+                self.database.mark_cancelled(job_id, self.worker_id, str(error))
+        except UrlPolicyError as error:
+            artifact_path.unlink(missing_ok=True)
+            self.database.mark_failed(job_id, self.worker_id, "target_blocked", str(error))
+        except CaptureError as error:
+            artifact_path.unlink(missing_ok=True)
+            self.database.mark_failed(job_id, self.worker_id, "capture_failed", str(error))
+        except Exception as error:
+            artifact_path.unlink(missing_ok=True)
+            self.database.mark_failed(
+                job_id,
+                self.worker_id,
+                "internal_error",
+                f"Capture failed: {error}",
+            )
+        finally:
+            heartbeat_stop.set()
+            if heartbeat_thread:
+                heartbeat_thread.join(timeout=1)
+            try:
+                _remove_tree_if_present(work_dir, self.settings.work_dir)
+            except OSError:
+                pass
+
+
+def _job_stats(job: dict[str, Any]) -> dict[str, int]:
+    return {
+        "pages_found": int(job.get("pages_found", 0)),
+        "pages_fetched": int(job.get("pages_fetched", 0)),
+        "assets_found": int(job.get("assets_found", 0)),
+        "assets_fetched": int(job.get("assets_fetched", 0)),
+        "bytes_downloaded": int(job.get("bytes_downloaded", 0)),
+        "files_written": int(job.get("files_written", 0)),
+    }
+
+
+def _remove_tree_if_present(path: Path, root: Path) -> None:
+    if path.exists():
+        _remove_tree(path, root)

app/config.py

@@ -0,0 +1,102 @@
+from __future__ import annotations
+
+import os
+from dataclasses import dataclass
+from pathlib import Path
+
+
+def _as_bool(name: str, default: bool) -> bool:
+    value = os.getenv(name)
+    if value is None:
+        return default
+    return value.strip().lower() in {"1", "true", "yes", "on"}
+
+
+def _as_int(name: str, default: int, minimum: int = 1) -> int:
+    value = os.getenv(name)
+    if value is None:
+        return default
+    try:
+        return max(minimum, int(value))
+    except ValueError:
+        return default
+
+
+@dataclass(frozen=True, slots=True)
+class Settings:
+    project_root: Path
+    data_dir: Path
+    database_path: Path
+    work_dir: Path
+    artifacts_dir: Path
+    reports_dir: Path
+    host: str
+    port: int
+    worker_enabled: bool
+    worker_concurrency: int
+    fetch_concurrency: int
+    queue_poll_seconds: float
+    lease_seconds: int
+    allow_private_networks: bool
+    allow_nonstandard_ports: bool
+    respect_robots: bool
+    default_retention_days: int
+    max_retention_days: int
+    max_pages_cap: int
+    max_depth_cap: int
+    max_bytes_cap: int
+    max_duration_seconds_cap: int
+    max_response_bytes: int
+    max_files: int
+    max_external_hosts: int
+    max_redirects: int
+    user_agent: str
+    proxy_url: str | None
+
+    @classmethod
+    def from_environment(cls) -> Settings:
+        project_root = Path(__file__).resolve().parents[1]
+        data_dir = Path(os.getenv("SITEHARBOR_DATA_DIR", project_root / "data")).expanduser()
+        if not data_dir.is_absolute():
+            data_dir = project_root / data_dir
+
+        return cls(
+            project_root=project_root,
+            data_dir=data_dir,
+            database_path=data_dir / "siteharbor.sqlite3",
+            work_dir=data_dir / "work",
+            artifacts_dir=data_dir / "artifacts",
+            reports_dir=data_dir / "reports",
+            host=os.getenv("SITEHARBOR_HOST", "127.0.0.1"),
+            port=_as_int("SITEHARBOR_PORT", 8787),
+            worker_enabled=_as_bool("SITEHARBOR_WORKER_ENABLED", True),
+            worker_concurrency=_as_int("SITEHARBOR_WORKER_CONCURRENCY", 1),
+            fetch_concurrency=_as_int("SITEHARBOR_FETCH_CONCURRENCY", 6),
+            queue_poll_seconds=max(0.1, float(os.getenv("SITEHARBOR_QUEUE_POLL_SECONDS", "0.75"))),
+            lease_seconds=_as_int("SITEHARBOR_LEASE_SECONDS", 120, minimum=15),
+            allow_private_networks=_as_bool("SITEHARBOR_ALLOW_PRIVATE_NETWORKS", False),
+            allow_nonstandard_ports=_as_bool("SITEHARBOR_ALLOW_NONSTANDARD_PORTS", False),
+            respect_robots=_as_bool("SITEHARBOR_RESPECT_ROBOTS", True),
+            default_retention_days=_as_int("SITEHARBOR_DEFAULT_RETENTION_DAYS", 7),
+            max_retention_days=_as_int("SITEHARBOR_MAX_RETENTION_DAYS", 30),
+            max_pages_cap=_as_int("SITEHARBOR_MAX_PAGES", 1_000),
+            max_depth_cap=_as_int("SITEHARBOR_MAX_DEPTH", 12),
+            max_bytes_cap=_as_int("SITEHARBOR_MAX_BYTES", 1024 * 1024 * 1024),
+            max_duration_seconds_cap=_as_int("SITEHARBOR_MAX_DURATION_SECONDS", 3_600),
+            max_response_bytes=_as_int("SITEHARBOR_MAX_RESPONSE_BYTES", 32 * 1024 * 1024),
+            max_files=_as_int("SITEHARBOR_MAX_FILES", 5_000),
+            max_external_hosts=_as_int("SITEHARBOR_MAX_EXTERNAL_HOSTS", 20),
+            max_redirects=_as_int("SITEHARBOR_MAX_REDIRECTS", 10),
+            user_agent=os.getenv(
+                "SITEHARBOR_USER_AGENT",
+                "SiteHarbor/0.1 (+local standalone capture; respect robots.txt)",
+            ),
+            proxy_url=os.getenv("SITEHARBOR_PROXY_URL", "").strip() or None,
+        )
+
+    def ensure_directories(self) -> None:
+        for directory in (self.data_dir, self.work_dir, self.artifacts_dir, self.reports_dir):
+            directory.mkdir(parents=True, exist_ok=True)
+
+
+settings = Settings.from_environment()

app/db.py

@@ -0,0 +1,684 @@
+from __future__ import annotations
+
+import json
+import sqlite3
+import uuid
+from collections.abc import Iterator
+from contextlib import contextmanager
+from datetime import UTC, datetime, timedelta
+from pathlib import Path
+from typing import Any
+
+
+def utcnow() -> datetime:
+    return datetime.now(UTC)
+
+
+def timestamp(value: datetime | None = None) -> str:
+    return (value or utcnow()).isoformat()
+
+
+class Database:
+    """Small SQLite persistence layer with lease-based job claiming."""
+
+    def __init__(self, path: Path) -> None:
+        self.path = path
+
+    @contextmanager
+    def connection(self) -> Iterator[sqlite3.Connection]:
+        connection = sqlite3.connect(self.path, timeout=15, isolation_level=None)
+        connection.row_factory = sqlite3.Row
+        connection.execute("PRAGMA foreign_keys = ON")
+        connection.execute("PRAGMA busy_timeout = 15000")
+        try:
+            yield connection
+        finally:
+            connection.close()
+
+    def initialize(self) -> None:
+        with self.connection() as connection:
+            connection.execute("PRAGMA journal_mode = WAL")
+            connection.execute("PRAGMA synchronous = NORMAL")
+            connection.executescript(
+                """
+                CREATE TABLE IF NOT EXISTS jobs (
+                    id TEXT PRIMARY KEY,
+                    source_url TEXT NOT NULL,
+                    normalized_url TEXT NOT NULL,
+                    owner_token TEXT NOT NULL DEFAULT '',
+                    config_json TEXT NOT NULL,
+                    state TEXT NOT NULL,
+                    phase TEXT NOT NULL,
+                    message TEXT NOT NULL,
+                    error_code TEXT,
+                    error_message TEXT,
+                    warnings_json TEXT NOT NULL DEFAULT '[]',
+                    pages_found INTEGER NOT NULL DEFAULT 0,
+                    pages_fetched INTEGER NOT NULL DEFAULT 0,
+                    assets_found INTEGER NOT NULL DEFAULT 0,
+                    assets_fetched INTEGER NOT NULL DEFAULT 0,
+                    bytes_downloaded INTEGER NOT NULL DEFAULT 0,
+                    files_written INTEGER NOT NULL DEFAULT 0,
+                    artifact_path TEXT,
+                    report_path TEXT,
+                    archive_size INTEGER,
+                    entry_point TEXT,
+                    cancel_requested INTEGER NOT NULL DEFAULT 0,
+                    worker_id TEXT,
+                    lease_expires_at TEXT,
+                    attempt_count INTEGER NOT NULL DEFAULT 0,
+                    created_at TEXT NOT NULL,
+                    updated_at TEXT NOT NULL,
+                    started_at TEXT,
+                    completed_at TEXT,
+                    expires_at TEXT
+                );
+
+                CREATE INDEX IF NOT EXISTS jobs_state_created_idx
+                    ON jobs(state, created_at);
+
+                CREATE INDEX IF NOT EXISTS jobs_lease_idx
+                    ON jobs(lease_expires_at);
+
+                CREATE TABLE IF NOT EXISTS job_events (
+                    id INTEGER PRIMARY KEY AUTOINCREMENT,
+                    job_id TEXT NOT NULL REFERENCES jobs(id) ON DELETE CASCADE,
+                    kind TEXT NOT NULL,
+                    payload_json TEXT NOT NULL,
+                    created_at TEXT NOT NULL
+                );
+
+                CREATE INDEX IF NOT EXISTS job_events_job_id_idx
+                    ON job_events(job_id, id);
+                """
+            )
+            columns = {
+                str(row["name"])
+                for row in connection.execute("PRAGMA table_info(jobs)").fetchall()
+            }
+            if "owner_token" not in columns:
+                connection.execute(
+                    "ALTER TABLE jobs ADD COLUMN owner_token TEXT NOT NULL DEFAULT ''"
+                )
+            connection.execute(
+                "CREATE INDEX IF NOT EXISTS jobs_owner_created_idx "
+                "ON jobs(owner_token, created_at DESC)"
+            )
+
+    def create_job(
+        self,
+        source_url: str,
+        normalized_url: str,
+        config: dict[str, Any],
+        owner_token: str = "",
+    ) -> dict[str, Any]:
+        job_id = f"cap_{uuid.uuid4().hex}"
+        now = timestamp()
+        payload = json.dumps(config, separators=(",", ":"))
+        with self.connection() as connection:
+            connection.execute(
+                """
+                INSERT INTO jobs (
+                    id, source_url, normalized_url, owner_token, config_json, state, phase, message,
+                    created_at, updated_at, expires_at
+                ) VALUES (
+                    ?, ?, ?, ?, ?, 'queued', 'Queued', 'Waiting for a local worker', ?, ?, NULL
+                )
+                """,
+                (
+                    job_id,
+                    source_url,
+                    normalized_url,
+                    owner_token,
+                    payload,
+                    now,
+                    now,
+                ),
+            )
+            self._append_event(
+                connection,
+                job_id,
+                "state",
+                {"state": "queued", "phase": "Queued", "message": "Waiting for a local worker"},
+            )
+        return self.get_job(job_id) or {}
+
+    def get_job(self, job_id: str) -> dict[str, Any] | None:
+        with self.connection() as connection:
+            row = connection.execute("SELECT * FROM jobs WHERE id = ?", (job_id,)).fetchone()
+        return self._row_to_job(row) if row else None
+
+    def get_owned_job(self, job_id: str, owner_token: str) -> dict[str, Any] | None:
+        with self.connection() as connection:
+            row = connection.execute(
+                "SELECT * FROM jobs WHERE id = ? AND owner_token = ?", (job_id, owner_token)
+            ).fetchone()
+        return self._row_to_job(row) if row else None
+
+    def list_jobs(self, limit: int = 30) -> list[dict[str, Any]]:
+        with self.connection() as connection:
+            rows = connection.execute(
+                "SELECT * FROM jobs ORDER BY created_at DESC LIMIT ?", (limit,)
+            ).fetchall()
+        return [self._row_to_job(row) for row in rows]
+
+    def list_owned_jobs(self, owner_token: str, limit: int = 30) -> list[dict[str, Any]]:
+        with self.connection() as connection:
+            rows = connection.execute(
+                """
+                SELECT * FROM jobs
+                WHERE owner_token = ?
+                ORDER BY created_at DESC
+                LIMIT ?
+                """,
+                (owner_token, limit),
+            ).fetchall()
+        return [self._row_to_job(row) for row in rows]
+
+    def public_stats(self) -> dict[str, int]:
+        with self.connection() as connection:
+            row = connection.execute(
+                """
+                SELECT
+                    COUNT(CASE WHEN state = 'ready' THEN 1 END) AS websites_cloned,
+                    COALESCE(SUM(bytes_downloaded), 0) AS bytes_scraped,
+                    COUNT(CASE WHEN state IN ('queued', 'running') THEN 1 END) AS active_crawls,
+                    COALESCE(SUM(files_written), 0) AS files_archived
+                FROM jobs
+                """
+            ).fetchone()
+        return {
+            "websites_cloned": int(row["websites_cloned"] or 0),
+            "bytes_scraped": int(row["bytes_scraped"] or 0),
+            "active_crawls": int(row["active_crawls"] or 0),
+            "files_archived": int(row["files_archived"] or 0),
+        }
+
+    def get_events(self, job_id: str, after_id: int = 0) -> list[dict[str, Any]]:
+        with self.connection() as connection:
+            rows = connection.execute(
+                """
+                SELECT id, kind, payload_json, created_at
+                FROM job_events
+                WHERE job_id = ? AND id > ?
+                ORDER BY id ASC
+                LIMIT 100
+                """,
+                (job_id, after_id),
+            ).fetchall()
+        return [
+            {
+                "id": row["id"],
+                "kind": row["kind"],
+                "payload": json.loads(row["payload_json"]),
+                "created_at": row["created_at"],
+            }
+            for row in rows
+        ]
+
+    def claim_next(self, worker_id: str, lease_seconds: int) -> dict[str, Any] | None:
+        now = utcnow()
+        now_text = timestamp(now)
+        lease = timestamp(now + timedelta(seconds=lease_seconds))
+        with self.connection() as connection:
+            try:
+                connection.execute("BEGIN IMMEDIATE")
+                cancelled_rows = connection.execute(
+                    """
+                    SELECT id FROM jobs
+                    WHERE cancel_requested = 1
+                      AND (
+                        state = 'queued'
+                        OR (
+                            state = 'running'
+                            AND lease_expires_at IS NOT NULL
+                            AND lease_expires_at < ?
+                        )
+                      )
+                    """,
+                    (now_text,),
+                ).fetchall()
+                connection.execute(
+                    """
+                    UPDATE jobs
+                    SET state = 'cancelled', phase = 'Cancelled',
+                        message = 'Cancelled before starting',
+                        completed_at = ?, updated_at = ?
+                    WHERE cancel_requested = 1
+                      AND (
+                        state = 'queued'
+                        OR (
+                            state = 'running'
+                            AND lease_expires_at IS NOT NULL
+                            AND lease_expires_at < ?
+                        )
+                      )
+                    """,
+                    (now_text, now_text, now_text),
+                )
+                for cancelled_row in cancelled_rows:
+                    self._append_event(
+                        connection,
+                        cancelled_row["id"],
+                        "state",
+                        {
+                            "state": "cancelled",
+                            "phase": "Cancelled",
+                            "message": "Cancelled before starting",
+                        },
+                    )
+                row = connection.execute(
+                    """
+                    SELECT * FROM jobs
+                    WHERE cancel_requested = 0
+                      AND (
+                        state = 'queued'
+                        OR (
+                            state = 'running'
+                            AND lease_expires_at IS NOT NULL
+                            AND lease_expires_at < ?
+                        )
+                      )
+                    ORDER BY created_at ASC
+                    LIMIT 1
+                    """,
+                    (now_text,),
+                ).fetchone()
+                if not row:
+                    connection.execute("COMMIT")
+                    return None
+
+                job_id = row["id"]
+                message = (
+                    "Recovering a leased capture"
+                    if row["state"] == "running"
+                    else "Preparing capture"
+                )
+                connection.execute(
+                    """
+                    UPDATE jobs
+                    SET state = 'running', phase = 'Preparing', message = ?, worker_id = ?,
+                        lease_expires_at = ?, attempt_count = attempt_count + 1,
+                        started_at = COALESCE(started_at, ?), updated_at = ?
+                    WHERE id = ?
+                    """,
+                    (message, worker_id, lease, now_text, now_text, job_id),
+                )
+                self._append_event(
+                    connection,
+                    job_id,
+                    "state",
+                    {"state": "running", "phase": "Preparing", "message": message},
+                )
+                claimed = connection.execute(
+                    "SELECT * FROM jobs WHERE id = ?", (job_id,)
+                ).fetchone()
+                connection.execute("COMMIT")
+                return self._row_to_job(claimed)
+            except Exception:
+                connection.execute("ROLLBACK")
+                raise
+
+    def update_progress(
+        self,
+        job_id: str,
+        worker_id: str,
+        lease_seconds: int,
+        *,
+        phase: str,
+        message: str,
+        stats: dict[str, int],
+        warnings: list[str] | None = None,
+    ) -> bool:
+        now = utcnow()
+        values = {
+            "phase": phase,
+            "message": message,
+            "pages_found": stats.get("pages_found", 0),
+            "pages_fetched": stats.get("pages_fetched", 0),
+            "assets_found": stats.get("assets_found", 0),
+            "assets_fetched": stats.get("assets_fetched", 0),
+            "bytes_downloaded": stats.get("bytes_downloaded", 0),
+            "files_written": stats.get("files_written", 0),
+            "warnings_json": json.dumps(warnings or []),
+            "worker_id": worker_id,
+            "lease_expires_at": timestamp(now + timedelta(seconds=lease_seconds)),
+            "updated_at": timestamp(now),
+            "current_time": timestamp(now),
+        }
+        with self.connection() as connection:
+            cursor = connection.execute(
+                """
+                UPDATE jobs SET
+                    phase = :phase, message = :message,
+                    pages_found = :pages_found, pages_fetched = :pages_fetched,
+                    assets_found = :assets_found, assets_fetched = :assets_fetched,
+                    bytes_downloaded = :bytes_downloaded, files_written = :files_written,
+                    warnings_json = :warnings_json, worker_id = :worker_id,
+                    lease_expires_at = :lease_expires_at, updated_at = :updated_at
+                WHERE id = :job_id
+                  AND state = 'running'
+                  AND worker_id = :worker_id
+                  AND lease_expires_at >= :current_time
+                """,
+                {**values, "job_id": job_id},
+            )
+            if cursor.rowcount:
+                self._append_event(
+                    connection,
+                    job_id,
+                    "progress",
+                    {
+                        "phase": phase,
+                        "message": message,
+                        "stats": stats,
+                        "warnings": warnings or [],
+                    },
+                )
+                return True
+        return False
+
+    def renew_lease(self, job_id: str, worker_id: str, lease_seconds: int) -> bool:
+        now = utcnow()
+        now_text = timestamp(now)
+        with self.connection() as connection:
+            cursor = connection.execute(
+                """
+                UPDATE jobs
+                SET lease_expires_at = ?, updated_at = ?
+                WHERE id = ?
+                  AND state = 'running'
+                  AND worker_id = ?
+                  AND lease_expires_at >= ?
+                """,
+                (
+                    timestamp(now + timedelta(seconds=lease_seconds)),
+                    now_text,
+                    job_id,
+                    worker_id,
+                    now_text,
+                ),
+            )
+        return bool(cursor.rowcount)
+
+    def is_cancel_requested(self, job_id: str) -> bool:
+        with self.connection() as connection:
+            row = connection.execute(
+                "SELECT cancel_requested FROM jobs WHERE id = ?", (job_id,)
+            ).fetchone()
+        return bool(row and row["cancel_requested"])
+
+    def owns_lease(self, job_id: str, worker_id: str) -> bool:
+        now = timestamp()
+        with self.connection() as connection:
+            row = connection.execute(
+                """
+                SELECT state, worker_id, lease_expires_at
+                FROM jobs
+                WHERE id = ?
+                """,
+                (job_id,),
+            ).fetchone()
+        return bool(
+            row
+            and row["state"] == "running"
+            and row["worker_id"] == worker_id
+            and row["lease_expires_at"]
+            and row["lease_expires_at"] >= now
+        )
+
+    def request_cancel(self, job_id: str) -> dict[str, Any] | None:
+        now = timestamp()
+        with self.connection() as connection:
+            try:
+                connection.execute("BEGIN IMMEDIATE")
+                row = connection.execute(
+                    "SELECT state FROM jobs WHERE id = ?", (job_id,)
+                ).fetchone()
+                if not row:
+                    connection.execute("COMMIT")
+                    return None
+                if row["state"] in {"ready", "failed", "cancelled", "expired"}:
+                    connection.execute("COMMIT")
+                    return self.get_job(job_id)
+
+                if row["state"] == "queued":
+                    cursor = connection.execute(
+                        """
+                        UPDATE jobs
+                        SET cancel_requested = 1, state = 'cancelled', phase = 'Cancelled',
+                            message = 'Cancelled before starting', completed_at = ?, updated_at = ?
+                        WHERE id = ? AND state = 'queued'
+                        """,
+                        (now, now, job_id),
+                    )
+                    payload = {
+                        "state": "cancelled",
+                        "phase": "Cancelled",
+                        "message": "Cancelled before starting",
+                    }
+                else:
+                    cursor = connection.execute(
+                        """
+                        UPDATE jobs
+                        SET cancel_requested = 1, message = 'Cancellation requested', updated_at = ?
+                        WHERE id = ? AND state = 'running' AND cancel_requested = 0
+                        """,
+                        (now, job_id),
+                    )
+                    payload = {
+                        "state": "running",
+                        "phase": "Cancelling",
+                        "message": "Cancellation requested",
+                    }
+                if cursor.rowcount:
+                    self._append_event(connection, job_id, "state", payload)
+                connection.execute("COMMIT")
+            except Exception:
+                connection.execute("ROLLBACK")
+                raise
+        return self.get_job(job_id)
+
+    def mark_ready(
+        self,
+        job_id: str,
+        worker_id: str,
+        *,
+        artifact_path: str,
+        report_path: str,
+        archive_size: int,
+        entry_point: str,
+        stats: dict[str, int],
+        warnings: list[str],
+        retention_days: int,
+    ) -> bool:
+        now_datetime = utcnow()
+        now = timestamp(now_datetime)
+        with self.connection() as connection:
+            cursor = connection.execute(
+                """
+                UPDATE jobs
+                SET state = 'ready', phase = 'Ready', message = 'Archive is ready to download',
+                    artifact_path = ?, report_path = ?, archive_size = ?, entry_point = ?,
+                    pages_found = ?, pages_fetched = ?, assets_found = ?, assets_fetched = ?,
+                    bytes_downloaded = ?, files_written = ?, warnings_json = ?,
+                    worker_id = NULL, lease_expires_at = NULL, expires_at = ?,
+                    completed_at = ?, updated_at = ?
+                WHERE id = ?
+                  AND state = 'running'
+                  AND worker_id = ?
+                  AND cancel_requested = 0
+                  AND lease_expires_at >= ?
+                """,
+                (
+                    artifact_path,
+                    report_path,
+                    archive_size,
+                    entry_point,
+                    stats.get("pages_found", 0),
+                    stats.get("pages_fetched", 0),
+                    stats.get("assets_found", 0),
+                    stats.get("assets_fetched", 0),
+                    stats.get("bytes_downloaded", 0),
+                    stats.get("files_written", 0),
+                    json.dumps(warnings),
+                    timestamp(now_datetime + timedelta(days=retention_days)),
+                    now,
+                    now,
+                    job_id,
+                    worker_id,
+                    now,
+                ),
+            )
+            if cursor.rowcount:
+                self._append_event(
+                    connection,
+                    job_id,
+                    "state",
+                    {"state": "ready", "phase": "Ready", "message": "Archive is ready to download"},
+                )
+                return True
+        return False
+
+    def mark_cancelled(
+        self, job_id: str, worker_id: str, message: str = "Capture cancelled"
+    ) -> bool:
+        now = timestamp()
+        with self.connection() as connection:
+            cursor = connection.execute(
+                """
+                UPDATE jobs
+                SET state = 'cancelled', phase = 'Cancelled', message = ?, worker_id = NULL,
+                    lease_expires_at = NULL, completed_at = ?, updated_at = ?
+                WHERE id = ?
+                  AND state = 'running'
+                  AND worker_id = ?
+                  AND cancel_requested = 1
+                  AND lease_expires_at >= ?
+                """,
+                (message, now, now, job_id, worker_id, now),
+            )
+            if cursor.rowcount:
+                self._append_event(
+                    connection,
+                    job_id,
+                    "state",
+                    {"state": "cancelled", "phase": "Cancelled", "message": message},
+                )
+                return True
+        return False
+
+    def mark_failed(self, job_id: str, worker_id: str, code: str, message: str) -> bool:
+        now = timestamp()
+        with self.connection() as connection:
+            cursor = connection.execute(
+                """
+                UPDATE jobs
+                SET state = 'failed', phase = 'Failed', message = ?, error_code = ?,
+                    error_message = ?, worker_id = NULL, lease_expires_at = NULL,
+                    completed_at = ?, updated_at = ?
+                WHERE id = ?
+                  AND state = 'running'
+                  AND worker_id = ?
+                  AND cancel_requested = 0
+                  AND lease_expires_at >= ?
+                """,
+                (message, code, message, now, now, job_id, worker_id, now),
+            )
+            if cursor.rowcount:
+                self._append_event(
+                    connection,
+                    job_id,
+                    "state",
+                    {"state": "failed", "phase": "Failed", "message": message, "code": code},
+                )
+                return True
+        return False
+
+    def mark_expired(self, job_id: str) -> None:
+        now = timestamp()
+        with self.connection() as connection:
+            connection.execute(
+                """
+                UPDATE jobs
+                SET state = 'expired', phase = 'Expired',
+                    message = 'Artifact retention period ended',
+                    artifact_path = NULL, report_path = NULL, updated_at = ?
+                WHERE id = ?
+                """,
+                (now, job_id),
+            )
+            self._append_event(
+                connection,
+                job_id,
+                "state",
+                {
+                    "state": "expired",
+                    "phase": "Expired",
+                    "message": "Artifact retention period ended",
+                },
+            )
+
+    def mark_deleted(self, job_id: str) -> None:
+        now = timestamp()
+        with self.connection() as connection:
+            cursor = connection.execute(
+                """
+                UPDATE jobs
+                SET state = 'expired', phase = 'Deleted', message = 'Artifact deleted by user',
+                    artifact_path = NULL, report_path = NULL, updated_at = ?
+                WHERE id = ? AND state = 'ready'
+                """,
+                (now, job_id),
+            )
+            if cursor.rowcount:
+                self._append_event(
+                    connection,
+                    job_id,
+                    "state",
+                    {
+                        "state": "expired",
+                        "phase": "Deleted",
+                        "message": "Artifact deleted by user",
+                    },
+                )
+
+    def expired_ready_jobs(self) -> list[dict[str, Any]]:
+        now = timestamp()
+        with self.connection() as connection:
+            rows = connection.execute(
+                "SELECT * FROM jobs WHERE state = 'ready' AND expires_at <= ?", (now,)
+            ).fetchall()
+        return [self._row_to_job(row) for row in rows]
+
+    def referenced_data_paths(self) -> set[str]:
+        with self.connection() as connection:
+            rows = connection.execute(
+                """
+                SELECT artifact_path, report_path
+                FROM jobs
+                WHERE state = 'ready'
+                """
+            ).fetchall()
+        return {
+            path
+            for row in rows
+            for path in (row["artifact_path"], row["report_path"])
+            if isinstance(path, str) and path
+        }
+
+    def _append_event(
+        self, connection: sqlite3.Connection, job_id: str, kind: str, payload: dict[str, Any]
+    ) -> None:
+        connection.execute(
+            "INSERT INTO job_events (job_id, kind, payload_json, created_at) VALUES (?, ?, ?, ?)",
+            (job_id, kind, json.dumps(payload, separators=(",", ":")), timestamp()),
+        )
+
+    @staticmethod
+    def _row_to_job(row: sqlite3.Row) -> dict[str, Any]:
+        data = dict(row)
+        data["config"] = json.loads(data.pop("config_json"))
+        data["warnings"] = json.loads(data.pop("warnings_json"))
+        data["cancel_requested"] = bool(data["cancel_requested"])
+        return data

app/main.py

@@ -0,0 +1,395 @@
+from __future__ import annotations
+
+import asyncio
+import json
+import re
+from collections.abc import AsyncIterator
+from contextlib import asynccontextmanager
+from pathlib import Path
+from typing import Any
+
+import uvicorn
+from fastapi import FastAPI, HTTPException, Request, Response, status
+from fastapi.responses import FileResponse, HTMLResponse, JSONResponse, StreamingResponse
+from fastapi.staticfiles import StaticFiles
+from fastapi.templating import Jinja2Templates
+
+from app.config import settings
+from app.db import Database
+from app.models import SUPPORTED_ASSET_TYPES, CaptureRequest
+from app.scheduler import JobRunner
+from app.url_policy import NetworkRules, UrlPolicyError, normalize_url
+
+settings.ensure_directories()
+database = Database(settings.database_path)
+runner = JobRunner(database, settings)
+templates = Jinja2Templates(directory=str(settings.project_root / "app" / "templates"))
+OWNER_TOKEN_HEADER = "X-SiteHarbor-Session"
+OWNER_TOKEN_RE = re.compile(r"[A-Za-z0-9_-]{24,200}")
+
+
+@asynccontextmanager
+async def lifespan(_: FastAPI) -> AsyncIterator[None]:
+    database.initialize()
+    runner.start()
+    try:
+        yield
+    finally:
+        runner.stop()
+
+
+app = FastAPI(
+    title="SiteHarbor",
+    version="0.1.0",
+    description="Standalone, bounded website capture for offline archives.",
+    lifespan=lifespan,
+)
+app.mount(
+    "/static", StaticFiles(directory=str(settings.project_root / "app" / "static")), name="static"
+)
+
+
+@app.middleware("http")
+async def secure_headers(_: Request, call_next: Any) -> Response:
+    response = await call_next(_)
+    response.headers.setdefault("X-Content-Type-Options", "nosniff")
+    response.headers.setdefault("Referrer-Policy", "same-origin")
+    response.headers.setdefault("X-Frame-Options", "DENY")
+    response.headers.setdefault(
+        "Content-Security-Policy",
+        "default-src 'self'; base-uri 'none'; object-src 'none'; frame-ancestors 'none'; "
+        "form-action 'self'; connect-src 'self'; img-src 'self' data:; style-src 'self'; "
+        "script-src 'self'",
+    )
+    return response
+
+
+@app.get("/", response_class=HTMLResponse)
+async def index(request: Request) -> Response:
+    return templates.TemplateResponse(
+        request=request,
+        name="index.html",
+        context={"app_name": "SiteHarbor"},
+    )
+
+
+@app.get("/api/health")
+async def health() -> dict[str, Any]:
+    return {
+        "status": "ok",
+        "worker_enabled": settings.worker_enabled,
+        "worker_concurrency": settings.worker_concurrency,
+        "storage": "local-sqlite",
+    }
+
+
+@app.get("/api/capabilities")
+async def capabilities() -> dict[str, Any]:
+    return {
+        "defaults": {
+            "max_pages": min(150, settings.max_pages_cap),
+            "max_depth": min(5, settings.max_depth_cap),
+            "max_bytes_mb": min(100, max(1, settings.max_bytes_cap // (1024 * 1024))),
+            "max_duration_seconds": min(300, settings.max_duration_seconds_cap),
+            "retention_days": min(settings.default_retention_days, settings.max_retention_days),
+            "include_external_assets": True,
+            "parallel_connections": min(6, settings.fetch_concurrency),
+            "download_asset_types": list(SUPPORTED_ASSET_TYPES),
+            "skip_file_types": [],
+            "use_proxy": False,
+        },
+        "limits": {
+            "max_pages": settings.max_pages_cap,
+            "max_depth": settings.max_depth_cap,
+            "max_bytes_mb": settings.max_bytes_cap // (1024 * 1024),
+            "max_duration_seconds": settings.max_duration_seconds_cap,
+            "max_retention_days": settings.max_retention_days,
+            "max_parallel_connections": settings.fetch_concurrency,
+        },
+        "proxy_available": bool(settings.proxy_url),
+        "local_only_warning": not settings.allow_private_networks,
+    }
+
+
+@app.get("/api/stats")
+async def public_stats() -> dict[str, int]:
+    return database.public_stats()
+
+
+@app.post("/api/captures", status_code=status.HTTP_202_ACCEPTED)
+async def create_capture(payload: CaptureRequest, request: Request) -> dict[str, Any]:
+    owner_token = _owner_token(request)
+    config = _validated_config(payload)
+    rules = NetworkRules(
+        allow_private_networks=settings.allow_private_networks,
+        allow_nonstandard_ports=settings.allow_nonstandard_ports,
+    )
+    try:
+        normalized_url = normalize_url(payload.url, rules)
+    except UrlPolicyError as error:
+        raise HTTPException(status_code=422, detail=str(error)) from error
+
+    job = database.create_job(payload.url.strip(), normalized_url, config, owner_token)
+    return _public_job(job)
+
+
+@app.get("/api/captures")
+async def list_captures(request: Request) -> dict[str, list[dict[str, Any]]]:
+    owner_token = _owner_token(request)
+    return {"items": [_public_job(job) for job in database.list_owned_jobs(owner_token)]}
+
+
+@app.get("/api/captures/{job_id}")
+async def get_capture(job_id: str, request: Request) -> dict[str, Any]:
+    return _require_owned_job(job_id, _owner_token(request))
+
+
+@app.post("/api/captures/{job_id}/cancel")
+async def cancel_capture(job_id: str, request: Request) -> dict[str, Any]:
+    _require_owned_job(job_id, _owner_token(request))
+    job = database.request_cancel(job_id)
+    if not job:
+        raise HTTPException(status_code=404, detail="Capture not found.")
+    return _public_job(job)
+
+
+@app.delete("/api/captures/{job_id}")
+async def delete_capture(job_id: str, request: Request) -> dict[str, Any]:
+    job = _owned_job(job_id, _owner_token(request))
+    if job["state"] in {"queued", "running"}:
+        cancelled = database.request_cancel(job_id)
+        return _public_job(cancelled or job)
+    if job["state"] == "ready":
+        artifact_path = _safe_data_file(str(job.get("artifact_path") or ""), settings.artifacts_dir)
+        report_path = _safe_data_file(str(job.get("report_path") or ""), settings.reports_dir)
+        if artifact_path:
+            artifact_path.unlink(missing_ok=True)
+        if report_path:
+            report_path.unlink(missing_ok=True)
+        database.mark_deleted(job_id)
+        updated = database.get_owned_job(job_id, _owner_token(request))
+        return _public_job(updated or job)
+    return _public_job(job)
+
+
+@app.get("/api/captures/{job_id}/report")
+async def capture_report(job_id: str, request: Request) -> JSONResponse:
+    job = _owned_job(job_id, _owner_token(request))
+    if job["state"] != "ready" or not job.get("report_path"):
+        raise HTTPException(status_code=409, detail="A capture report is not available yet.")
+    report_path = _safe_data_file(str(job["report_path"]), settings.reports_dir)
+    if not report_path or not report_path.is_file():
+        raise HTTPException(status_code=410, detail="The capture report has expired.")
+    return JSONResponse(json.loads(report_path.read_text(encoding="utf-8")))
+
+
+@app.get("/api/captures/{job_id}/download")
+async def download_capture(job_id: str, request: Request) -> FileResponse:
+    job = _owned_job(job_id, _owner_token(request))
+    if job["state"] != "ready" or not job.get("artifact_path"):
+        raise HTTPException(status_code=409, detail="The archive is not available yet.")
+    archive_path = _safe_data_file(str(job["artifact_path"]), settings.artifacts_dir)
+    if not archive_path or not archive_path.is_file():
+        raise HTTPException(status_code=410, detail="The archive has expired.")
+    return FileResponse(
+        archive_path,
+        media_type="application/zip",
+        filename=f"siteharbor-{job_id}.zip",
+        headers={"Cache-Control": "no-store", "X-Content-Type-Options": "nosniff"},
+    )
+
+
+@app.get("/api/captures/{job_id}/events")
+async def capture_events(request: Request, job_id: str) -> StreamingResponse:
+    _owned_job(job_id, _owner_token(request, allow_query=True))
+    after_header = request.headers.get("last-event-id", "0")
+    try:
+        after_id = max(0, int(after_header))
+    except ValueError:
+        after_id = 0
+    return StreamingResponse(
+        _event_stream(request, job_id, after_id),
+        media_type="text/event-stream",
+        headers={"Cache-Control": "no-cache", "X-Accel-Buffering": "no"},
+    )
+
+
+async def _event_stream(request: Request, job_id: str, after_id: int) -> AsyncIterator[str]:
+    terminal_states = {"ready", "failed", "cancelled", "expired"}
+    last_id = after_id
+    while True:
+        if await request.is_disconnected():
+            return
+        events = database.get_events(job_id, last_id)
+        for event in events:
+            last_id = int(event["id"])
+            payload = json.dumps(event["payload"], separators=(",", ":"))
+            yield f"id: {last_id}\nevent: {event['kind']}\ndata: {payload}\n\n"
+
+        job = database.get_job(job_id)
+        if not job or job["state"] in terminal_states:
+            terminal = json.dumps({"state": job["state"] if job else "missing"})
+            yield f"event: terminal\ndata: {terminal}\n\n"
+            return
+        yield ": keepalive\n\n"
+        await asyncio.sleep(0.8)
+
+
+def _validated_config(payload: CaptureRequest) -> dict[str, Any]:
+    max_bytes = payload.max_bytes_mb * 1024 * 1024
+    limits = {
+        "max_pages": (payload.max_pages, settings.max_pages_cap, "page"),
+        "max_depth": (payload.max_depth, settings.max_depth_cap, "depth"),
+        "max_bytes": (max_bytes, settings.max_bytes_cap, "download size"),
+        "max_duration_seconds": (
+            payload.max_duration_seconds,
+            settings.max_duration_seconds_cap,
+            "capture duration",
+        ),
+        "retention_days": (payload.retention_days, settings.max_retention_days, "retention"),
+    }
+    for _, (value, maximum, label) in limits.items():
+        if value > maximum:
+            raise HTTPException(
+                status_code=422, detail=f"Requested {label} exceeds this instance's limit."
+            )
+    if payload.parallel_connections > settings.fetch_concurrency:
+        raise HTTPException(
+            status_code=422,
+            detail="Requested parallel connections exceed this instance's limit.",
+        )
+    if payload.use_proxy and not settings.proxy_url:
+        raise HTTPException(
+            status_code=422,
+            detail="This service does not have an approved proxy route configured.",
+        )
+    skip_file_types = _normalized_file_types(payload.skip_file_types)
+    return {
+        "include_external_assets": payload.include_external_assets,
+        "parallel_connections": payload.parallel_connections,
+        "download_asset_types": list(dict.fromkeys(payload.download_asset_types)),
+        "skip_file_types": skip_file_types,
+        "use_proxy": payload.use_proxy,
+        "max_pages": payload.max_pages,
+        "max_depth": payload.max_depth,
+        "max_bytes": max_bytes,
+        "max_duration_seconds": payload.max_duration_seconds,
+        "retention_days": payload.retention_days,
+    }
+
+
+def _normalized_file_types(values: list[str]) -> list[str]:
+    normalized: list[str] = []
+    for value in values:
+        for item in value.split(","):
+            extension = item.strip().lower().lstrip(".")
+            if not extension:
+                continue
+            if not re.fullmatch(r"[a-z0-9]{1,16}", extension):
+                raise HTTPException(
+                    status_code=422,
+                    detail=(
+                        "File rules must be comma-separated extensions such as pdf, zip, or mp4."
+                    ),
+                )
+            if extension not in normalized:
+                normalized.append(extension)
+    if len(normalized) > 24:
+        raise HTTPException(status_code=422, detail="At most 24 file types can be skipped.")
+    return normalized
+
+
+def _owner_token(request: Request, *, allow_query: bool = False) -> str:
+    value = request.headers.get(OWNER_TOKEN_HEADER)
+    if not value and allow_query:
+        value = request.query_params.get("session")
+    token = (value or "").strip()
+    if not OWNER_TOKEN_RE.fullmatch(token):
+        raise HTTPException(
+            status_code=status.HTTP_401_UNAUTHORIZED,
+            detail="A browser-local session is required to access captures.",
+        )
+    return token
+
+
+def _owned_job(job_id: str, owner_token: str) -> dict[str, Any]:
+    job = database.get_owned_job(job_id, owner_token)
+    if not job:
+        raise HTTPException(status_code=404, detail="Capture not found.")
+    return job
+
+
+def _require_owned_job(job_id: str, owner_token: str) -> dict[str, Any]:
+    return _public_job(_owned_job(job_id, owner_token))
+
+
+def _public_job(job: dict[str, Any]) -> dict[str, Any]:
+    config = job["config"]
+    return {
+        "id": job["id"],
+        "url": job["normalized_url"],
+        "state": job["state"],
+        "phase": job["phase"],
+        "message": job["message"],
+        "error_code": job["error_code"],
+        "error_message": job["error_message"],
+        "warnings": job["warnings"],
+        "created_at": job["created_at"],
+        "updated_at": job["updated_at"],
+        "started_at": job["started_at"],
+        "completed_at": job["completed_at"],
+        "expires_at": job["expires_at"],
+        "cancel_requested": job["cancel_requested"],
+        "entry_point": job["entry_point"],
+        "stats": {
+            "pages_found": job["pages_found"],
+            "pages_fetched": job["pages_fetched"],
+            "assets_found": job["assets_found"],
+            "assets_fetched": job["assets_fetched"],
+            "bytes_downloaded": job["bytes_downloaded"],
+            "files_written": job["files_written"],
+        },
+        "options": {
+            "include_external_assets": config.get("include_external_assets", True),
+            "parallel_connections": config.get(
+                "parallel_connections", settings.fetch_concurrency
+            ),
+            "download_asset_types": config.get(
+                "download_asset_types", list(SUPPORTED_ASSET_TYPES)
+            ),
+            "skip_file_types": config.get("skip_file_types", []),
+            "use_proxy": config.get("use_proxy", False),
+            "max_pages": config["max_pages"],
+            "max_depth": config["max_depth"],
+            "max_bytes": config["max_bytes"],
+            "max_duration_seconds": config["max_duration_seconds"],
+            "retention_days": config["retention_days"],
+        },
+        "archive": {
+            "available": job["state"] == "ready" and bool(job["artifact_path"]),
+            "size_bytes": job["archive_size"],
+            "download_url": f"/api/captures/{job['id']}/download"
+            if job["state"] == "ready"
+            else None,
+            "report_url": f"/api/captures/{job['id']}/report" if job["state"] == "ready" else None,
+        },
+    }
+
+
+def _safe_data_file(path_value: str, root: Path) -> Path | None:
+    path = Path(path_value)
+    try:
+        resolved = path.resolve()
+        if resolved.is_relative_to(root.resolve()):
+            return resolved
+    except OSError:
+        return None
+    return None
+
+
+def run() -> None:
+    uvicorn.run("app.main:app", host=settings.host, port=settings.port, reload=False)
+
+
+if __name__ == "__main__":
+    run()

app/models.py

@@ -0,0 +1,31 @@
+from __future__ import annotations
+
+from typing import Literal
+
+from pydantic import BaseModel, Field
+
+AssetType = Literal["styles", "scripts", "images", "fonts", "media", "other"]
+SUPPORTED_ASSET_TYPES: tuple[AssetType, ...] = (
+    "styles",
+    "scripts",
+    "images",
+    "fonts",
+    "media",
+    "other",
+)
+
+
+class CaptureRequest(BaseModel):
+    url: str = Field(min_length=1, max_length=2_048)
+    include_external_assets: bool = True
+    parallel_connections: int = Field(default=6, ge=1)
+    download_asset_types: list[AssetType] = Field(
+        default_factory=lambda: list(SUPPORTED_ASSET_TYPES), max_length=len(SUPPORTED_ASSET_TYPES)
+    )
+    skip_file_types: list[str] = Field(default_factory=list, max_length=24)
+    use_proxy: bool = False
+    max_pages: int = Field(default=150, ge=1)
+    max_depth: int = Field(default=5, ge=0)
+    max_bytes_mb: int = Field(default=100, ge=1)
+    max_duration_seconds: int = Field(default=300, ge=10)
+    retention_days: int = Field(default=7, ge=1)

app/pinned_transport.py

@@ -0,0 +1,96 @@
+from __future__ import annotations
+
+import threading
+from collections.abc import Iterable
+
+import httpcore
+import httpx
+
+
+class PinnedAddressBook:
+    """Thread-safe, short-lived host-to-validated-IP mapping for direct HTTP connections."""
+
+    def __init__(self) -> None:
+        self._addresses: dict[tuple[str, int], tuple[str, ...]] = {}
+        self._lock = threading.Lock()
+
+    def pin(self, host: str, port: int, addresses: Iterable[str]) -> None:
+        values = tuple(addresses)
+        if not values:
+            raise ValueError("At least one validated address is required.")
+        with self._lock:
+            self._addresses[(host.rstrip(".").lower(), port)] = values
+
+    def addresses_for(self, host: str, port: int) -> tuple[str, ...]:
+        with self._lock:
+            addresses = self._addresses.get((host.rstrip(".").lower(), port))
+        if not addresses:
+            raise httpcore.ConnectError(f"No validated address is available for {host}:{port}")
+        return addresses
+
+
+class PinnedNetworkBackend(httpcore.NetworkBackend):
+    """Dials only validated numeric addresses while httpcore retains the logical hostname."""
+
+    def __init__(self, address_book: PinnedAddressBook) -> None:
+        self._address_book = address_book
+        self._backend = httpcore.SyncBackend()
+
+    def connect_tcp(
+        self,
+        host: str,
+        port: int,
+        timeout: float | None = None,
+        local_address: str | None = None,
+        socket_options: Iterable[httpcore.SOCKET_OPTION] | None = None,
+    ) -> httpcore.NetworkStream:
+        last_error: Exception | None = None
+        for address in self._address_book.addresses_for(host, port):
+            try:
+                return self._backend.connect_tcp(
+                    address,
+                    port,
+                    timeout=timeout,
+                    local_address=local_address,
+                    socket_options=socket_options,
+                )
+            except (httpcore.ConnectError, httpcore.ConnectTimeout) as error:
+                last_error = error
+        if last_error:
+            raise last_error
+        raise httpcore.ConnectError(f"Could not connect to a validated address for {host}:{port}")
+
+    def connect_unix_socket(
+        self,
+        path: str,
+        timeout: float | None = None,
+        socket_options: Iterable[httpcore.SOCKET_OPTION] | None = None,
+    ) -> httpcore.NetworkStream:
+        return self._backend.connect_unix_socket(
+            path, timeout=timeout, socket_options=socket_options
+        )
+
+    def sleep(self, seconds: float) -> None:
+        self._backend.sleep(seconds)
+
+
+class PinnedHTTPTransport(httpx.HTTPTransport):
+    """HTTPX transport that prevents a second, uncontrolled DNS lookup at connect time."""
+
+    def __init__(
+        self,
+        address_book: PinnedAddressBook,
+        limits: httpx.Limits,
+        proxy: str | None = None,
+    ) -> None:
+        super().__init__(
+            verify=True,
+            trust_env=False,
+            http1=True,
+            http2=False,
+            limits=limits,
+            proxy=proxy,
+            retries=0,
+        )
+        # HTTPX constructs a direct httpcore ConnectionPool for this transport.
+        self._pool._network_backend = PinnedNetworkBackend(address_book)

app/scheduler.py

@@ -0,0 +1,121 @@
+from __future__ import annotations
+
+import logging
+import os
+import shutil
+import socket
+import threading
+import time
+import uuid
+from pathlib import Path
+
+from app.capture import CaptureService
+from app.config import Settings
+from app.db import Database
+
+logger = logging.getLogger(__name__)
+
+
+class JobRunner:
+    """SQLite-backed local workers. Every process safely claims jobs through a database lease."""
+
+    def __init__(self, database: Database, settings: Settings) -> None:
+        self.database = database
+        self.settings = settings
+        self.stop_event = threading.Event()
+        self.threads: list[threading.Thread] = []
+        self.maintenance_thread: threading.Thread | None = None
+        self._last_expiry_sweep = 0.0
+
+    def start(self) -> None:
+        if not self.maintenance_thread:
+            self.maintenance_thread = threading.Thread(
+                target=self._run_maintenance,
+                name="siteharbor-maintenance",
+                daemon=True,
+            )
+            self.maintenance_thread.start()
+        if self.settings.worker_enabled and not self.threads:
+            for index in range(self.settings.worker_concurrency):
+                worker_id = f"{socket.gethostname()}-{os.getpid()}-{index}-{uuid.uuid4().hex[:8]}"
+                thread = threading.Thread(
+                    target=self._run_worker,
+                    args=(worker_id,),
+                    name=f"siteharbor-worker-{index}",
+                    daemon=True,
+                )
+                thread.start()
+                self.threads.append(thread)
+
+    def stop(self) -> None:
+        self.stop_event.set()
+        for thread in self.threads:
+            thread.join(timeout=5)
+        if self.maintenance_thread:
+            self.maintenance_thread.join(timeout=2)
+
+    def _run_worker(self, worker_id: str) -> None:
+        service = CaptureService(self.database, self.settings, worker_id)
+        while not self.stop_event.is_set():
+            try:
+                job = self.database.claim_next(worker_id, self.settings.lease_seconds)
+                if not job:
+                    self.stop_event.wait(self.settings.queue_poll_seconds)
+                    continue
+                service.run(job)
+            except Exception:
+                logger.exception("SiteHarbor worker loop failed")
+                self.stop_event.wait(self.settings.queue_poll_seconds)
+
+    def _run_maintenance(self) -> None:
+        while not self.stop_event.is_set():
+            try:
+                self._expire_artifacts_if_due()
+                self._sweep_orphaned_attempts()
+            except Exception:
+                logger.exception("SiteHarbor maintenance loop failed")
+            self.stop_event.wait(10)
+
+    def _expire_artifacts_if_due(self) -> None:
+        now = time.monotonic()
+        if now - self._last_expiry_sweep < 60:
+            return
+        self._last_expiry_sweep = now
+        for job in self.database.expired_ready_jobs():
+            self._unlink_under_root(job.get("artifact_path"), self.settings.artifacts_dir)
+            self._unlink_under_root(job.get("report_path"), self.settings.reports_dir)
+            self.database.mark_expired(str(job["id"]))
+
+    def _sweep_orphaned_attempts(self) -> None:
+        cutoff = time.time() - max(7_200, self.settings.max_duration_seconds_cap + 3_600)
+        referenced_paths = {Path(path).resolve() for path in self.database.referenced_data_paths()}
+
+        for path in self.settings.work_dir.iterdir():
+            try:
+                if path.stat().st_mtime >= cutoff:
+                    continue
+                if path.is_dir():
+                    shutil.rmtree(path, ignore_errors=True)
+            except OSError:
+                logger.warning("Could not remove orphaned SiteHarbor work directory: %s", path)
+
+        for root in (self.settings.artifacts_dir, self.settings.reports_dir):
+            for path in root.iterdir():
+                try:
+                    if path.resolve() in referenced_paths or path.stat().st_mtime >= cutoff:
+                        continue
+                    if path.is_file():
+                        path.unlink(missing_ok=True)
+                except OSError:
+                    logger.warning("Could not remove orphaned SiteHarbor output: %s", path)
+
+    @staticmethod
+    def _unlink_under_root(path_value: object, root: Path) -> None:
+        if not isinstance(path_value, str) or not path_value:
+            return
+        path = Path(path_value)
+        try:
+            if path.resolve().is_relative_to(root.resolve()):
+                path.unlink(missing_ok=True)
+        except OSError:
+            logger.warning("Could not remove expired SiteHarbor artifact: %s", path)

app/static/app.js

@@ -0,0 +1,679 @@
+const form = document.querySelector("#capture-form");
+const urlInput = document.querySelector("#capture-url");
+const submitButton = document.querySelector("#capture-submit");
+const formMessage = document.querySelector("#form-message");
+const jobGrid = document.querySelector("#job-grid");
+const emptyState = document.querySelector("#empty-state");
+const refreshButton = document.querySelector("#refresh-jobs");
+const serviceStatus = document.querySelector("#service-status");
+const reportDialog = document.querySelector("#report-dialog");
+const reportContent = document.querySelector("#report-content");
+const closeReportButton = document.querySelector("#close-report");
+
+const controls = {
+  externalAssets: document.querySelector("#external-assets"),
+  parallelConnections: document.querySelector("#parallel-connections"),
+  parallelValue: document.querySelector("#parallel-value"),
+  maxPages: document.querySelector("#max-pages"),
+  maxDepth: document.querySelector("#max-depth"),
+  maxBytes: document.querySelector("#max-bytes"),
+  maxDuration: document.querySelector("#max-duration"),
+  retentionDays: document.querySelector("#retention-days"),
+  budgetSummary: document.querySelector("#budget-summary"),
+  assetTypes: [...document.querySelectorAll("[data-asset-type]")],
+  skipFileTypes: document.querySelector("#skip-file-types"),
+  useProxy: document.querySelector("#use-proxy"),
+  proxySummary: document.querySelector("#proxy-summary"),
+  proxyDetail: document.querySelector("#proxy-detail"),
+  proxyRow: document.querySelector(".proxy-row"),
+};
+
+const publicStats = {
+  websites: document.querySelector("#stat-websites"),
+  data: document.querySelector("#stat-data"),
+  active: document.querySelector("#stat-active"),
+  files: document.querySelector("#stat-files"),
+};
+
+const OWNER_STORAGE_KEY = "siteharbor.owner-token.v1";
+const SESSION_STORAGE_KEY = "siteharbor.sessions.v1";
+const OWNER_HEADER = "X-SiteHarbor-Session";
+const MAX_STORED_SESSIONS = 40;
+const TOKEN_PATTERN = /^[A-Za-z0-9_-]{24,200}$/;
+
+const jobs = new Map();
+const streams = new Map();
+let storageAvailable = true;
+const ownerToken = getOwnerToken();
+
+function createOwnerToken() {
+  if (window.crypto?.randomUUID) return window.crypto.randomUUID();
+  if (window.crypto?.getRandomValues) {
+    const bytes = new Uint8Array(24);
+    window.crypto.getRandomValues(bytes);
+    return [...bytes].map((byte) => byte.toString(16).padStart(2, "0")).join("");
+  }
+  return `browser-${Date.now()}-${Math.random().toString(36).slice(2).padEnd(16, "0")}`;
+}
+
+function getOwnerToken() {
+  try {
+    const existing = window.localStorage.getItem(OWNER_STORAGE_KEY);
+    if (existing && TOKEN_PATTERN.test(existing)) return existing;
+    const token = createOwnerToken();
+    window.localStorage.setItem(OWNER_STORAGE_KEY, token);
+    return token;
+  } catch {
+    storageAvailable = false;
+    return createOwnerToken();
+  }
+}
+
+function readStoredSessions() {
+  if (!storageAvailable) return [];
+  try {
+    const raw = window.localStorage.getItem(SESSION_STORAGE_KEY);
+    if (!raw) return [];
+    const parsed = JSON.parse(raw);
+    const sessions = Array.isArray(parsed) ? parsed : parsed.sessions;
+    if (!Array.isArray(sessions)) return [];
+    return sessions.filter((session) => typeof session?.id === "string" && session.id.length > 0);
+  } catch {
+    storageAvailable = false;
+    return [];
+  }
+}
+
+function writeStoredSessions(sessions) {
+  if (!storageAvailable) return;
+  try {
+    window.localStorage.setItem(
+      SESSION_STORAGE_KEY,
+      JSON.stringify({ version: 1, sessions: sessions.slice(0, MAX_STORED_SESSIONS) }),
+    );
+  } catch {
+    storageAvailable = false;
+  }
+}
+
+function snapshotJob(job) {
+  return {
+    id: job.id,
+    url: job.url,
+    state: job.state,
+    phase: job.phase,
+    message: job.message,
+    error_message: job.error_message,
+    warnings: Array.isArray(job.warnings) ? job.warnings.slice(0, 5) : [],
+    created_at: job.created_at,
+    updated_at: job.updated_at,
+    started_at: job.started_at,
+    completed_at: job.completed_at,
+    expires_at: job.expires_at,
+    cancel_requested: job.cancel_requested,
+    stats: job.stats || {},
+    options: job.options || {},
+    archive: job.archive || {},
+  };
+}
+
+function rememberJob(job) {
+  const sessions = readStoredSessions();
+  const record = { id: job.id, created_at: job.created_at, job: snapshotJob(job) };
+  const existingIndex = sessions.findIndex((session) => session.id === job.id);
+  if (existingIndex >= 0) sessions[existingIndex] = { ...sessions[existingIndex], ...record };
+  else sessions.unshift(record);
+  sessions.sort((left, right) => sortByCreated(right.job || right, left.job || left));
+  writeStoredSessions(sessions);
+}
+
+function forgetJob(jobId) {
+  jobs.delete(jobId);
+  closeEvents(jobId);
+  writeStoredSessions(readStoredSessions().filter((session) => session.id !== jobId));
+  renderJobs();
+}
+
+function sortByCreated(left, right) {
+  return String(left.created_at || "").localeCompare(String(right.created_at || ""));
+}
+
+function formatBytes(value) {
+  const amount = Number(value) || 0;
+  if (!amount) return "0 B";
+  const units = ["B", "KB", "MB", "GB", "TB"];
+  const index = Math.min(Math.floor(Math.log(amount) / Math.log(1024)), units.length - 1);
+  const rounded = amount / (1024 ** index);
+  return `${rounded >= 10 || index === 0 ? rounded.toFixed(0) : rounded.toFixed(1)} ${units[index]}`;
+}
+
+function formatCount(value) {
+  return new Intl.NumberFormat(undefined, { notation: "compact", maximumFractionDigits: 1 }).format(
+    Number(value) || 0,
+  );
+}
+
+function formatTime(value) {
+  if (!value) return "--";
+  const date = new Date(value);
+  return Number.isNaN(date.getTime())
+    ? "--"
+    : date.toLocaleString([], { dateStyle: "medium", timeStyle: "short" });
+}
+
+function hostname(url) {
+  try {
+    return new URL(url).hostname;
+  } catch {
+    return url;
+  }
+}
+
+function setMessage(message = "", kind = "") {
+  formMessage.textContent = message;
+  formMessage.className = `form-message ${kind}`.trim();
+}
+
+function setServiceStatus(message, online = false) {
+  serviceStatus.querySelector("span").textContent = message;
+  serviceStatus.dataset.online = String(online);
+}
+
+function readNumber(control, fallback) {
+  const value = Number.parseInt(control.value, 10);
+  return Number.isFinite(value) ? value : fallback;
+}
+
+function formatDuration(seconds) {
+  const value = Number(seconds) || 0;
+  if (value >= 60 && value % 60 === 0) return `${value / 60} min`;
+  return `${value} sec`;
+}
+
+function updateBudgetSummary() {
+  controls.budgetSummary.textContent = [
+    `${readNumber(controls.maxPages, 150)} pages`,
+    `${readNumber(controls.maxBytes, 100)} MB`,
+    formatDuration(readNumber(controls.maxDuration, 300)),
+  ].join(" / ");
+}
+
+function updateParallelValue() {
+  controls.parallelValue.value = controls.parallelConnections.value;
+  controls.parallelValue.textContent = controls.parallelConnections.value;
+}
+
+function updateProxySummary() {
+  if (controls.useProxy.disabled) {
+    controls.proxySummary.textContent = "Not configured";
+  } else if (controls.useProxy.checked) {
+    controls.proxySummary.textContent = "Approved proxy";
+  } else {
+    controls.proxySummary.textContent = "Direct connection";
+  }
+}
+
+function setControlLimits(capabilities) {
+  const defaults = capabilities.defaults || {};
+  const limits = capabilities.limits || {};
+  const bindings = [
+    [controls.maxPages, "max_pages"],
+    [controls.maxDepth, "max_depth"],
+    [controls.maxBytes, "max_bytes_mb"],
+    [controls.maxDuration, "max_duration_seconds"],
+    [controls.retentionDays, "max_retention_days"],
+  ];
+  bindings.forEach(([control, key]) => {
+    if (limits[key] !== undefined) control.max = limits[key];
+  });
+  if (limits.max_parallel_connections !== undefined) {
+    controls.parallelConnections.max = limits.max_parallel_connections;
+  }
+
+  controls.maxPages.value = defaults.max_pages ?? controls.maxPages.value;
+  controls.maxDepth.value = defaults.max_depth ?? controls.maxDepth.value;
+  controls.maxBytes.value = defaults.max_bytes_mb ?? controls.maxBytes.value;
+  controls.maxDuration.value = defaults.max_duration_seconds ?? controls.maxDuration.value;
+  controls.retentionDays.value = defaults.retention_days ?? controls.retentionDays.value;
+  controls.externalAssets.checked = defaults.include_external_assets ?? true;
+  controls.parallelConnections.value = defaults.parallel_connections ?? controls.parallelConnections.value;
+
+  const selectedTypes = new Set(defaults.download_asset_types || []);
+  controls.assetTypes.forEach((control) => {
+    control.checked = selectedTypes.has(control.dataset.assetType);
+  });
+  controls.skipFileTypes.value = (defaults.skip_file_types || []).join(", ");
+  controls.useProxy.checked = Boolean(defaults.use_proxy);
+  controls.useProxy.disabled = !capabilities.proxy_available;
+  controls.proxyRow.classList.toggle("is-unavailable", !capabilities.proxy_available);
+  controls.proxyDetail.textContent = capabilities.proxy_available
+    ? "Use the operator-approved route for this capture."
+    : "No operator-approved proxy route is available on this service.";
+
+  updateBudgetSummary();
+  updateParallelValue();
+  updateProxySummary();
+}
+
+async function request(path, options = {}, { session = true } = {}) {
+  const headers = new Headers(options.headers || {});
+  if (session) headers.set(OWNER_HEADER, ownerToken);
+  if (options.body && !headers.has("Content-Type")) headers.set("Content-Type", "application/json");
+
+  const response = await fetch(path, { cache: "no-store", ...options, headers });
+  const body = response.status === 204 ? null : await response.json().catch(() => null);
+  if (!response.ok) {
+    const error = new Error(body?.detail || "SiteHarbor could not complete that request.");
+    error.status = response.status;
+    throw error;
+  }
+  return body;
+}
+
+function isActive(job) {
+  return ["queued", "running"].includes(job.state);
+}
+
+function upsertJob(job, { persist = true, connect = true } = {}) {
+  jobs.set(job.id, job);
+  if (persist) rememberJob(job);
+  renderJobs();
+  if (connect && isActive(job)) connectEvents(job.id);
+  else if (!isActive(job)) closeEvents(job.id);
+}
+
+function renderJobs() {
+  const orderedJobs = [...jobs.values()].sort((left, right) => sortByCreated(right, left));
+  emptyState.hidden = orderedJobs.length > 0;
+  jobGrid.replaceChildren(...orderedJobs.map(createJobCard));
+}
+
+function createElement(tag, className, text) {
+  const element = document.createElement(tag);
+  if (className) element.className = className;
+  if (text !== undefined) element.textContent = text;
+  return element;
+}
+
+function jobProgress(job) {
+  if (job.state === "ready") return 100;
+  if (["failed", "cancelled", "expired"].includes(job.state)) return 100;
+  const phase = String(job.phase || "").toLowerCase();
+  if (phase.includes("packag")) return 94;
+  if (phase.includes("rewrit")) return 88;
+  if (phase.includes("validat") || phase.includes("prepar")) return 6;
+  const maximum = Number(job.options?.max_pages) || 0;
+  const fetched = Number(job.stats?.pages_fetched) || 0;
+  if (!maximum) return 16;
+  return Math.max(10, Math.min(82, Math.round((fetched / maximum) * 82)));
+}
+
+function createJobCard(job) {
+  const card = createElement("article", "job-card");
+  card.dataset.state = job.state;
+
+  const topLine = createElement("div", "job-topline");
+  topLine.append(
+    createElement("div", "job-host", hostname(job.url)),
+    createElement("span", `state-pill ${job.state}`, job.phase || job.state),
+  );
+  card.append(topLine);
+  card.append(createElement("p", "job-url", job.url));
+  card.append(createElement("p", "job-phase", job.error_message || job.message || "Waiting for status"));
+
+  const progress = createElement("div", "job-progress");
+  progress.append(createElement("span"));
+  progress.firstElementChild.style.setProperty("--progress", `${jobProgress(job)}%`);
+  card.append(progress);
+
+  const stats = createElement("div", "job-stats");
+  const statItems = [
+    [job.stats?.pages_fetched || 0, "pages"],
+    [job.stats?.files_written || 0, "files"],
+    [formatBytes(job.stats?.bytes_downloaded || 0), "collected"],
+  ];
+  statItems.forEach(([value, label]) => {
+    const item = document.createElement("div");
+    item.append(createElement("strong", "", String(value)), createElement("span", "", label));
+    stats.append(item);
+  });
+  card.append(stats);
+
+  const meta = createElement("div", "job-meta");
+  meta.append(
+    createElement("span", "", `Created ${formatTime(job.created_at)}`),
+    createElement(
+      "span",
+      "",
+      job.archive?.available
+        ? `Archive ${formatBytes(job.archive.size_bytes)}`
+        : `${job.options?.max_pages || "--"} page budget`,
+    ),
+  );
+  card.append(meta);
+
+  const tags = createElement("div", "job-tags");
+  const parallel = job.options?.parallel_connections;
+  if (parallel) tags.append(createElement("span", "", `${parallel} connections`));
+  tags.append(
+    createElement(
+      "span",
+      "",
+      job.options?.include_external_assets ? "external assets" : "first-party only",
+    ),
+  );
+  if (job.options?.skip_file_types?.length) {
+    tags.append(createElement("span", "", `skip .${job.options.skip_file_types[0]}`));
+  }
+  card.append(tags);
+
+  if (job.warnings?.length) card.append(createElement("p", "job-warning", job.warnings[0]));
+
+  const actions = createElement("div", "job-actions");
+  if (job.archive?.available) {
+    const download = createElement("button", "job-action primary", "Download ZIP");
+    download.type = "button";
+    download.addEventListener("click", () => downloadArchive(job, download));
+    actions.append(download);
+
+    const report = createElement("button", "job-action", "View report");
+    report.type = "button";
+    report.addEventListener("click", () => showReport(job.id));
+    actions.append(report);
+
+    const remove = createElement("button", "job-action danger", "Delete archive");
+    remove.type = "button";
+    remove.addEventListener("click", () => deleteJob(job.id));
+    actions.append(remove);
+  }
+  if (isActive(job)) {
+    const cancel = createElement("button", "job-action danger", job.cancel_requested ? "Cancelling" : "Cancel");
+    cancel.type = "button";
+    cancel.disabled = Boolean(job.cancel_requested);
+    cancel.addEventListener("click", () => cancelJob(job.id));
+    actions.append(cancel);
+  }
+  if (!isActive(job) && !job.archive?.available) {
+    const forget = createElement("button", "job-action", "Forget session");
+    forget.type = "button";
+    forget.addEventListener("click", () => forgetJob(job.id));
+    actions.append(forget);
+  }
+  card.append(actions);
+  return card;
+}
+
+function connectEvents(jobId) {
+  if (streams.has(jobId)) return;
+  const eventUrl = `/api/captures/${encodeURIComponent(jobId)}/events?session=${encodeURIComponent(ownerToken)}`;
+  const stream = new EventSource(eventUrl);
+  streams.set(jobId, stream);
+  ["state", "progress"].forEach((eventName) => {
+    stream.addEventListener(eventName, () => {
+      void refreshJob(jobId);
+    });
+  });
+  stream.addEventListener("terminal", () => {
+    closeEvents(jobId);
+    void refreshJob(jobId);
+    void loadPublicStats();
+  });
+  stream.onerror = () => {
+    window.setTimeout(() => {
+      if (streams.get(jobId) === stream) void refreshJob(jobId);
+    }, 1_000);
+  };
+}
+
+function closeEvents(jobId) {
+  const stream = streams.get(jobId);
+  if (stream) stream.close();
+  streams.delete(jobId);
+}
+
+async function refreshJob(jobId) {
+  try {
+    upsertJob(await request(`/api/captures/${encodeURIComponent(jobId)}`));
+  } catch (error) {
+    closeEvents(jobId);
+    if (error.status === 404) forgetJob(jobId);
+  }
+}
+
+async function refreshKnownJobs() {
+  const storedIds = readStoredSessions().map((session) => session.id);
+  const knownIds = [...new Set([...storedIds, ...jobs.keys()])];
+  await Promise.all(knownIds.map((jobId) => refreshJob(jobId)));
+}
+
+async function loadStoredJobs() {
+  const sessions = readStoredSessions();
+  sessions.forEach((session) => {
+    if (session.job?.id && !jobs.has(session.id)) jobs.set(session.id, session.job);
+  });
+  renderJobs();
+  await refreshKnownJobs();
+}
+
+async function loadPublicStats() {
+  try {
+    const stats = await request("/api/stats", {}, { session: false });
+    publicStats.websites.textContent = formatCount(stats.websites_cloned);
+    publicStats.data.textContent = formatBytes(stats.bytes_scraped);
+    publicStats.active.textContent = formatCount(stats.active_crawls);
+    publicStats.files.textContent = formatCount(stats.files_archived);
+    setServiceStatus("Capture service online", true);
+  } catch {
+    setServiceStatus("Service unavailable", false);
+  }
+}
+
+async function cancelJob(jobId) {
+  try {
+    upsertJob(await request(`/api/captures/${encodeURIComponent(jobId)}/cancel`, { method: "POST" }));
+    void loadPublicStats();
+  } catch (error) {
+    setMessage(error.message);
+  }
+}
+
+async function deleteJob(jobId) {
+  try {
+    const updated = await request(`/api/captures/${encodeURIComponent(jobId)}`, { method: "DELETE" });
+    if (updated.state === "expired") forgetJob(jobId);
+    else upsertJob(updated);
+    void loadPublicStats();
+  } catch (error) {
+    setMessage(error.message);
+  }
+}
+
+async function downloadArchive(job, button) {
+  if (!job.archive?.download_url) return;
+  const originalText = button.textContent;
+  button.disabled = true;
+  button.textContent = "Preparing download";
+  try {
+    const response = await fetch(job.archive.download_url, {
+      headers: { [OWNER_HEADER]: ownerToken },
+    });
+    if (!response.ok) {
+      const body = await response.json().catch(() => null);
+      const error = new Error(body?.detail || "The archive could not be downloaded.");
+      error.status = response.status;
+      throw error;
+    }
+    const objectUrl = URL.createObjectURL(await response.blob());
+    const anchor = document.createElement("a");
+    anchor.href = objectUrl;
+    anchor.download = `siteharbor-${job.id}.zip`;
+    document.body.append(anchor);
+    anchor.click();
+    anchor.remove();
+    window.setTimeout(() => URL.revokeObjectURL(objectUrl), 1_000);
+  } catch (error) {
+    setMessage(error.message);
+  } finally {
+    button.disabled = false;
+    button.textContent = originalText;
+  }
+}
+
+async function showReport(jobId) {
+  try {
+    const report = await request(`/api/captures/${encodeURIComponent(jobId)}/report`);
+    renderReport(report);
+    reportDialog.showModal();
+  } catch (error) {
+    setMessage(error.message);
+  }
+}
+
+function renderReport(report) {
+  reportContent.replaceChildren();
+  const summary = createElement("div", "report-summary");
+  const stats = report.stats || {};
+  [
+    [stats.pages_fetched || 0, "pages"],
+    [stats.assets_fetched || 0, "assets"],
+    [formatBytes(stats.bytes_downloaded || 0), "downloaded"],
+    [report.quality || "unknown", "capture quality"],
+  ].forEach(([value, label]) => {
+    const item = document.createElement("div");
+    item.append(createElement("strong", "", String(value)), createElement("span", "", label));
+    summary.append(item);
+  });
+  reportContent.append(summary);
+
+  const details = createElement("ul", "report-list");
+  [
+    `Entry point: ${report.entry_point || "not available"}`,
+    `Capture duration: ${report.elapsed_seconds || 0} seconds`,
+    `Captured resources: ${(report.resources || []).length}`,
+  ].forEach((line) => details.append(createElement("li", "", line)));
+  reportContent.append(details);
+
+  if (report.warnings?.length) {
+    reportContent.append(createElement("h3", "report-subhead", "Warnings"));
+    const warnings = createElement("ul", "report-list");
+    report.warnings.forEach((warning) => warnings.append(createElement("li", "", warning)));
+    reportContent.append(warnings);
+  }
+  if (report.errors?.length) {
+    reportContent.append(createElement("h3", "report-subhead", "Resource failures"));
+    const errors = createElement("ul", "report-list");
+    report.errors.slice(0, 20).forEach((error) => {
+      errors.append(createElement("li", "", `${error.url}: ${error.message}`));
+    });
+    reportContent.append(errors);
+  }
+}
+
+function selectedAssetTypes() {
+  return controls.assetTypes
+    .filter((control) => control.checked)
+    .map((control) => control.dataset.assetType);
+}
+
+function skippedFileTypes() {
+  return controls.skipFileTypes.value
+    .split(",")
+    .map((item) => item.trim())
+    .filter(Boolean);
+}
+
+form.addEventListener("submit", async (event) => {
+  event.preventDefault();
+  const url = urlInput.value.trim();
+  if (!url) {
+    setMessage("Enter a public website URL to begin.");
+    urlInput.focus();
+    return;
+  }
+
+  setMessage("");
+  submitButton.disabled = true;
+  submitButton.querySelector("span").textContent = "Creating crawl";
+  try {
+    const job = await request("/api/captures", {
+      method: "POST",
+      body: JSON.stringify({
+        url,
+        include_external_assets: controls.externalAssets.checked,
+        parallel_connections: readNumber(controls.parallelConnections, 6),
+        download_asset_types: selectedAssetTypes(),
+        skip_file_types: skippedFileTypes(),
+        use_proxy: controls.useProxy.checked && !controls.useProxy.disabled,
+        max_pages: readNumber(controls.maxPages, 150),
+        max_depth: readNumber(controls.maxDepth, 5),
+        max_bytes_mb: readNumber(controls.maxBytes, 100),
+        max_duration_seconds: readNumber(controls.maxDuration, 300),
+        retention_days: readNumber(controls.retentionDays, 7),
+      }),
+    });
+    upsertJob(job);
+    setMessage("Crawl queued. This browser will reconnect to it automatically.", "success");
+    urlInput.value = "";
+    void loadPublicStats();
+  } catch (error) {
+    setMessage(error.message);
+  } finally {
+    submitButton.disabled = false;
+    submitButton.querySelector("span").textContent = "Start private crawl";
+  }
+});
+
+[
+  controls.maxPages,
+  controls.maxDepth,
+  controls.maxBytes,
+  controls.maxDuration,
+  controls.retentionDays,
+].forEach((control) => control.addEventListener("input", updateBudgetSummary));
+controls.parallelConnections.addEventListener("input", updateParallelValue);
+controls.useProxy.addEventListener("change", updateProxySummary);
+
+refreshButton.addEventListener("click", async () => {
+  refreshButton.disabled = true;
+  try {
+    await Promise.all([refreshKnownJobs(), loadPublicStats()]);
+  } finally {
+    refreshButton.disabled = false;
+  }
+});
+closeReportButton.addEventListener("click", () => reportDialog.close());
+reportDialog.addEventListener("click", (event) => {
+  if (event.target === reportDialog) reportDialog.close();
+});
+
+document.addEventListener("visibilitychange", () => {
+  if (!document.hidden) {
+    void refreshKnownJobs();
+    void loadPublicStats();
+  }
+});
+window.addEventListener("online", () => {
+  void refreshKnownJobs();
+  void loadPublicStats();
+});
+window.addEventListener("beforeunload", () => {
+  streams.forEach((stream) => stream.close());
+});
+
+async function initialize() {
+  if (!storageAvailable) {
+    setMessage("Browser storage is unavailable, so this session cannot be restored after this tab closes.");
+  }
+  try {
+    setControlLimits(await request("/api/capabilities", {}, { session: false }));
+  } catch (error) {
+    setMessage(error.message);
+  }
+  await Promise.all([loadStoredJobs(), loadPublicStats()]);
+  window.setInterval(() => {
+    void refreshKnownJobs();
+    void loadPublicStats();
+  }, 15_000);
+}
+
+void initialize();

app/templates/index.html

@@ -0,0 +1,238 @@
+<!doctype html>
+<html lang="en">
+  <head>
+    <meta charset="utf-8">
+    <meta name="viewport" content="width=device-width, initial-scale=1">
+    <meta name="theme-color" content="#151715">
+    <meta
+      name="description"
+      content="Create browser-owned offline archives of public websites with SiteHarbor."
+    >
+    <title>{{ app_name }} - Public web capture</title>
+    <link rel="stylesheet" href="{{ url_for('static', path='app.css') }}">
+    <script defer src="{{ url_for('static', path='app.js') }}"></script>
+  </head>
+  <body>
+    <div class="site-shell">
+      <header class="topbar">
+        <a class="brand" href="/" aria-label="SiteHarbor home">
+          <span class="brand-mark" aria-hidden="true"><i></i><i></i><i></i></span>
+          <span>siteharbor</span>
+        </a>
+        <div class="topbar-meta">
+          <span class="public-label">PUBLIC WEB CAPTURE</span>
+          <span class="service-status" id="service-status" aria-live="polite">
+            <i></i><span>Checking service</span>
+          </span>
+        </div>
+      </header>
+
+      <main>
+        <section class="workbench" aria-label="Website capture workspace">
+          <div class="observatory">
+            <div class="observatory-copy">
+              <p class="eyebrow">LIVE NETWORK SIGNAL</p>
+              <h1>Copy the web.<br><em>Keep it close.</em></h1>
+              <p class="lede">
+                Build an offline-ready snapshot of a public site. The service runs the crawl;
+                this browser keeps the key to its own sessions.
+              </p>
+            </div>
+
+            <div class="metric-grid" aria-live="polite" aria-label="Public service statistics">
+              <article class="metric-card metric-primary">
+                <span class="metric-value" id="stat-websites">--</span>
+                <span class="metric-label">Websites cloned</span>
+              </article>
+              <article class="metric-card">
+                <span class="metric-value" id="stat-data">--</span>
+                <span class="metric-label">Data collected</span>
+              </article>
+              <article class="metric-card">
+                <span class="metric-value" id="stat-active">--</span>
+                <span class="metric-label">Crawls running</span>
+              </article>
+              <article class="metric-card">
+                <span class="metric-value" id="stat-files">--</span>
+                <span class="metric-label">Files preserved</span>
+              </article>
+            </div>
+
+            <div class="observatory-foot">
+              <span class="pulse-dot" aria-hidden="true"></span>
+              <p>Metrics are shared totals. Crawl URLs and archives stay private to the browser session that created them.</p>
+            </div>
+          </div>
+
+          <form class="capture-console" id="capture-form" novalidate>
+            <div class="console-head">
+              <div>
+                <p class="eyebrow">NEW SESSION</p>
+                <h2>Set your crawl.</h2>
+              </div>
+              <span class="console-step">01 / 01</span>
+            </div>
+
+            <label class="url-field" for="capture-url">
+              <span class="field-label">Public website URL</span>
+              <span class="url-input-wrap">
+                <span aria-hidden="true">//</span>
+                <input
+                  id="capture-url"
+                  name="url"
+                  type="url"
+                  inputmode="url"
+                  autocomplete="url"
+                  placeholder="https://example.com"
+                  required
+                >
+              </span>
+            </label>
+
+            <div class="quick-settings">
+              <label class="switch-row" for="external-assets">
+                <span>
+                  <strong>Include external assets</strong>
+                  <small>Keep linked CDN styles, scripts, fonts, and media.</small>
+                </span>
+                <span class="switch">
+                  <input id="external-assets" type="checkbox" checked>
+                  <span aria-hidden="true"></span>
+                </span>
+              </label>
+
+              <label class="connection-control" for="parallel-connections">
+                <span>
+                  <strong>Parallel connections</strong>
+                  <small>Balance capture speed with a site's capacity.</small>
+                </span>
+                <span class="connection-value"><output id="parallel-value" for="parallel-connections">6</output> streams</span>
+                <input id="parallel-connections" type="range" min="1" max="6" value="6">
+              </label>
+            </div>
+
+            <div class="advanced-settings">
+              <details class="setting-panel">
+                <summary>
+                  <span>Crawl limits</span>
+                  <span class="summary-value" id="budget-summary">150 pages / 100 MB / 5 min</span>
+                </summary>
+                <div class="limit-grid">
+                  <label>
+                    <span>Pages</span>
+                    <input id="max-pages" type="number" min="1" step="1" value="150">
+                  </label>
+                  <label>
+                    <span>Depth</span>
+                    <input id="max-depth" type="number" min="0" step="1" value="5">
+                  </label>
+                  <label>
+                    <span>Size (MB)</span>
+                    <input id="max-bytes" type="number" min="1" step="1" value="100">
+                  </label>
+                  <label>
+                    <span>Time (sec)</span>
+                    <input id="max-duration" type="number" min="10" step="10" value="300">
+                  </label>
+                  <label>
+                    <span>Keep (days)</span>
+                    <input id="retention-days" type="number" min="1" step="1" value="7">
+                  </label>
+                </div>
+              </details>
+
+              <details class="setting-panel">
+                <summary>
+                  <span>File rules</span>
+                  <span class="summary-value">Choose what comes aboard</span>
+                </summary>
+                <div class="file-rules">
+                  <p>Pages are always copied. Select the asset groups that should be downloaded.</p>
+                  <div class="asset-choices" role="group" aria-label="Asset types to download">
+                    <label><input data-asset-type="styles" type="checkbox" checked><span>Styles</span></label>
+                    <label><input data-asset-type="scripts" type="checkbox" checked><span>Scripts</span></label>
+                    <label><input data-asset-type="images" type="checkbox" checked><span>Images</span></label>
+                    <label><input data-asset-type="fonts" type="checkbox" checked><span>Fonts</span></label>
+                    <label><input data-asset-type="media" type="checkbox" checked><span>Media</span></label>
+                    <label><input data-asset-type="other" type="checkbox" checked><span>Other files</span></label>
+                  </div>
+                  <label class="text-rule" for="skip-file-types">
+                    <span>Skip specific extensions</span>
+                    <input
+                      id="skip-file-types"
+                      type="text"
+                      autocomplete="off"
+                      placeholder="pdf, zip, mp4"
+                    >
+                  </label>
+                </div>
+              </details>
+
+              <details class="setting-panel">
+                <summary>
+                  <span>Network route</span>
+                  <span class="summary-value" id="proxy-summary">Direct connection</span>
+                </summary>
+                <label class="switch-row proxy-row" for="use-proxy">
+                  <span>
+                    <strong>Use service proxy</strong>
+                    <small id="proxy-detail">An operator-approved proxy can be used for this crawl.</small>
+                  </span>
+                  <span class="switch">
+                    <input id="use-proxy" type="checkbox">
+                    <span aria-hidden="true"></span>
+                  </span>
+                </label>
+              </details>
+            </div>
+
+            <p class="form-message" id="form-message" aria-live="polite"></p>
+            <div class="console-actions">
+              <button class="capture-button" id="capture-submit" type="submit">
+                <span>Start private crawl</span>
+                <span aria-hidden="true">-&gt;</span>
+              </button>
+              <p>Only crawl sites you are allowed to archive. Private networks and unsafe ports are blocked.</p>
+            </div>
+          </form>
+        </section>
+
+        <section class="activity-section" aria-labelledby="activity-title">
+          <div class="section-heading">
+            <div>
+              <p class="eyebrow">THIS BROWSER</p>
+              <h2 id="activity-title">Your local sessions</h2>
+              <p>Session records stay in this browser's local storage. Reopen this page to reconnect to an active crawl.</p>
+            </div>
+            <button class="quiet-button" id="refresh-jobs" type="button">Refresh sessions</button>
+          </div>
+
+          <div class="empty-state" id="empty-state">
+            <span class="empty-mark" aria-hidden="true">+</span>
+            <div>
+              <h3>No local sessions yet</h3>
+              <p>Start a crawl above. Its status and archive will remain available from this browser after a reconnect.</p>
+            </div>
+          </div>
+          <div class="job-grid" id="job-grid" aria-live="polite"></div>
+        </section>
+      </main>
+
+      <footer class="footer-note">
+        <span>SiteHarbor captures browser-delivered static content.</span>
+        <span>Dynamic APIs, logins, service workers, and server behavior may not work offline.</span>
+      </footer>
+    </div>
+
+    <dialog class="report-dialog" id="report-dialog" aria-labelledby="report-title">
+      <div class="dialog-head">
+        <div>
+          <p class="eyebrow">CAPTURE REPORT</p>
+          <h2 id="report-title">Archive details</h2>
+        </div>
+        <button class="icon-button" id="close-report" type="button" aria-label="Close report">x</button>
+      </div>
+      <div class="report-content" id="report-content"></div>
+    </dialog>
+  </body>
+</html>

app/url_policy.py

@@ -0,0 +1,109 @@
+from __future__ import annotations
+
+import ipaddress
+import re
+import socket
+from dataclasses import dataclass
+from urllib.parse import quote, urlsplit, urlunsplit
+
+
+class UrlPolicyError(ValueError):
+    """Raised when a URL is structurally unsafe or points at a blocked network."""
+
+
+@dataclass(frozen=True, slots=True)
+class NetworkRules:
+    allow_private_networks: bool = False
+    allow_nonstandard_ports: bool = False
+
+
+def normalize_url(value: str, rules: NetworkRules) -> str:
+    raw = value.strip()
+    if not raw:
+        raise UrlPolicyError("Enter a website address.")
+
+    if "://" not in raw:
+        raw = f"https://{raw}"
+
+    try:
+        parsed = urlsplit(raw)
+        port = parsed.port
+    except ValueError as error:
+        raise UrlPolicyError("The website address has an invalid port.") from error
+
+    if parsed.scheme.lower() not in {"http", "https"}:
+        raise UrlPolicyError("Only HTTP and HTTPS websites can be captured.")
+    if not parsed.hostname:
+        raise UrlPolicyError("The website address needs a hostname.")
+    if parsed.username or parsed.password:
+        raise UrlPolicyError("Website addresses with embedded credentials are not allowed.")
+
+    host = parsed.hostname.rstrip(".").lower()
+    try:
+        host = host.encode("idna").decode("ascii")
+    except UnicodeError as error:
+        raise UrlPolicyError("The website hostname is not valid.") from error
+
+    default_port = 443 if parsed.scheme.lower() == "https" else 80
+    if port and port != default_port and not rules.allow_nonstandard_ports:
+        raise UrlPolicyError("Only ports 80 and 443 are allowed by this SiteHarbor instance.")
+
+    host_for_netloc = f"[{host}]" if ":" in host else host
+    netloc = host_for_netloc if not port or port == default_port else f"{host_for_netloc}:{port}"
+    path_value = parsed.path or "/"
+    if re.search(r"%(?![0-9A-Fa-f]{2})", path_value):
+        raise UrlPolicyError("The website address has an invalid percent escape.")
+    # Keep encoded delimiters such as %2F intact: decoding them changes resource identity.
+    path = quote(path_value, safe="/%:@!$&'()*+,;=-._~")
+    return urlunsplit((parsed.scheme.lower(), netloc, path, parsed.query, ""))
+
+
+def canonical_url(value: str, rules: NetworkRules) -> str:
+    """Return a stable URL key with its fragment removed."""
+    return normalize_url(value, rules)
+
+
+def hostname(value: str) -> str:
+    host = urlsplit(value).hostname
+    if not host:
+        raise UrlPolicyError("The URL has no hostname.")
+    return host.rstrip(".").lower()
+
+
+def is_public_address(address: ipaddress.IPv4Address | ipaddress.IPv6Address) -> bool:
+    return bool(
+        address.is_global
+        and not address.is_loopback
+        and not address.is_link_local
+        and not address.is_multicast
+        and not address.is_unspecified
+        and not address.is_reserved
+        and not address.is_private
+    )
+
+
+def resolve_and_validate(value: str, rules: NetworkRules) -> tuple[str, ...]:
+    """Resolve a host and reject non-public destinations unless explicitly in dev mode."""
+    host = hostname(value)
+    try:
+        results = socket.getaddrinfo(host, None, type=socket.SOCK_STREAM)
+    except socket.gaierror as error:
+        raise UrlPolicyError(f"Could not resolve {host}.") from error
+
+    addresses: set[str] = set()
+    for _, _, _, _, sockaddr in results:
+        address = ipaddress.ip_address(sockaddr[0])
+        addresses.add(str(address))
+        if not rules.allow_private_networks and not is_public_address(address):
+            raise UrlPolicyError("Private, local, and reserved network targets are blocked.")
+
+    if not addresses:
+        raise UrlPolicyError(f"Could not resolve {host}.")
+    return tuple(sorted(addresses))
+
+
+def same_site_host(left: str, right: str) -> bool:
+    """Allow an exact host or the common example.com/www.example.com redirect pair."""
+    left = left.lower().removeprefix("www.")
+    right = right.lower().removeprefix("www.")
+    return left == right