__init__

app/main.py

@@ -0,0 +1,395 @@
+from __future__ import annotations
+
+import asyncio
+import json
+import re
+from collections.abc import AsyncIterator
+from contextlib import asynccontextmanager
+from pathlib import Path
+from typing import Any
+
+import uvicorn
+from fastapi import FastAPI, HTTPException, Request, Response, status
+from fastapi.responses import FileResponse, HTMLResponse, JSONResponse, StreamingResponse
+from fastapi.staticfiles import StaticFiles
+from fastapi.templating import Jinja2Templates
+
+from app.config import settings
+from app.db import Database
+from app.models import SUPPORTED_ASSET_TYPES, CaptureRequest
+from app.scheduler import JobRunner
+from app.url_policy import NetworkRules, UrlPolicyError, normalize_url
+
+settings.ensure_directories()
+database = Database(settings.database_path)
+runner = JobRunner(database, settings)
+templates = Jinja2Templates(directory=str(settings.project_root / "app" / "templates"))
+OWNER_TOKEN_HEADER = "X-SiteHarbor-Session"
+OWNER_TOKEN_RE = re.compile(r"[A-Za-z0-9_-]{24,200}")
+
+
+@asynccontextmanager
+async def lifespan(_: FastAPI) -> AsyncIterator[None]:
+    database.initialize()
+    runner.start()
+    try:
+        yield
+    finally:
+        runner.stop()
+
+
+app = FastAPI(
+    title="SiteHarbor",
+    version="0.1.0",
+    description="Standalone, bounded website capture for offline archives.",
+    lifespan=lifespan,
+)
+app.mount(
+    "/static", StaticFiles(directory=str(settings.project_root / "app" / "static")), name="static"
+)
+
+
+@app.middleware("http")
+async def secure_headers(_: Request, call_next: Any) -> Response:
+    response = await call_next(_)
+    response.headers.setdefault("X-Content-Type-Options", "nosniff")
+    response.headers.setdefault("Referrer-Policy", "same-origin")
+    response.headers.setdefault("X-Frame-Options", "DENY")
+    response.headers.setdefault(
+        "Content-Security-Policy",
+        "default-src 'self'; base-uri 'none'; object-src 'none'; frame-ancestors 'none'; "
+        "form-action 'self'; connect-src 'self'; img-src 'self' data:; style-src 'self'; "
+        "script-src 'self'",
+    )
+    return response
+
+
+@app.get("/", response_class=HTMLResponse)
+async def index(request: Request) -> Response:
+    return templates.TemplateResponse(
+        request=request,
+        name="index.html",
+        context={"app_name": "SiteHarbor"},
+    )
+
+
+@app.get("/api/health")
+async def health() -> dict[str, Any]:
+    return {
+        "status": "ok",
+        "worker_enabled": settings.worker_enabled,
+        "worker_concurrency": settings.worker_concurrency,
+        "storage": "local-sqlite",
+    }
+
+
+@app.get("/api/capabilities")
+async def capabilities() -> dict[str, Any]:
+    return {
+        "defaults": {
+            "max_pages": min(150, settings.max_pages_cap),
+            "max_depth": min(5, settings.max_depth_cap),
+            "max_bytes_mb": min(100, max(1, settings.max_bytes_cap // (1024 * 1024))),
+            "max_duration_seconds": min(300, settings.max_duration_seconds_cap),
+            "retention_days": min(settings.default_retention_days, settings.max_retention_days),
+            "include_external_assets": True,
+            "parallel_connections": min(6, settings.fetch_concurrency),
+            "download_asset_types": list(SUPPORTED_ASSET_TYPES),
+            "skip_file_types": [],
+            "use_proxy": False,
+        },
+        "limits": {
+            "max_pages": settings.max_pages_cap,
+            "max_depth": settings.max_depth_cap,
+            "max_bytes_mb": settings.max_bytes_cap // (1024 * 1024),
+            "max_duration_seconds": settings.max_duration_seconds_cap,
+            "max_retention_days": settings.max_retention_days,
+            "max_parallel_connections": settings.fetch_concurrency,
+        },
+        "proxy_available": bool(settings.proxy_url),
+        "local_only_warning": not settings.allow_private_networks,
+    }
+
+
+@app.get("/api/stats")
+async def public_stats() -> dict[str, int]:
+    return database.public_stats()
+
+
+@app.post("/api/captures", status_code=status.HTTP_202_ACCEPTED)
+async def create_capture(payload: CaptureRequest, request: Request) -> dict[str, Any]:
+    owner_token = _owner_token(request)
+    config = _validated_config(payload)
+    rules = NetworkRules(
+        allow_private_networks=settings.allow_private_networks,
+        allow_nonstandard_ports=settings.allow_nonstandard_ports,
+    )
+    try:
+        normalized_url = normalize_url(payload.url, rules)
+    except UrlPolicyError as error:
+        raise HTTPException(status_code=422, detail=str(error)) from error
+
+    job = database.create_job(payload.url.strip(), normalized_url, config, owner_token)
+    return _public_job(job)
+
+
+@app.get("/api/captures")
+async def list_captures(request: Request) -> dict[str, list[dict[str, Any]]]:
+    owner_token = _owner_token(request)
+    return {"items": [_public_job(job) for job in database.list_owned_jobs(owner_token)]}
+
+
+@app.get("/api/captures/{job_id}")
+async def get_capture(job_id: str, request: Request) -> dict[str, Any]:
+    return _require_owned_job(job_id, _owner_token(request))
+
+
+@app.post("/api/captures/{job_id}/cancel")
+async def cancel_capture(job_id: str, request: Request) -> dict[str, Any]:
+    _require_owned_job(job_id, _owner_token(request))
+    job = database.request_cancel(job_id)
+    if not job:
+        raise HTTPException(status_code=404, detail="Capture not found.")
+    return _public_job(job)
+
+
+@app.delete("/api/captures/{job_id}")
+async def delete_capture(job_id: str, request: Request) -> dict[str, Any]:
+    job = _owned_job(job_id, _owner_token(request))
+    if job["state"] in {"queued", "running"}:
+        cancelled = database.request_cancel(job_id)
+        return _public_job(cancelled or job)
+    if job["state"] == "ready":
+        artifact_path = _safe_data_file(str(job.get("artifact_path") or ""), settings.artifacts_dir)
+        report_path = _safe_data_file(str(job.get("report_path") or ""), settings.reports_dir)
+        if artifact_path:
+            artifact_path.unlink(missing_ok=True)
+        if report_path:
+            report_path.unlink(missing_ok=True)
+        database.mark_deleted(job_id)
+        updated = database.get_owned_job(job_id, _owner_token(request))
+        return _public_job(updated or job)
+    return _public_job(job)
+
+
+@app.get("/api/captures/{job_id}/report")
+async def capture_report(job_id: str, request: Request) -> JSONResponse:
+    job = _owned_job(job_id, _owner_token(request))
+    if job["state"] != "ready" or not job.get("report_path"):
+        raise HTTPException(status_code=409, detail="A capture report is not available yet.")
+    report_path = _safe_data_file(str(job["report_path"]), settings.reports_dir)
+    if not report_path or not report_path.is_file():
+        raise HTTPException(status_code=410, detail="The capture report has expired.")
+    return JSONResponse(json.loads(report_path.read_text(encoding="utf-8")))
+
+
+@app.get("/api/captures/{job_id}/download")
+async def download_capture(job_id: str, request: Request) -> FileResponse:
+    job = _owned_job(job_id, _owner_token(request))
+    if job["state"] != "ready" or not job.get("artifact_path"):
+        raise HTTPException(status_code=409, detail="The archive is not available yet.")
+    archive_path = _safe_data_file(str(job["artifact_path"]), settings.artifacts_dir)
+    if not archive_path or not archive_path.is_file():
+        raise HTTPException(status_code=410, detail="The archive has expired.")
+    return FileResponse(
+        archive_path,
+        media_type="application/zip",
+        filename=f"siteharbor-{job_id}.zip",
+        headers={"Cache-Control": "no-store", "X-Content-Type-Options": "nosniff"},
+    )
+
+
+@app.get("/api/captures/{job_id}/events")
+async def capture_events(request: Request, job_id: str) -> StreamingResponse:
+    _owned_job(job_id, _owner_token(request, allow_query=True))
+    after_header = request.headers.get("last-event-id", "0")
+    try:
+        after_id = max(0, int(after_header))
+    except ValueError:
+        after_id = 0
+    return StreamingResponse(
+        _event_stream(request, job_id, after_id),
+        media_type="text/event-stream",
+        headers={"Cache-Control": "no-cache", "X-Accel-Buffering": "no"},
+    )
+
+
+async def _event_stream(request: Request, job_id: str, after_id: int) -> AsyncIterator[str]:
+    terminal_states = {"ready", "failed", "cancelled", "expired"}
+    last_id = after_id
+    while True:
+        if await request.is_disconnected():
+            return
+        events = database.get_events(job_id, last_id)
+        for event in events:
+            last_id = int(event["id"])
+            payload = json.dumps(event["payload"], separators=(",", ":"))
+            yield f"id: {last_id}\nevent: {event['kind']}\ndata: {payload}\n\n"
+
+        job = database.get_job(job_id)
+        if not job or job["state"] in terminal_states:
+            terminal = json.dumps({"state": job["state"] if job else "missing"})
+            yield f"event: terminal\ndata: {terminal}\n\n"
+            return
+        yield ": keepalive\n\n"
+        await asyncio.sleep(0.8)
+
+
+def _validated_config(payload: CaptureRequest) -> dict[str, Any]:
+    max_bytes = payload.max_bytes_mb * 1024 * 1024
+    limits = {
+        "max_pages": (payload.max_pages, settings.max_pages_cap, "page"),
+        "max_depth": (payload.max_depth, settings.max_depth_cap, "depth"),
+        "max_bytes": (max_bytes, settings.max_bytes_cap, "download size"),
+        "max_duration_seconds": (
+            payload.max_duration_seconds,
+            settings.max_duration_seconds_cap,
+            "capture duration",
+        ),
+        "retention_days": (payload.retention_days, settings.max_retention_days, "retention"),
+    }
+    for _, (value, maximum, label) in limits.items():
+        if value > maximum:
+            raise HTTPException(
+                status_code=422, detail=f"Requested {label} exceeds this instance's limit."
+            )
+    if payload.parallel_connections > settings.fetch_concurrency:
+        raise HTTPException(
+            status_code=422,
+            detail="Requested parallel connections exceed this instance's limit.",
+        )
+    if payload.use_proxy and not settings.proxy_url:
+        raise HTTPException(
+            status_code=422,
+            detail="This service does not have an approved proxy route configured.",
+        )
+    skip_file_types = _normalized_file_types(payload.skip_file_types)
+    return {
+        "include_external_assets": payload.include_external_assets,
+        "parallel_connections": payload.parallel_connections,
+        "download_asset_types": list(dict.fromkeys(payload.download_asset_types)),
+        "skip_file_types": skip_file_types,
+        "use_proxy": payload.use_proxy,
+        "max_pages": payload.max_pages,
+        "max_depth": payload.max_depth,
+        "max_bytes": max_bytes,
+        "max_duration_seconds": payload.max_duration_seconds,
+        "retention_days": payload.retention_days,
+    }
+
+
+def _normalized_file_types(values: list[str]) -> list[str]:
+    normalized: list[str] = []
+    for value in values:
+        for item in value.split(","):
+            extension = item.strip().lower().lstrip(".")
+            if not extension:
+                continue
+            if not re.fullmatch(r"[a-z0-9]{1,16}", extension):
+                raise HTTPException(
+                    status_code=422,
+                    detail=(
+                        "File rules must be comma-separated extensions such as pdf, zip, or mp4."
+                    ),
+                )
+            if extension not in normalized:
+                normalized.append(extension)
+    if len(normalized) > 24:
+        raise HTTPException(status_code=422, detail="At most 24 file types can be skipped.")
+    return normalized
+
+
+def _owner_token(request: Request, *, allow_query: bool = False) -> str:
+    value = request.headers.get(OWNER_TOKEN_HEADER)
+    if not value and allow_query:
+        value = request.query_params.get("session")
+    token = (value or "").strip()
+    if not OWNER_TOKEN_RE.fullmatch(token):
+        raise HTTPException(
+            status_code=status.HTTP_401_UNAUTHORIZED,
+            detail="A browser-local session is required to access captures.",
+        )
+    return token
+
+
+def _owned_job(job_id: str, owner_token: str) -> dict[str, Any]:
+    job = database.get_owned_job(job_id, owner_token)
+    if not job:
+        raise HTTPException(status_code=404, detail="Capture not found.")
+    return job
+
+
+def _require_owned_job(job_id: str, owner_token: str) -> dict[str, Any]:
+    return _public_job(_owned_job(job_id, owner_token))
+
+
+def _public_job(job: dict[str, Any]) -> dict[str, Any]:
+    config = job["config"]
+    return {
+        "id": job["id"],
+        "url": job["normalized_url"],
+        "state": job["state"],
+        "phase": job["phase"],
+        "message": job["message"],
+        "error_code": job["error_code"],
+        "error_message": job["error_message"],
+        "warnings": job["warnings"],
+        "created_at": job["created_at"],
+        "updated_at": job["updated_at"],
+        "started_at": job["started_at"],
+        "completed_at": job["completed_at"],
+        "expires_at": job["expires_at"],
+        "cancel_requested": job["cancel_requested"],
+        "entry_point": job["entry_point"],
+        "stats": {
+            "pages_found": job["pages_found"],
+            "pages_fetched": job["pages_fetched"],
+            "assets_found": job["assets_found"],
+            "assets_fetched": job["assets_fetched"],
+            "bytes_downloaded": job["bytes_downloaded"],
+            "files_written": job["files_written"],
+        },
+        "options": {
+            "include_external_assets": config.get("include_external_assets", True),
+            "parallel_connections": config.get(
+                "parallel_connections", settings.fetch_concurrency
+            ),
+            "download_asset_types": config.get(
+                "download_asset_types", list(SUPPORTED_ASSET_TYPES)
+            ),
+            "skip_file_types": config.get("skip_file_types", []),
+            "use_proxy": config.get("use_proxy", False),
+            "max_pages": config["max_pages"],
+            "max_depth": config["max_depth"],
+            "max_bytes": config["max_bytes"],
+            "max_duration_seconds": config["max_duration_seconds"],
+            "retention_days": config["retention_days"],
+        },
+        "archive": {
+            "available": job["state"] == "ready" and bool(job["artifact_path"]),
+            "size_bytes": job["archive_size"],
+            "download_url": f"/api/captures/{job['id']}/download"
+            if job["state"] == "ready"
+            else None,
+            "report_url": f"/api/captures/{job['id']}/report" if job["state"] == "ready" else None,
+        },
+    }
+
+
+def _safe_data_file(path_value: str, root: Path) -> Path | None:
+    path = Path(path_value)
+    try:
+        resolved = path.resolve()
+        if resolved.is_relative_to(root.resolve()):
+            return resolved
+    except OSError:
+        return None
+    return None
+
+
+def run() -> None:
+    uvicorn.run("app.main:app", host=settings.host, port=settings.port, reload=False)
+
+
+if __name__ == "__main__":
+    run()