from __future__ import annotations import argparse import asyncio import json import re from collections.abc import AsyncIterator from contextlib import asynccontextmanager from pathlib import Path from typing import Any import uvicorn from fastapi import FastAPI, HTTPException, Request, Response, status from fastapi.responses import FileResponse, HTMLResponse, JSONResponse, StreamingResponse from fastapi.staticfiles import StaticFiles from fastapi.templating import Jinja2Templates from app.config import settings from app.db import Database from app.models import SUPPORTED_ASSET_TYPES, CaptureRequest from app.scheduler import JobRunner from app.url_policy import NetworkRules, UrlPolicyError, normalize_url settings.ensure_directories() database = Database(settings.database_path) runner = JobRunner(database, settings) templates = Jinja2Templates(directory=str(settings.project_root / "app" / "templates")) OWNER_TOKEN_HEADER = "X-SiteHarbor-Session" OWNER_TOKEN_RE = re.compile(r"[A-Za-z0-9_-]{24,200}") @asynccontextmanager async def lifespan(_: FastAPI) -> AsyncIterator[None]: database.initialize() runner.start() try: yield finally: runner.stop() app = FastAPI( title="SiteHarbor", version="0.1.0", description="Standalone, bounded website capture for offline archives.", lifespan=lifespan, ) app.mount( "/static", StaticFiles(directory=str(settings.project_root / "app" / "static")), name="static" ) @app.middleware("http") async def secure_headers(_: Request, call_next: Any) -> Response: response = await call_next(_) response.headers.setdefault("X-Content-Type-Options", "nosniff") response.headers.setdefault("Referrer-Policy", "same-origin") response.headers.setdefault("X-Frame-Options", "DENY") response.headers.setdefault( "Content-Security-Policy", "default-src 'self'; base-uri 'none'; object-src 'none'; frame-ancestors 'none'; " "form-action 'self'; connect-src 'self'; img-src 'self' data:; style-src 'self'; " "script-src 'self'", ) return response @app.get("/", response_class=HTMLResponse) async def index(request: Request) -> Response: return templates.TemplateResponse( request=request, name="index.html", context={"app_name": "SiteHarbor"}, ) @app.get("/api/health") async def health() -> dict[str, Any]: return { "status": "ok", "worker_enabled": settings.worker_enabled, "worker_concurrency": settings.worker_concurrency, "storage": "local-sqlite", } @app.get("/api/capabilities") async def capabilities() -> dict[str, Any]: return { "defaults": { "max_pages": min(150, settings.max_pages_cap), "max_depth": min(5, settings.max_depth_cap), "max_bytes_mb": min(100, max(1, settings.max_bytes_cap // (1024 * 1024))), "max_duration_seconds": min(300, settings.max_duration_seconds_cap), "retention_days": min(settings.default_retention_days, settings.max_retention_days), "include_external_assets": True, "parallel_connections": min(6, settings.fetch_concurrency), "download_asset_types": list(SUPPORTED_ASSET_TYPES), "skip_file_types": [], "use_proxy": False, }, "limits": { "max_pages": settings.max_pages_cap, "max_depth": settings.max_depth_cap, "max_bytes_mb": settings.max_bytes_cap // (1024 * 1024), "max_duration_seconds": settings.max_duration_seconds_cap, "max_retention_days": settings.max_retention_days, "max_parallel_connections": settings.fetch_concurrency, }, "proxy_available": bool(settings.proxy_url), "local_only_warning": not settings.allow_private_networks, } @app.get("/api/stats") async def public_stats() -> dict[str, int]: return database.public_stats() @app.post("/api/captures", status_code=status.HTTP_202_ACCEPTED) async def create_capture(payload: CaptureRequest, request: Request) -> dict[str, Any]: owner_token = _owner_token(request) config = _validated_config(payload) rules = NetworkRules( allow_private_networks=settings.allow_private_networks, allow_nonstandard_ports=settings.allow_nonstandard_ports, ) try: normalized_url = normalize_url(payload.url, rules) except UrlPolicyError as error: raise HTTPException(status_code=422, detail=str(error)) from error job = database.create_job(payload.url.strip(), normalized_url, config, owner_token) return _public_job(job) @app.get("/api/captures") async def list_captures(request: Request) -> dict[str, list[dict[str, Any]]]: owner_token = _owner_token(request) return {"items": [_public_job(job) for job in database.list_owned_jobs(owner_token)]} @app.get("/api/captures/{job_id}") async def get_capture(job_id: str, request: Request) -> dict[str, Any]: return _require_owned_job(job_id, _owner_token(request)) @app.post("/api/captures/{job_id}/cancel") async def cancel_capture(job_id: str, request: Request) -> dict[str, Any]: _require_owned_job(job_id, _owner_token(request)) job = database.request_cancel(job_id) if not job: raise HTTPException(status_code=404, detail="Capture not found.") return _public_job(job) @app.delete("/api/captures/{job_id}") async def delete_capture(job_id: str, request: Request) -> dict[str, Any]: job = _owned_job(job_id, _owner_token(request)) if job["state"] in {"queued", "running"}: cancelled = database.request_cancel(job_id) return _public_job(cancelled or job) if job["state"] == "ready": artifact_path = _safe_data_file(str(job.get("artifact_path") or ""), settings.artifacts_dir) report_path = _safe_data_file(str(job.get("report_path") or ""), settings.reports_dir) if artifact_path: artifact_path.unlink(missing_ok=True) if report_path: report_path.unlink(missing_ok=True) database.mark_deleted(job_id) updated = database.get_owned_job(job_id, _owner_token(request)) return _public_job(updated or job) return _public_job(job) @app.get("/api/captures/{job_id}/report") async def capture_report(job_id: str, request: Request) -> JSONResponse: job = _owned_job(job_id, _owner_token(request)) if job["state"] != "ready" or not job.get("report_path"): raise HTTPException(status_code=409, detail="A capture report is not available yet.") report_path = _safe_data_file(str(job["report_path"]), settings.reports_dir) if not report_path or not report_path.is_file(): raise HTTPException(status_code=410, detail="The capture report has expired.") return JSONResponse(json.loads(report_path.read_text(encoding="utf-8"))) @app.get("/api/captures/{job_id}/download") async def download_capture(job_id: str, request: Request) -> FileResponse: job = _owned_job(job_id, _owner_token(request)) if job["state"] != "ready" or not job.get("artifact_path"): raise HTTPException(status_code=409, detail="The archive is not available yet.") archive_path = _safe_data_file(str(job["artifact_path"]), settings.artifacts_dir) if not archive_path or not archive_path.is_file(): raise HTTPException(status_code=410, detail="The archive has expired.") return FileResponse( archive_path, media_type="application/zip", filename=f"siteharbor-{job_id}.zip", headers={"Cache-Control": "no-store", "X-Content-Type-Options": "nosniff"}, ) @app.get("/api/captures/{job_id}/events") async def capture_events(request: Request, job_id: str) -> StreamingResponse: _owned_job(job_id, _owner_token(request, allow_query=True)) after_header = request.headers.get("last-event-id", "0") try: after_id = max(0, int(after_header)) except ValueError: after_id = 0 return StreamingResponse( _event_stream(request, job_id, after_id), media_type="text/event-stream", headers={"Cache-Control": "no-cache", "X-Accel-Buffering": "no"}, ) async def _event_stream(request: Request, job_id: str, after_id: int) -> AsyncIterator[str]: terminal_states = {"ready", "failed", "cancelled", "expired"} last_id = after_id while True: if await request.is_disconnected(): return events = database.get_events(job_id, last_id) for event in events: last_id = int(event["id"]) payload = json.dumps(event["payload"], separators=(",", ":")) yield f"id: {last_id}\nevent: {event['kind']}\ndata: {payload}\n\n" job = database.get_job(job_id) if not job or job["state"] in terminal_states: terminal = json.dumps({"state": job["state"] if job else "missing"}) yield f"event: terminal\ndata: {terminal}\n\n" return yield ": keepalive\n\n" await asyncio.sleep(0.8) def _validated_config(payload: CaptureRequest) -> dict[str, Any]: max_bytes = payload.max_bytes_mb * 1024 * 1024 limits = { "max_pages": (payload.max_pages, settings.max_pages_cap, "page"), "max_depth": (payload.max_depth, settings.max_depth_cap, "depth"), "max_bytes": (max_bytes, settings.max_bytes_cap, "download size"), "max_duration_seconds": ( payload.max_duration_seconds, settings.max_duration_seconds_cap, "capture duration", ), "retention_days": (payload.retention_days, settings.max_retention_days, "retention"), } for _, (value, maximum, label) in limits.items(): if value > maximum: raise HTTPException( status_code=422, detail=f"Requested {label} exceeds this instance's limit." ) if payload.parallel_connections > settings.fetch_concurrency: raise HTTPException( status_code=422, detail="Requested parallel connections exceed this instance's limit.", ) if payload.use_proxy and not settings.proxy_url: raise HTTPException( status_code=422, detail="This service does not have an approved proxy route configured.", ) skip_file_types = _normalized_file_types(payload.skip_file_types) return { "include_external_assets": payload.include_external_assets, "parallel_connections": payload.parallel_connections, "download_asset_types": list(dict.fromkeys(payload.download_asset_types)), "skip_file_types": skip_file_types, "use_proxy": payload.use_proxy, "max_pages": payload.max_pages, "max_depth": payload.max_depth, "max_bytes": max_bytes, "max_duration_seconds": payload.max_duration_seconds, "retention_days": payload.retention_days, } def _normalized_file_types(values: list[str]) -> list[str]: normalized: list[str] = [] for value in values: for item in value.split(","): extension = item.strip().lower().lstrip(".") if not extension: continue if not re.fullmatch(r"[a-z0-9]{1,16}", extension): raise HTTPException( status_code=422, detail=( "File rules must be comma-separated extensions such as pdf, zip, or mp4." ), ) if extension not in normalized: normalized.append(extension) if len(normalized) > 24: raise HTTPException(status_code=422, detail="At most 24 file types can be skipped.") return normalized def _owner_token(request: Request, *, allow_query: bool = False) -> str: value = request.headers.get(OWNER_TOKEN_HEADER) if not value and allow_query: value = request.query_params.get("session") token = (value or "").strip() if not OWNER_TOKEN_RE.fullmatch(token): raise HTTPException( status_code=status.HTTP_401_UNAUTHORIZED, detail="A browser-local session is required to access captures.", ) return token def _owned_job(job_id: str, owner_token: str) -> dict[str, Any]: job = database.get_owned_job(job_id, owner_token) if not job: raise HTTPException(status_code=404, detail="Capture not found.") return job def _require_owned_job(job_id: str, owner_token: str) -> dict[str, Any]: return _public_job(_owned_job(job_id, owner_token)) def _public_job(job: dict[str, Any]) -> dict[str, Any]: config = job["config"] return { "id": job["id"], "url": job["normalized_url"], "state": job["state"], "phase": job["phase"], "message": job["message"], "error_code": job["error_code"], "error_message": job["error_message"], "warnings": job["warnings"], "created_at": job["created_at"], "updated_at": job["updated_at"], "started_at": job["started_at"], "completed_at": job["completed_at"], "expires_at": job["expires_at"], "cancel_requested": job["cancel_requested"], "entry_point": job["entry_point"], "stats": { "pages_found": job["pages_found"], "pages_fetched": job["pages_fetched"], "assets_found": job["assets_found"], "assets_fetched": job["assets_fetched"], "bytes_downloaded": job["bytes_downloaded"], "files_written": job["files_written"], }, "options": { "include_external_assets": config.get("include_external_assets", True), "parallel_connections": config.get( "parallel_connections", settings.fetch_concurrency ), "download_asset_types": config.get( "download_asset_types", list(SUPPORTED_ASSET_TYPES) ), "skip_file_types": config.get("skip_file_types", []), "use_proxy": config.get("use_proxy", False), "max_pages": config["max_pages"], "max_depth": config["max_depth"], "max_bytes": config["max_bytes"], "max_duration_seconds": config["max_duration_seconds"], "retention_days": config["retention_days"], }, "archive": { "available": job["state"] == "ready" and bool(job["artifact_path"]), "size_bytes": job["archive_size"], "download_url": f"/api/captures/{job['id']}/download" if job["state"] == "ready" else None, "report_url": f"/api/captures/{job['id']}/report" if job["state"] == "ready" else None, }, } def _safe_data_file(path_value: str, root: Path) -> Path | None: path = Path(path_value) try: resolved = path.resolve() if resolved.is_relative_to(root.resolve()): return resolved except OSError: return None return None def _positive_integer(value: str) -> int: try: number = int(value) except ValueError as error: raise argparse.ArgumentTypeError("must be a whole number") from error if number < 1: raise argparse.ArgumentTypeError("must be at least 1") return number def _port_number(value: str) -> int: port = _positive_integer(value) if port > 65_535: raise argparse.ArgumentTypeError("must be between 1 and 65535") return port def _parse_server_arguments(argv: list[str] | None = None) -> argparse.Namespace: parser = argparse.ArgumentParser(description="Run the SiteHarbor FastAPI server with Uvicorn.") parser.add_argument( "--port", type=_port_number, default=settings.port, metavar="PORT", help=f"bind port (default: SITEHARBOR_PORT or {settings.port})", ) parser.add_argument( "--workers", type=_positive_integer, default=1, metavar="WORKERS", help="number of Uvicorn worker processes (default: 1)", ) return parser.parse_args(argv) def run(argv: list[str] | None = None) -> None: arguments = _parse_server_arguments(argv) uvicorn.run( "app.main:app", host=settings.host, port=arguments.port, workers=arguments.workers, reload=False, ) if __name__ == "__main__": run()