kstyagi23
2059c275c2
- Implement job logging functionality to track progress and events for each job. - Add a new section in the job card to display session logs with detailed entries. - Update job statistics to include pages fetched, pages found, and assets processed. - Modify the UI theme and text for better clarity and aesthetics. - Adjust tests to validate new logging features and ensure proper event handling.
445 lines
16 KiB
Python
445 lines
16 KiB
Python
from __future__ import annotations
|
|
|
|
import argparse
|
|
import asyncio
|
|
import json
|
|
import re
|
|
from collections.abc import AsyncIterator
|
|
from contextlib import asynccontextmanager
|
|
from pathlib import Path
|
|
from typing import Any
|
|
|
|
import uvicorn
|
|
from fastapi import FastAPI, HTTPException, Request, Response, status
|
|
from fastapi.responses import FileResponse, HTMLResponse, JSONResponse, StreamingResponse
|
|
from fastapi.staticfiles import StaticFiles
|
|
from fastapi.templating import Jinja2Templates
|
|
|
|
from app.config import settings
|
|
from app.db import Database
|
|
from app.models import SUPPORTED_ASSET_TYPES, CaptureRequest
|
|
from app.scheduler import JobRunner
|
|
from app.url_policy import NetworkRules, UrlPolicyError, normalize_url
|
|
|
|
settings.ensure_directories()
|
|
database = Database(settings.database_path)
|
|
runner = JobRunner(database, settings)
|
|
templates = Jinja2Templates(directory=str(settings.project_root / "app" / "templates"))
|
|
OWNER_TOKEN_HEADER = "X-SiteHarbor-Session"
|
|
OWNER_TOKEN_RE = re.compile(r"[A-Za-z0-9_-]{24,200}")
|
|
|
|
|
|
@asynccontextmanager
|
|
async def lifespan(_: FastAPI) -> AsyncIterator[None]:
|
|
database.initialize()
|
|
runner.start()
|
|
try:
|
|
yield
|
|
finally:
|
|
runner.stop()
|
|
|
|
|
|
app = FastAPI(
|
|
title="SiteHarbor",
|
|
version="0.1.0",
|
|
description="Standalone, bounded website capture for offline archives.",
|
|
lifespan=lifespan,
|
|
)
|
|
app.mount(
|
|
"/static", StaticFiles(directory=str(settings.project_root / "app" / "static")), name="static"
|
|
)
|
|
|
|
|
|
@app.middleware("http")
|
|
async def secure_headers(_: Request, call_next: Any) -> Response:
|
|
response = await call_next(_)
|
|
response.headers.setdefault("X-Content-Type-Options", "nosniff")
|
|
response.headers.setdefault("Referrer-Policy", "same-origin")
|
|
response.headers.setdefault("X-Frame-Options", "DENY")
|
|
response.headers.setdefault(
|
|
"Content-Security-Policy",
|
|
"default-src 'self'; base-uri 'none'; object-src 'none'; frame-ancestors 'none'; "
|
|
"form-action 'self'; connect-src 'self'; img-src 'self' data:; "
|
|
"style-src 'self' https://fonts.googleapis.com; font-src 'self' https://fonts.gstatic.com; "
|
|
"script-src 'self'",
|
|
)
|
|
return response
|
|
|
|
|
|
@app.get("/", response_class=HTMLResponse)
|
|
def index(request: Request) -> Response:
|
|
return templates.TemplateResponse(
|
|
request=request,
|
|
name="index.html",
|
|
context={"app_name": "SiteHarbor"},
|
|
)
|
|
|
|
|
|
@app.get("/api/health")
|
|
def health() -> dict[str, Any]:
|
|
return {
|
|
"status": "ok",
|
|
"worker_enabled": settings.worker_enabled,
|
|
"worker_concurrency": settings.worker_concurrency,
|
|
"storage": "local-sqlite",
|
|
}
|
|
|
|
|
|
@app.get("/api/capabilities")
|
|
def capabilities() -> dict[str, Any]:
|
|
return {
|
|
"defaults": {
|
|
"max_pages": min(150, settings.max_pages_cap),
|
|
"max_depth": min(5, settings.max_depth_cap),
|
|
"max_bytes_mb": min(100, max(1, settings.max_bytes_cap // (1024 * 1024))),
|
|
"max_duration_seconds": min(300, settings.max_duration_seconds_cap),
|
|
"retention_days": min(settings.default_retention_days, settings.max_retention_days),
|
|
"include_external_assets": True,
|
|
"parallel_connections": min(6, settings.fetch_concurrency),
|
|
"download_asset_types": list(SUPPORTED_ASSET_TYPES),
|
|
"skip_file_types": [],
|
|
"use_proxy": False,
|
|
},
|
|
"limits": {
|
|
"max_pages": settings.max_pages_cap,
|
|
"max_depth": settings.max_depth_cap,
|
|
"max_bytes_mb": settings.max_bytes_cap // (1024 * 1024),
|
|
"max_duration_seconds": settings.max_duration_seconds_cap,
|
|
"max_retention_days": settings.max_retention_days,
|
|
"max_parallel_connections": settings.fetch_concurrency,
|
|
},
|
|
"proxy_available": bool(settings.proxy_url),
|
|
"local_only_warning": not settings.allow_private_networks,
|
|
}
|
|
|
|
|
|
@app.get("/api/stats")
|
|
def public_stats() -> dict[str, int]:
|
|
return database.public_stats()
|
|
|
|
|
|
@app.post("/api/captures", status_code=status.HTTP_202_ACCEPTED)
|
|
def create_capture(payload: CaptureRequest, request: Request) -> dict[str, Any]:
|
|
owner_token = _owner_token(request)
|
|
config = _validated_config(payload)
|
|
rules = NetworkRules(
|
|
allow_private_networks=settings.allow_private_networks,
|
|
allow_nonstandard_ports=settings.allow_nonstandard_ports,
|
|
)
|
|
try:
|
|
normalized_url = normalize_url(payload.url, rules)
|
|
except UrlPolicyError as error:
|
|
raise HTTPException(status_code=422, detail=str(error)) from error
|
|
|
|
job = database.create_job(payload.url.strip(), normalized_url, config, owner_token)
|
|
return _public_job(job)
|
|
|
|
|
|
@app.get("/api/captures")
|
|
def list_captures(request: Request) -> dict[str, list[dict[str, Any]]]:
|
|
owner_token = _owner_token(request)
|
|
return {"items": [_public_job(job) for job in database.list_owned_jobs(owner_token)]}
|
|
|
|
|
|
@app.get("/api/captures/{job_id}")
|
|
def get_capture(job_id: str, request: Request) -> dict[str, Any]:
|
|
return _require_owned_job(job_id, _owner_token(request))
|
|
|
|
|
|
@app.post("/api/captures/{job_id}/cancel")
|
|
def cancel_capture(job_id: str, request: Request) -> dict[str, Any]:
|
|
_require_owned_job(job_id, _owner_token(request))
|
|
job = database.request_cancel(job_id)
|
|
if not job:
|
|
raise HTTPException(status_code=404, detail="Capture not found.")
|
|
return _public_job(job)
|
|
|
|
|
|
@app.delete("/api/captures/{job_id}")
|
|
def delete_capture(job_id: str, request: Request) -> dict[str, Any]:
|
|
job = _owned_job(job_id, _owner_token(request))
|
|
if job["state"] in {"queued", "running"}:
|
|
cancelled = database.request_cancel(job_id)
|
|
return _public_job(cancelled or job)
|
|
if job["state"] == "ready":
|
|
artifact_path = _safe_data_file(str(job.get("artifact_path") or ""), settings.artifacts_dir)
|
|
report_path = _safe_data_file(str(job.get("report_path") or ""), settings.reports_dir)
|
|
if artifact_path:
|
|
artifact_path.unlink(missing_ok=True)
|
|
if report_path:
|
|
report_path.unlink(missing_ok=True)
|
|
database.mark_deleted(job_id)
|
|
updated = database.get_owned_job(job_id, _owner_token(request))
|
|
return _public_job(updated or job)
|
|
return _public_job(job)
|
|
|
|
|
|
@app.get("/api/captures/{job_id}/report")
|
|
def capture_report(job_id: str, request: Request) -> JSONResponse:
|
|
job = _owned_job(job_id, _owner_token(request))
|
|
if job["state"] != "ready" or not job.get("report_path"):
|
|
raise HTTPException(status_code=409, detail="A capture report is not available yet.")
|
|
report_path = _safe_data_file(str(job["report_path"]), settings.reports_dir)
|
|
if not report_path or not report_path.is_file():
|
|
raise HTTPException(status_code=410, detail="The capture report has expired.")
|
|
return JSONResponse(json.loads(report_path.read_text(encoding="utf-8")))
|
|
|
|
|
|
@app.get("/api/captures/{job_id}/download")
|
|
def download_capture(job_id: str, request: Request) -> FileResponse:
|
|
job = _owned_job(job_id, _owner_token(request))
|
|
if job["state"] != "ready" or not job.get("artifact_path"):
|
|
raise HTTPException(status_code=409, detail="The archive is not available yet.")
|
|
archive_path = _safe_data_file(str(job["artifact_path"]), settings.artifacts_dir)
|
|
if not archive_path or not archive_path.is_file():
|
|
raise HTTPException(status_code=410, detail="The archive has expired.")
|
|
return FileResponse(
|
|
archive_path,
|
|
media_type="application/zip",
|
|
filename=f"siteharbor-{job_id}.zip",
|
|
headers={"Cache-Control": "no-store", "X-Content-Type-Options": "nosniff"},
|
|
)
|
|
|
|
|
|
@app.get("/api/captures/{job_id}/events")
|
|
def capture_events(request: Request, job_id: str) -> StreamingResponse:
|
|
_owned_job(job_id, _owner_token(request, allow_query=True))
|
|
after_header = request.headers.get("last-event-id", "0")
|
|
try:
|
|
after_id = max(0, int(after_header))
|
|
except ValueError:
|
|
after_id = 0
|
|
return StreamingResponse(
|
|
_event_stream(request, job_id, after_id),
|
|
media_type="text/event-stream",
|
|
headers={"Cache-Control": "no-cache", "X-Accel-Buffering": "no"},
|
|
)
|
|
|
|
|
|
async def _event_stream(request: Request, job_id: str, after_id: int) -> AsyncIterator[str]:
|
|
terminal_states = {"ready", "failed", "cancelled", "expired"}
|
|
last_id = after_id
|
|
while True:
|
|
if await request.is_disconnected():
|
|
return
|
|
events = await asyncio.to_thread(database.get_events, job_id, last_id)
|
|
for event in events:
|
|
last_id = int(event["id"])
|
|
payload = json.dumps(
|
|
{**event["payload"], "logged_at": event["created_at"]}, separators=(",", ":")
|
|
)
|
|
yield f"id: {last_id}\nevent: {event['kind']}\ndata: {payload}\n\n"
|
|
if events:
|
|
continue
|
|
|
|
job = await asyncio.to_thread(database.get_job, job_id)
|
|
if not job or job["state"] in terminal_states:
|
|
terminal = json.dumps({"state": job["state"] if job else "missing"})
|
|
yield f"event: terminal\ndata: {terminal}\n\n"
|
|
return
|
|
yield ": keepalive\n\n"
|
|
await asyncio.sleep(0.8)
|
|
|
|
|
|
def _validated_config(payload: CaptureRequest) -> dict[str, Any]:
|
|
max_bytes = payload.max_bytes_mb * 1024 * 1024
|
|
limits = {
|
|
"max_pages": (payload.max_pages, settings.max_pages_cap, "page"),
|
|
"max_depth": (payload.max_depth, settings.max_depth_cap, "depth"),
|
|
"max_bytes": (max_bytes, settings.max_bytes_cap, "download size"),
|
|
"max_duration_seconds": (
|
|
payload.max_duration_seconds,
|
|
settings.max_duration_seconds_cap,
|
|
"capture duration",
|
|
),
|
|
"retention_days": (payload.retention_days, settings.max_retention_days, "retention"),
|
|
}
|
|
for _, (value, maximum, label) in limits.items():
|
|
if value > maximum:
|
|
raise HTTPException(
|
|
status_code=422, detail=f"Requested {label} exceeds this instance's limit."
|
|
)
|
|
if payload.parallel_connections > settings.fetch_concurrency:
|
|
raise HTTPException(
|
|
status_code=422,
|
|
detail="Requested parallel connections exceed this instance's limit.",
|
|
)
|
|
if payload.use_proxy and not settings.proxy_url:
|
|
raise HTTPException(
|
|
status_code=422,
|
|
detail="This service does not have an approved proxy route configured.",
|
|
)
|
|
skip_file_types = _normalized_file_types(payload.skip_file_types)
|
|
return {
|
|
"include_external_assets": payload.include_external_assets,
|
|
"parallel_connections": payload.parallel_connections,
|
|
"download_asset_types": list(dict.fromkeys(payload.download_asset_types)),
|
|
"skip_file_types": skip_file_types,
|
|
"use_proxy": payload.use_proxy,
|
|
"max_pages": payload.max_pages,
|
|
"max_depth": payload.max_depth,
|
|
"max_bytes": max_bytes,
|
|
"max_duration_seconds": payload.max_duration_seconds,
|
|
"retention_days": payload.retention_days,
|
|
}
|
|
|
|
|
|
def _normalized_file_types(values: list[str]) -> list[str]:
|
|
normalized: list[str] = []
|
|
for value in values:
|
|
for item in value.split(","):
|
|
extension = item.strip().lower().lstrip(".")
|
|
if not extension:
|
|
continue
|
|
if not re.fullmatch(r"[a-z0-9]{1,16}", extension):
|
|
raise HTTPException(
|
|
status_code=422,
|
|
detail=(
|
|
"File rules must be comma-separated extensions such as pdf, zip, or mp4."
|
|
),
|
|
)
|
|
if extension not in normalized:
|
|
normalized.append(extension)
|
|
if len(normalized) > 24:
|
|
raise HTTPException(status_code=422, detail="At most 24 file types can be skipped.")
|
|
return normalized
|
|
|
|
|
|
def _owner_token(request: Request, *, allow_query: bool = False) -> str:
|
|
value = request.headers.get(OWNER_TOKEN_HEADER)
|
|
if not value and allow_query:
|
|
value = request.query_params.get("session")
|
|
token = (value or "").strip()
|
|
if not OWNER_TOKEN_RE.fullmatch(token):
|
|
raise HTTPException(
|
|
status_code=status.HTTP_401_UNAUTHORIZED,
|
|
detail="A browser-local session is required to access captures.",
|
|
)
|
|
return token
|
|
|
|
|
|
def _owned_job(job_id: str, owner_token: str) -> dict[str, Any]:
|
|
job = database.get_owned_job(job_id, owner_token)
|
|
if not job:
|
|
raise HTTPException(status_code=404, detail="Capture not found.")
|
|
return job
|
|
|
|
|
|
def _require_owned_job(job_id: str, owner_token: str) -> dict[str, Any]:
|
|
return _public_job(_owned_job(job_id, owner_token))
|
|
|
|
|
|
def _public_job(job: dict[str, Any]) -> dict[str, Any]:
|
|
config = job["config"]
|
|
return {
|
|
"id": job["id"],
|
|
"url": job["normalized_url"],
|
|
"state": job["state"],
|
|
"phase": job["phase"],
|
|
"message": job["message"],
|
|
"error_code": job["error_code"],
|
|
"error_message": job["error_message"],
|
|
"warnings": job["warnings"],
|
|
"created_at": job["created_at"],
|
|
"updated_at": job["updated_at"],
|
|
"started_at": job["started_at"],
|
|
"completed_at": job["completed_at"],
|
|
"expires_at": job["expires_at"],
|
|
"cancel_requested": job["cancel_requested"],
|
|
"entry_point": job["entry_point"],
|
|
"stats": {
|
|
"pages_found": job["pages_found"],
|
|
"pages_fetched": job["pages_fetched"],
|
|
"assets_found": job["assets_found"],
|
|
"assets_fetched": job["assets_fetched"],
|
|
"bytes_downloaded": job["bytes_downloaded"],
|
|
"files_written": job["files_written"],
|
|
},
|
|
"options": {
|
|
"include_external_assets": config.get("include_external_assets", True),
|
|
"parallel_connections": config.get(
|
|
"parallel_connections", settings.fetch_concurrency
|
|
),
|
|
"download_asset_types": config.get(
|
|
"download_asset_types", list(SUPPORTED_ASSET_TYPES)
|
|
),
|
|
"skip_file_types": config.get("skip_file_types", []),
|
|
"use_proxy": config.get("use_proxy", False),
|
|
"max_pages": config["max_pages"],
|
|
"max_depth": config["max_depth"],
|
|
"max_bytes": config["max_bytes"],
|
|
"max_duration_seconds": config["max_duration_seconds"],
|
|
"retention_days": config["retention_days"],
|
|
},
|
|
"archive": {
|
|
"available": job["state"] == "ready" and bool(job["artifact_path"]),
|
|
"size_bytes": job["archive_size"],
|
|
"download_url": f"/api/captures/{job['id']}/download"
|
|
if job["state"] == "ready"
|
|
else None,
|
|
"report_url": f"/api/captures/{job['id']}/report" if job["state"] == "ready" else None,
|
|
},
|
|
}
|
|
|
|
|
|
def _safe_data_file(path_value: str, root: Path) -> Path | None:
|
|
path = Path(path_value)
|
|
try:
|
|
resolved = path.resolve()
|
|
if resolved.is_relative_to(root.resolve()):
|
|
return resolved
|
|
except OSError:
|
|
return None
|
|
return None
|
|
|
|
|
|
def _positive_integer(value: str) -> int:
|
|
try:
|
|
number = int(value)
|
|
except ValueError as error:
|
|
raise argparse.ArgumentTypeError("must be a whole number") from error
|
|
if number < 1:
|
|
raise argparse.ArgumentTypeError("must be at least 1")
|
|
return number
|
|
|
|
|
|
def _port_number(value: str) -> int:
|
|
port = _positive_integer(value)
|
|
if port > 65_535:
|
|
raise argparse.ArgumentTypeError("must be between 1 and 65535")
|
|
return port
|
|
|
|
|
|
def _parse_server_arguments(argv: list[str] | None = None) -> argparse.Namespace:
|
|
parser = argparse.ArgumentParser(description="Run the SiteHarbor FastAPI server with Uvicorn.")
|
|
parser.add_argument(
|
|
"--port",
|
|
type=_port_number,
|
|
default=settings.port,
|
|
metavar="PORT",
|
|
help=f"bind port (default: SITEHARBOR_PORT or {settings.port})",
|
|
)
|
|
parser.add_argument(
|
|
"--workers",
|
|
type=_positive_integer,
|
|
default=1,
|
|
metavar="WORKERS",
|
|
help="number of Uvicorn worker processes (default: 1)",
|
|
)
|
|
return parser.parse_args(argv)
|
|
|
|
|
|
def run(argv: list[str] | None = None) -> None:
|
|
arguments = _parse_server_arguments(argv)
|
|
uvicorn.run(
|
|
"app.main:app",
|
|
host=settings.host,
|
|
port=arguments.port,
|
|
workers=arguments.workers,
|
|
reload=False,
|
|
)
|
|
|
|
|
|
if __name__ == "__main__":
|
|
run()
|