from __future__ import annotations import os from dataclasses import dataclass from pathlib import Path MAX_ARCHIVE_RETENTION_DAYS = 2 def _as_bool(name: str, default: bool) -> bool: value = os.getenv(name) if value is None: return default return value.strip().lower() in {"1", "true", "yes", "on"} def _as_int(name: str, default: int, minimum: int = 1) -> int: value = os.getenv(name) if value is None: return default try: return max(minimum, int(value)) except ValueError: return default @dataclass(frozen=True, slots=True) class Settings: project_root: Path data_dir: Path database_path: Path work_dir: Path artifacts_dir: Path reports_dir: Path host: str port: int worker_enabled: bool worker_concurrency: int fetch_concurrency: int queue_poll_seconds: float lease_seconds: int allow_private_networks: bool allow_nonstandard_ports: bool respect_robots: bool default_retention_days: int max_retention_days: int max_pages_cap: int max_depth_cap: int max_bytes_cap: int max_duration_seconds_cap: int max_response_bytes: int max_files: int max_external_hosts: int max_redirects: int user_agent: str proxy_url: str | None @classmethod def from_environment(cls) -> Settings: project_root = Path(__file__).resolve().parents[1] data_dir = Path(os.getenv("SITEHARBOR_DATA_DIR", project_root / "data")).expanduser() if not data_dir.is_absolute(): data_dir = project_root / data_dir max_retention_days = min( _as_int("SITEHARBOR_MAX_RETENTION_DAYS", MAX_ARCHIVE_RETENTION_DAYS), MAX_ARCHIVE_RETENTION_DAYS, ) default_retention_days = min( _as_int("SITEHARBOR_DEFAULT_RETENTION_DAYS", MAX_ARCHIVE_RETENTION_DAYS), max_retention_days, ) return cls( project_root=project_root, data_dir=data_dir, database_path=data_dir / "siteharbor.sqlite3", work_dir=data_dir / "work", artifacts_dir=data_dir / "artifacts", reports_dir=data_dir / "reports", host=os.getenv("SITEHARBOR_HOST", "127.0.0.1"), port=_as_int("SITEHARBOR_PORT", 8787), worker_enabled=_as_bool("SITEHARBOR_WORKER_ENABLED", True), worker_concurrency=_as_int("SITEHARBOR_WORKER_CONCURRENCY", 1), fetch_concurrency=_as_int("SITEHARBOR_FETCH_CONCURRENCY", 6), queue_poll_seconds=max(0.1, float(os.getenv("SITEHARBOR_QUEUE_POLL_SECONDS", "0.75"))), lease_seconds=_as_int("SITEHARBOR_LEASE_SECONDS", 120, minimum=15), allow_private_networks=_as_bool("SITEHARBOR_ALLOW_PRIVATE_NETWORKS", False), allow_nonstandard_ports=_as_bool("SITEHARBOR_ALLOW_NONSTANDARD_PORTS", False), respect_robots=_as_bool("SITEHARBOR_RESPECT_ROBOTS", True), default_retention_days=default_retention_days, max_retention_days=max_retention_days, max_pages_cap=_as_int("SITEHARBOR_MAX_PAGES", 1_000), max_depth_cap=_as_int("SITEHARBOR_MAX_DEPTH", 12), max_bytes_cap=_as_int("SITEHARBOR_MAX_BYTES", 1024 * 1024 * 1024), max_duration_seconds_cap=_as_int("SITEHARBOR_MAX_DURATION_SECONDS", 3_600), max_response_bytes=_as_int("SITEHARBOR_MAX_RESPONSE_BYTES", 32 * 1024 * 1024), max_files=_as_int("SITEHARBOR_MAX_FILES", 5_000), max_external_hosts=_as_int("SITEHARBOR_MAX_EXTERNAL_HOSTS", 20), max_redirects=_as_int("SITEHARBOR_MAX_REDIRECTS", 10), user_agent=os.getenv( "SITEHARBOR_USER_AGENT", "SiteHarbor/0.1 (+local standalone capture; respect robots.txt)", ), proxy_url=os.getenv("SITEHARBOR_PROXY_URL", "").strip() or None, ) def ensure_directories(self) -> None: for directory in (self.data_dir, self.work_dir, self.artifacts_dir, self.reports_dir): directory.mkdir(parents=True, exist_ok=True) settings = Settings.from_environment()