__init__

2026-09-05 06:07:25 +05:30
commit 07b953fed2
22 changed files with 5899 additions and 0 deletions
@@ -0,0 +1,102 @@
+from __future__ import annotations
+
+import os
+from dataclasses import dataclass
+from pathlib import Path
+
+
+def _as_bool(name: str, default: bool) -> bool:
+    value = os.getenv(name)
+    if value is None:
+        return default
+    return value.strip().lower() in {"1", "true", "yes", "on"}
+
+
+def _as_int(name: str, default: int, minimum: int = 1) -> int:
+    value = os.getenv(name)
+    if value is None:
+        return default
+    try:
+        return max(minimum, int(value))
+    except ValueError:
+        return default
+
+
+@dataclass(frozen=True, slots=True)
+class Settings:
+    project_root: Path
+    data_dir: Path
+    database_path: Path
+    work_dir: Path
+    artifacts_dir: Path
+    reports_dir: Path
+    host: str
+    port: int
+    worker_enabled: bool
+    worker_concurrency: int
+    fetch_concurrency: int
+    queue_poll_seconds: float
+    lease_seconds: int
+    allow_private_networks: bool
+    allow_nonstandard_ports: bool
+    respect_robots: bool
+    default_retention_days: int
+    max_retention_days: int
+    max_pages_cap: int
+    max_depth_cap: int
+    max_bytes_cap: int
+    max_duration_seconds_cap: int
+    max_response_bytes: int
+    max_files: int
+    max_external_hosts: int
+    max_redirects: int
+    user_agent: str
+    proxy_url: str | None
+
+    @classmethod
+    def from_environment(cls) -> Settings:
+        project_root = Path(__file__).resolve().parents[1]
+        data_dir = Path(os.getenv("SITEHARBOR_DATA_DIR", project_root / "data")).expanduser()
+        if not data_dir.is_absolute():
+            data_dir = project_root / data_dir
+
+        return cls(
+            project_root=project_root,
+            data_dir=data_dir,
+            database_path=data_dir / "siteharbor.sqlite3",
+            work_dir=data_dir / "work",
+            artifacts_dir=data_dir / "artifacts",
+            reports_dir=data_dir / "reports",
+            host=os.getenv("SITEHARBOR_HOST", "127.0.0.1"),
+            port=_as_int("SITEHARBOR_PORT", 8787),
+            worker_enabled=_as_bool("SITEHARBOR_WORKER_ENABLED", True),
+            worker_concurrency=_as_int("SITEHARBOR_WORKER_CONCURRENCY", 1),
+            fetch_concurrency=_as_int("SITEHARBOR_FETCH_CONCURRENCY", 6),
+            queue_poll_seconds=max(0.1, float(os.getenv("SITEHARBOR_QUEUE_POLL_SECONDS", "0.75"))),
+            lease_seconds=_as_int("SITEHARBOR_LEASE_SECONDS", 120, minimum=15),
+            allow_private_networks=_as_bool("SITEHARBOR_ALLOW_PRIVATE_NETWORKS", False),
+            allow_nonstandard_ports=_as_bool("SITEHARBOR_ALLOW_NONSTANDARD_PORTS", False),
+            respect_robots=_as_bool("SITEHARBOR_RESPECT_ROBOTS", True),
+            default_retention_days=_as_int("SITEHARBOR_DEFAULT_RETENTION_DAYS", 7),
+            max_retention_days=_as_int("SITEHARBOR_MAX_RETENTION_DAYS", 30),
+            max_pages_cap=_as_int("SITEHARBOR_MAX_PAGES", 1_000),
+            max_depth_cap=_as_int("SITEHARBOR_MAX_DEPTH", 12),
+            max_bytes_cap=_as_int("SITEHARBOR_MAX_BYTES", 1024 * 1024 * 1024),
+            max_duration_seconds_cap=_as_int("SITEHARBOR_MAX_DURATION_SECONDS", 3_600),
+            max_response_bytes=_as_int("SITEHARBOR_MAX_RESPONSE_BYTES", 32 * 1024 * 1024),
+            max_files=_as_int("SITEHARBOR_MAX_FILES", 5_000),
+            max_external_hosts=_as_int("SITEHARBOR_MAX_EXTERNAL_HOSTS", 20),
+            max_redirects=_as_int("SITEHARBOR_MAX_REDIRECTS", 10),
+            user_agent=os.getenv(
+                "SITEHARBOR_USER_AGENT",
+                "SiteHarbor/0.1 (+local standalone capture; respect robots.txt)",
+            ),
+            proxy_url=os.getenv("SITEHARBOR_PROXY_URL", "").strip() or None,
+        )
+
+    def ensure_directories(self) -> None:
+        for directory in (self.data_dir, self.work_dir, self.artifacts_dir, self.reports_dir):
+            directory.mkdir(parents=True, exist_ok=True)
+
+
+settings = Settings.from_environment()