113 lines
4.1 KiB
Python
113 lines
4.1 KiB
Python
from __future__ import annotations
|
|
|
|
import os
|
|
from dataclasses import dataclass
|
|
from pathlib import Path
|
|
|
|
MAX_ARCHIVE_RETENTION_DAYS = 2
|
|
|
|
|
|
def _as_bool(name: str, default: bool) -> bool:
|
|
value = os.getenv(name)
|
|
if value is None:
|
|
return default
|
|
return value.strip().lower() in {"1", "true", "yes", "on"}
|
|
|
|
|
|
def _as_int(name: str, default: int, minimum: int = 1) -> int:
|
|
value = os.getenv(name)
|
|
if value is None:
|
|
return default
|
|
try:
|
|
return max(minimum, int(value))
|
|
except ValueError:
|
|
return default
|
|
|
|
|
|
@dataclass(frozen=True, slots=True)
|
|
class Settings:
|
|
project_root: Path
|
|
data_dir: Path
|
|
database_path: Path
|
|
work_dir: Path
|
|
artifacts_dir: Path
|
|
reports_dir: Path
|
|
host: str
|
|
port: int
|
|
worker_enabled: bool
|
|
worker_concurrency: int
|
|
fetch_concurrency: int
|
|
queue_poll_seconds: float
|
|
lease_seconds: int
|
|
allow_private_networks: bool
|
|
allow_nonstandard_ports: bool
|
|
respect_robots: bool
|
|
default_retention_days: int
|
|
max_retention_days: int
|
|
max_pages_cap: int
|
|
max_depth_cap: int
|
|
max_bytes_cap: int
|
|
max_duration_seconds_cap: int
|
|
max_response_bytes: int
|
|
max_files: int
|
|
max_external_hosts: int
|
|
max_redirects: int
|
|
user_agent: str
|
|
proxy_url: str | None
|
|
|
|
@classmethod
|
|
def from_environment(cls) -> Settings:
|
|
project_root = Path(__file__).resolve().parents[1]
|
|
data_dir = Path(os.getenv("SITEHARBOR_DATA_DIR", project_root / "data")).expanduser()
|
|
if not data_dir.is_absolute():
|
|
data_dir = project_root / data_dir
|
|
max_retention_days = min(
|
|
_as_int("SITEHARBOR_MAX_RETENTION_DAYS", MAX_ARCHIVE_RETENTION_DAYS),
|
|
MAX_ARCHIVE_RETENTION_DAYS,
|
|
)
|
|
default_retention_days = min(
|
|
_as_int("SITEHARBOR_DEFAULT_RETENTION_DAYS", MAX_ARCHIVE_RETENTION_DAYS),
|
|
max_retention_days,
|
|
)
|
|
|
|
return cls(
|
|
project_root=project_root,
|
|
data_dir=data_dir,
|
|
database_path=data_dir / "siteharbor.sqlite3",
|
|
work_dir=data_dir / "work",
|
|
artifacts_dir=data_dir / "artifacts",
|
|
reports_dir=data_dir / "reports",
|
|
host=os.getenv("SITEHARBOR_HOST", "127.0.0.1"),
|
|
port=_as_int("SITEHARBOR_PORT", 8787),
|
|
worker_enabled=_as_bool("SITEHARBOR_WORKER_ENABLED", True),
|
|
worker_concurrency=_as_int("SITEHARBOR_WORKER_CONCURRENCY", 1),
|
|
fetch_concurrency=_as_int("SITEHARBOR_FETCH_CONCURRENCY", 6),
|
|
queue_poll_seconds=max(0.1, float(os.getenv("SITEHARBOR_QUEUE_POLL_SECONDS", "0.75"))),
|
|
lease_seconds=_as_int("SITEHARBOR_LEASE_SECONDS", 120, minimum=15),
|
|
allow_private_networks=_as_bool("SITEHARBOR_ALLOW_PRIVATE_NETWORKS", False),
|
|
allow_nonstandard_ports=_as_bool("SITEHARBOR_ALLOW_NONSTANDARD_PORTS", False),
|
|
respect_robots=_as_bool("SITEHARBOR_RESPECT_ROBOTS", True),
|
|
default_retention_days=default_retention_days,
|
|
max_retention_days=max_retention_days,
|
|
max_pages_cap=_as_int("SITEHARBOR_MAX_PAGES", 1_000),
|
|
max_depth_cap=_as_int("SITEHARBOR_MAX_DEPTH", 12),
|
|
max_bytes_cap=_as_int("SITEHARBOR_MAX_BYTES", 1024 * 1024 * 1024),
|
|
max_duration_seconds_cap=_as_int("SITEHARBOR_MAX_DURATION_SECONDS", 3_600),
|
|
max_response_bytes=_as_int("SITEHARBOR_MAX_RESPONSE_BYTES", 32 * 1024 * 1024),
|
|
max_files=_as_int("SITEHARBOR_MAX_FILES", 5_000),
|
|
max_external_hosts=_as_int("SITEHARBOR_MAX_EXTERNAL_HOSTS", 20),
|
|
max_redirects=_as_int("SITEHARBOR_MAX_REDIRECTS", 10),
|
|
user_agent=os.getenv(
|
|
"SITEHARBOR_USER_AGENT",
|
|
"SiteHarbor/0.1 (+local standalone capture; respect robots.txt)",
|
|
),
|
|
proxy_url=os.getenv("SITEHARBOR_PROXY_URL", "").strip() or None,
|
|
)
|
|
|
|
def ensure_directories(self) -> None:
|
|
for directory in (self.data_dir, self.work_dir, self.artifacts_dir, self.reports_dir):
|
|
directory.mkdir(parents=True, exist_ok=True)
|
|
|
|
|
|
settings = Settings.from_environment()
|