From 9c1a301c1a7df3f5da6fa22d16defc4fe8d1acbe Mon Sep 17 00:00:00 2001 From: kstyagi23 Date: Sat, 5 Sep 2026 06:37:35 +0530 Subject: [PATCH] update retention policy to a maximum of 2 days for completed archives and adjust related configurations --- README.md | 5 +- app/config.py | 14 +- app/db.py | 19 ++- app/models.py | 6 +- app/static/app.css | 326 ++++++++++++++++++++++++++++++++------- app/static/app.js | 2 +- app/templates/index.html | 104 ++++++++++--- tests/test_database.py | 7 +- tests/test_main.py | 17 +- tests/test_scheduler.py | 69 +++++++++ 10 files changed, 477 insertions(+), 92 deletions(-) create mode 100644 tests/test_scheduler.py diff --git a/README.md b/README.md index e2e26d0..bd7c20e 100644 --- a/README.md +++ b/README.md @@ -60,6 +60,9 @@ python -m uvicorn app.main:app --host 127.0.0.1 --port 8787 --workers 2 The default bind address is loopback. Do not expose SiteHarbor directly to the public Internet without authentication, a network egress policy, rate limits, and operational controls. +Completed ZIP archives and their reports are retained for at most two days after a capture finishes. +The maintenance worker removes expired files and marks the capture as expired. + ## Configuration All configuration uses environment variables prefixed with `SITEHARBOR_`. @@ -75,7 +78,7 @@ All configuration uses environment variables prefixed with `SITEHARBOR_`. | `SITEHARBOR_ALLOW_PRIVATE_NETWORKS` | `false` | Development-only override that permits loopback and private hosts. | | `SITEHARBOR_ALLOW_NONSTANDARD_PORTS` | `false` | Development-only override for ports other than 80 and 443. | | `SITEHARBOR_RESPECT_ROBOTS` | `true` | Respect robots rules during capture. | -| `SITEHARBOR_DEFAULT_RETENTION_DAYS` | `7` | Default retention period for completed artifacts. | +| `SITEHARBOR_DEFAULT_RETENTION_DAYS` | `2` | Default retention period for completed artifacts; values cannot exceed two days. | | `SITEHARBOR_PROXY_URL` | unset | Operator-approved HTTP(S) proxy route. Visitors can only opt into this configured route; they cannot supply proxy endpoints. | For a local fixture site on `localhost:8000`, use both development overrides: diff --git a/app/config.py b/app/config.py index f05f1c4..33b33e6 100644 --- a/app/config.py +++ b/app/config.py @@ -4,6 +4,8 @@ import os from dataclasses import dataclass from pathlib import Path +MAX_ARCHIVE_RETENTION_DAYS = 2 + def _as_bool(name: str, default: bool) -> bool: value = os.getenv(name) @@ -59,6 +61,14 @@ class Settings: data_dir = Path(os.getenv("SITEHARBOR_DATA_DIR", project_root / "data")).expanduser() if not data_dir.is_absolute(): data_dir = project_root / data_dir + max_retention_days = min( + _as_int("SITEHARBOR_MAX_RETENTION_DAYS", MAX_ARCHIVE_RETENTION_DAYS), + MAX_ARCHIVE_RETENTION_DAYS, + ) + default_retention_days = min( + _as_int("SITEHARBOR_DEFAULT_RETENTION_DAYS", MAX_ARCHIVE_RETENTION_DAYS), + max_retention_days, + ) return cls( project_root=project_root, @@ -77,8 +87,8 @@ class Settings: allow_private_networks=_as_bool("SITEHARBOR_ALLOW_PRIVATE_NETWORKS", False), allow_nonstandard_ports=_as_bool("SITEHARBOR_ALLOW_NONSTANDARD_PORTS", False), respect_robots=_as_bool("SITEHARBOR_RESPECT_ROBOTS", True), - default_retention_days=_as_int("SITEHARBOR_DEFAULT_RETENTION_DAYS", 7), - max_retention_days=_as_int("SITEHARBOR_MAX_RETENTION_DAYS", 30), + default_retention_days=default_retention_days, + max_retention_days=max_retention_days, max_pages_cap=_as_int("SITEHARBOR_MAX_PAGES", 1_000), max_depth_cap=_as_int("SITEHARBOR_MAX_DEPTH", 12), max_bytes_cap=_as_int("SITEHARBOR_MAX_BYTES", 1024 * 1024 * 1024), diff --git a/app/db.py b/app/db.py index 0ae8562..41972e3 100644 --- a/app/db.py +++ b/app/db.py @@ -9,6 +9,8 @@ from datetime import UTC, datetime, timedelta from pathlib import Path from typing import Any +from app.config import MAX_ARCHIVE_RETENTION_DAYS + def utcnow() -> datetime: return datetime.now(UTC) @@ -494,6 +496,7 @@ class Database: ) -> bool: now_datetime = utcnow() now = timestamp(now_datetime) + retention_days = min(retention_days, MAX_ARCHIVE_RETENTION_DAYS) with self.connection() as connection: cursor = connection.execute( """ @@ -644,10 +647,22 @@ class Database: ) def expired_ready_jobs(self) -> list[dict[str, Any]]: - now = timestamp() + now_datetime = utcnow() + now = timestamp(now_datetime) + maximum_completion_age = timestamp( + now_datetime - timedelta(days=MAX_ARCHIVE_RETENTION_DAYS) + ) with self.connection() as connection: rows = connection.execute( - "SELECT * FROM jobs WHERE state = 'ready' AND expires_at <= ?", (now,) + """ + SELECT * FROM jobs + WHERE state = 'ready' + AND ( + expires_at <= ? + OR COALESCE(completed_at, created_at) <= ? + ) + """, + (now, maximum_completion_age), ).fetchall() return [self._row_to_job(row) for row in rows] diff --git a/app/models.py b/app/models.py index 366fa42..46d2807 100644 --- a/app/models.py +++ b/app/models.py @@ -4,6 +4,8 @@ from typing import Literal from pydantic import BaseModel, Field +from app.config import MAX_ARCHIVE_RETENTION_DAYS + AssetType = Literal["styles", "scripts", "images", "fonts", "media", "other"] SUPPORTED_ASSET_TYPES: tuple[AssetType, ...] = ( "styles", @@ -28,4 +30,6 @@ class CaptureRequest(BaseModel): max_depth: int = Field(default=5, ge=0) max_bytes_mb: int = Field(default=100, ge=1) max_duration_seconds: int = Field(default=300, ge=10) - retention_days: int = Field(default=7, ge=1) + retention_days: int = Field( + default=MAX_ARCHIVE_RETENTION_DAYS, ge=1, le=MAX_ARCHIVE_RETENTION_DAYS + ) diff --git a/app/static/app.css b/app/static/app.css index ddbf301..f5188f8 100644 --- a/app/static/app.css +++ b/app/static/app.css @@ -77,75 +77,156 @@ a:focus-visible { margin: 0 auto; } -.topbar { - display: flex; +.site-header { + position: relative; + display: grid; + grid-template-columns: minmax(220px, 1fr) auto minmax(180px, 1fr); align-items: center; - justify-content: space-between; - min-height: 92px; + gap: 28px; + min-height: 108px; border-bottom: 1px solid rgba(255, 255, 255, 0.1); } +.site-header::after { + position: absolute; + bottom: -1px; + left: 0; + width: 118px; + height: 2px; + content: ""; + background: var(--mint); +} + .brand { - display: inline-flex; + display: inline-grid; + grid-template-columns: auto 1fr; align-items: center; - gap: 11px; + justify-self: start; + gap: 12px; color: var(--ink); - font-size: 1.06rem; - font-weight: 780; - letter-spacing: -0.045em; text-decoration: none; } .brand-mark { position: relative; - display: block; - width: 24px; - height: 24px; - transform: rotate(45deg); - border: 1px solid rgba(152, 242, 187, 0.74); - border-radius: 7px; - background: rgba(79, 212, 138, 0.1); + display: grid; + width: 38px; + height: 38px; + place-items: center; + overflow: hidden; + border: 1px solid rgba(152, 242, 187, 0.48); + border-radius: 11px; + background: + linear-gradient(135deg, rgba(142, 234, 184, 0.22), rgba(6, 59, 44, 0.45)), + #0a2118; + box-shadow: inset 0 1px 0 rgba(238, 255, 244, 0.16); +} + +.brand-mark::before { + position: absolute; + width: 23px; + height: 23px; + border: 1px solid rgba(197, 255, 218, 0.28); + border-radius: 50%; + content: ""; } .brand-mark i { position: absolute; display: block; - width: 5px; - height: 5px; + z-index: 1; + width: 4px; + height: 4px; border-radius: 50%; background: var(--mint); + box-shadow: 0 0 8px rgba(142, 234, 184, 0.7); } .brand-mark i:nth-child(1) { - top: 4px; - left: 4px; + top: 10px; + left: 10px; } .brand-mark i:nth-child(2) { - right: 4px; - bottom: 4px; + right: 10px; + bottom: 10px; opacity: 0.75; } .brand-mark i:nth-child(3) { - top: 4px; - right: 4px; + top: 10px; + right: 10px; width: 3px; height: 3px; opacity: 0.55; } -.topbar-meta, +.brand-copy { + display: grid; + gap: 3px; +} + +.brand-kicker, +.service-panel-label, +.footer-kicker, +.footer-index { + color: var(--quiet); + font-size: 0.58rem; + font-weight: 780; + letter-spacing: 0.14em; +} + +.brand strong { + color: var(--paper); + font-size: 1.16rem; + font-weight: 760; + letter-spacing: -0.05em; +} + +.site-nav { + display: flex; + align-items: center; + gap: 4px; + padding: 5px; + border: 1px solid rgba(235, 246, 235, 0.1); + border-radius: 999px; + background: rgba(1, 4, 2, 0.3); +} + +.site-nav a { + padding: 7px 10px; + border-radius: 999px; + color: #aeb6ad; + font-size: 0.68rem; + font-weight: 650; + text-decoration: none; + transition: color 160ms ease, background 160ms ease; +} + +.site-nav a:hover { + color: var(--mint-bright); + background: rgba(142, 234, 184, 0.1); +} + +.service-panel { + display: grid; + justify-items: end; + gap: 6px; +} + .service-status { display: flex; align-items: center; + gap: 8px; + color: #c3cbc2; + font-size: 0.72rem; } -.topbar-meta { - gap: 20px; +.service-status[data-online="false"] > i { + background: var(--danger); + box-shadow: 0 0 0 4px rgba(240, 151, 142, 0.08), 0 0 13px rgba(240, 151, 142, 0.55); } -.public-label, .eyebrow { color: var(--mint); font-size: 0.65rem; @@ -153,16 +234,6 @@ a:focus-visible { letter-spacing: 0.16em; } -.public-label { - color: var(--quiet); -} - -.service-status { - gap: 8px; - color: #b7bdb7; - font-size: 0.72rem; -} - .service-status > i, .pulse-dot { display: inline-block; @@ -1129,19 +1200,89 @@ h3 { opacity: 0.56; } -.footer-note { - display: flex; - justify-content: space-between; - gap: 22px; - padding: 24px 0 31px; +.site-footer { + margin-top: 7px; border-top: 1px solid rgba(255, 255, 255, 0.1); - color: #7c827b; - font-size: 0.68rem; - line-height: 1.48; } -.footer-note span:last-child { - max-width: 535px; +.footer-top { + display: grid; + grid-template-columns: minmax(0, 0.86fr) minmax(0, 1.54fr); + gap: clamp(34px, 6vw, 88px); + padding: 43px 0 37px; +} + +.footer-kicker { + margin: 0 0 13px; + color: var(--mint); +} + +.footer-intro h2 { + max-width: 390px; + margin: 0 0 14px; + font-size: clamp(1.55rem, 2.6vw, 2.2rem); + line-height: 1.03; +} + +.footer-intro > p:last-child { + max-width: 420px; + margin: 0; + color: #99a19a; + font-size: 0.78rem; + line-height: 1.62; +} + +.footer-points { + display: grid; + grid-template-columns: repeat(3, minmax(0, 1fr)); +} + +.footer-point { + min-width: 0; + padding: 0 20px; +} + +.footer-point:first-child { + padding-left: 0; +} + +.footer-point + .footer-point { + border-left: 1px solid var(--line); +} + +.footer-index { + display: block; + margin-bottom: 15px; + color: #768177; + font-family: ui-monospace, SFMono-Regular, Menlo, Consolas, monospace; +} + +.footer-point h3 { + margin: 0 0 8px; + color: #e4e9e2; + font-size: 0.87rem; + letter-spacing: -0.025em; +} + +.footer-point p { + margin: 0; + color: #838b83; + font-size: 0.7rem; + line-height: 1.58; +} + +.footer-bottom { + display: flex; + justify-content: space-between; + gap: 24px; + padding: 16px 0 27px; + border-top: 1px solid var(--line); + color: #687168; + font-size: 0.64rem; + line-height: 1.5; +} + +.footer-bottom span:last-child { text-align: right; } @@ -1261,6 +1402,15 @@ h3 { } @media (max-width: 930px) { + .site-header { + grid-template-columns: minmax(0, 1fr) auto; + min-height: 90px; + } + + .site-nav { + display: none; + } + .workbench { grid-template-columns: 1fr; } @@ -1272,6 +1422,11 @@ h3 { .capture-console { min-height: 0; } + + .footer-top { + grid-template-columns: 1fr; + gap: 33px; + } } @media (max-width: 680px) { @@ -1279,18 +1434,40 @@ h3 { width: min(100% - 28px, 1220px); } - .topbar { - min-height: 72px; + .site-header { + min-height: 80px; } - .public-label { + .brand-mark { + width: 33px; + height: 33px; + } + + .brand-mark::before { + width: 19px; + height: 19px; + } + + .brand-mark i:nth-child(1) { + top: 8px; + left: 8px; + } + + .brand-mark i:nth-child(2) { + right: 8px; + bottom: 8px; + } + + .brand-mark i:nth-child(3) { + top: 8px; + right: 8px; + } + + .brand-kicker, + .service-panel-label { display: none; } - .topbar-meta { - gap: 0; - } - .service-status { font-size: 0.66rem; } @@ -1338,14 +1515,12 @@ h3 { } .console-actions, - .section-heading, - .footer-note { + .section-heading { align-items: flex-start; flex-direction: column; } - .console-actions p, - .footer-note span:last-child { + .console-actions p { max-width: none; text-align: left; } @@ -1365,6 +1540,39 @@ h3 { .report-summary { grid-template-columns: repeat(2, minmax(0, 1fr)); } + + .footer-top { + padding: 34px 0 30px; + } + + .footer-points { + grid-template-columns: 1fr; + gap: 0; + } + + .footer-point, + .footer-point:first-child { + padding: 18px 0; + } + + .footer-point:first-child { + padding-top: 0; + } + + .footer-point + .footer-point { + border-top: 1px solid var(--line); + border-left: 0; + } + + .footer-bottom { + align-items: flex-start; + flex-direction: column; + gap: 5px; + } + + .footer-bottom span:last-child { + text-align: left; + } } @media (max-width: 390px) { diff --git a/app/static/app.js b/app/static/app.js index 1d96e12..9c0fe83 100644 --- a/app/static/app.js +++ b/app/static/app.js @@ -608,7 +608,7 @@ form.addEventListener("submit", async (event) => { max_depth: readNumber(controls.maxDepth, 5), max_bytes_mb: readNumber(controls.maxBytes, 100), max_duration_seconds: readNumber(controls.maxDuration, 300), - retention_days: readNumber(controls.retentionDays, 7), + retention_days: readNumber(controls.retentionDays, 2), }), }); upsertJob(job); diff --git a/app/templates/index.html b/app/templates/index.html index 5264e5e..570a3b5 100644 --- a/app/templates/index.html +++ b/app/templates/index.html @@ -4,38 +4,65 @@ + + - {{ app_name }} - Public web capture + + + + + + Website Downloader & Offline Archive Tool | {{ app_name }}
-
+
- siteharbor + + LOCAL WEB ARCHIVER + SiteHarbor + -
- PUBLIC WEB CAPTURE + +
+ LOCAL SERVICE Checking service
-
-
+
+
-

LIVE NETWORK SIGNAL

-

Copy the web.
Keep it close.

+

OFFLINE WEBSITE DOWNLOADER

+

Archive the web.
Keep it offline.

- Build an offline-ready snapshot of a public site. The service runs the crawl; - this browser keeps the key to its own sessions. + Create a private, offline-ready ZIP archive of a public website. SiteHarbor captures + supported pages and assets, then prepares local links for offline browsing.

@@ -67,7 +94,7 @@
-

NEW SESSION

+

NEW WEBSITE ARCHIVE

Set your crawl.

01 / 01 @@ -135,8 +162,8 @@
@@ -197,30 +224,59 @@
-
+

THIS BROWSER

-

Your local sessions

-

Session records stay in this browser's local storage. Reopen this page to reconnect to an active crawl.

+

Your website archives

+

Private session records stay in this browser. Return to monitor a crawl or download an available offline archive.

- +
-

No local sessions yet

-

Start a crawl above. Its status and archive will remain available from this browser after a reconnect.

+

No website archives yet

+

Start a crawl above. This browser can reconnect to its status and download it when ready.

-
diff --git a/tests/test_database.py b/tests/test_database.py index 9dc8550..9f45d63 100644 --- a/tests/test_database.py +++ b/tests/test_database.py @@ -1,5 +1,7 @@ from __future__ import annotations +from datetime import datetime, timedelta + from app.db import Database @@ -62,12 +64,15 @@ def test_sqlite_lease_claims_one_queued_job(tmp_path) -> None: "files_written": 1, }, warnings=[], - retention_days=1, + retention_days=30, ) ready = database.get_job(job["id"]) assert ready is not None assert ready["state"] == "ready" assert ready["expires_at"] > ready["completed_at"] + expires_at = datetime.fromisoformat(ready["expires_at"]) + completed_at = datetime.fromisoformat(ready["completed_at"]) + assert expires_at - completed_at <= timedelta(days=2) def test_worker_cannot_cancel_without_a_user_cancellation_request(tmp_path) -> None: diff --git a/tests/test_main.py b/tests/test_main.py index 1bcce9d..a79b8c9 100644 --- a/tests/test_main.py +++ b/tests/test_main.py @@ -19,6 +19,8 @@ def test_capture_routes_are_scoped_to_the_browser_session(tmp_path, monkeypatch) artifacts_dir=tmp_path / "artifacts", reports_dir=tmp_path / "reports", worker_enabled=False, + default_retention_days=2, + max_retention_days=2, ) test_settings.ensure_directories() test_database = Database(test_settings.database_path) @@ -31,7 +33,8 @@ def test_capture_routes_are_scoped_to_the_browser_session(tmp_path, monkeypatch) with TestClient(main.app) as client: homepage = client.get("/") assert homepage.status_code == 200 - assert "Copy the web." in homepage.text + assert "Archive the web." in homepage.text + assert "Website Downloader & Offline Archive Tool" in homepage.text assert client.get("/api/captures").status_code == 401 created = client.post( @@ -50,6 +53,18 @@ def test_capture_routes_are_scoped_to_the_browser_session(tmp_path, monkeypatch) assert created_job["options"]["parallel_connections"] == 2 assert created_job["options"]["download_asset_types"] == ["styles", "images"] assert created_job["options"]["skip_file_types"] == ["pdf", "zip"] + assert created_job["options"]["retention_days"] == 2 + + capabilities = client.get("/api/capabilities") + assert capabilities.status_code == 200 + assert capabilities.json()["limits"]["max_retention_days"] == 2 + + rejected_retention = client.post( + "/api/captures", + headers={"X-SiteHarbor-Session": owner_a}, + json={"url": "https://example.com", "retention_days": 3}, + ) + assert rejected_retention.status_code == 422 assert client.get( f"/api/captures/{job_id}", headers={"X-SiteHarbor-Session": owner_b} diff --git a/tests/test_scheduler.py b/tests/test_scheduler.py new file mode 100644 index 0000000..ab13d01 --- /dev/null +++ b/tests/test_scheduler.py @@ -0,0 +1,69 @@ +from __future__ import annotations + +from dataclasses import replace + +from app.config import settings +from app.db import Database +from app.scheduler import JobRunner + + +def test_maintenance_removes_archives_after_two_days(tmp_path) -> None: + test_settings = replace( + settings, + data_dir=tmp_path, + database_path=tmp_path / "siteharbor.sqlite3", + work_dir=tmp_path / "work", + artifacts_dir=tmp_path / "artifacts", + reports_dir=tmp_path / "reports", + worker_enabled=False, + ) + test_settings.ensure_directories() + database = Database(test_settings.database_path) + database.initialize() + job = database.create_job( + "https://example.com", + "https://example.com/", + { + "include_external_assets": True, + "max_pages": 20, + "max_depth": 3, + "max_bytes": 1_000_000, + "max_duration_seconds": 60, + "retention_days": 2, + }, + ) + assert database.claim_next("worker-a", lease_seconds=30) + + archive_path = test_settings.artifacts_dir / "capture.zip" + report_path = test_settings.reports_dir / "capture.json" + archive_path.write_bytes(b"zip contents") + report_path.write_text("{}", encoding="utf-8") + assert database.mark_ready( + job["id"], + "worker-a", + artifact_path=str(archive_path), + report_path=str(report_path), + archive_size=archive_path.stat().st_size, + entry_point="example.com/index.html", + stats={}, + warnings=[], + retention_days=2, + ) + with database.connection() as connection: + connection.execute( + """ + UPDATE jobs + SET completed_at = '2000-01-01T00:00:00+00:00', + expires_at = '2100-01-01T00:00:00+00:00' + WHERE id = ? + """, + (job["id"],), + ) + + JobRunner(database, test_settings)._expire_artifacts_if_due() + + expired = database.get_job(job["id"]) + assert expired is not None + assert expired["state"] == "expired" + assert not archive_path.exists() + assert not report_path.exists()