kstyagi/website-downloader
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

update retention policy to a maximum of 2 days for completed archives and adjust related configurations

Browse Source
This commit is contained in:
kstyagi23
2026-09-05 06:37:35 +05:30
parent 48402f7d61
commit 9c1a301c1a
10 changed files with 477 additions and 92 deletions
Download Patch File Download Diff File Expand all files Collapse all files
app/config.py
+12 -2
View File
@@ -4,6 +4,8 @@ import os
from dataclasses import dataclass
from pathlib import Path
MAX_ARCHIVE_RETENTION_DAYS = 2
def _as_bool(name: str, default: bool) -> bool:
value = os.getenv(name)
@@ -59,6 +61,14 @@ class Settings:
data_dir = Path(os.getenv("SITEHARBOR_DATA_DIR", project_root / "data")).expanduser()
if not data_dir.is_absolute():
data_dir = project_root / data_dir
max_retention_days = min(
_as_int("SITEHARBOR_MAX_RETENTION_DAYS", MAX_ARCHIVE_RETENTION_DAYS),
MAX_ARCHIVE_RETENTION_DAYS,
)
default_retention_days = min(
_as_int("SITEHARBOR_DEFAULT_RETENTION_DAYS", MAX_ARCHIVE_RETENTION_DAYS),
max_retention_days,
)
return cls(
project_root=project_root,
@@ -77,8 +87,8 @@ class Settings:
allow_private_networks=_as_bool("SITEHARBOR_ALLOW_PRIVATE_NETWORKS", False),
allow_nonstandard_ports=_as_bool("SITEHARBOR_ALLOW_NONSTANDARD_PORTS", False),
respect_robots=_as_bool("SITEHARBOR_RESPECT_ROBOTS", True),
default_retention_days=_as_int("SITEHARBOR_DEFAULT_RETENTION_DAYS", 7),
max_retention_days=_as_int("SITEHARBOR_MAX_RETENTION_DAYS", 30),
default_retention_days=default_retention_days,
max_retention_days=max_retention_days,
max_pages_cap=_as_int("SITEHARBOR_MAX_PAGES", 1_000),
max_depth_cap=_as_int("SITEHARBOR_MAX_DEPTH", 12),
max_bytes_cap=_as_int("SITEHARBOR_MAX_BYTES", 1024 * 1024 * 1024),
app/db.py
+17 -2
View File
@@ -9,6 +9,8 @@ from datetime import UTC, datetime, timedelta
from pathlib import Path
from typing import Any
from app.config import MAX_ARCHIVE_RETENTION_DAYS
def utcnow() -> datetime:
return datetime.now(UTC)
@@ -494,6 +496,7 @@ class Database:
) -> bool:
now_datetime = utcnow()
now = timestamp(now_datetime)
retention_days = min(retention_days, MAX_ARCHIVE_RETENTION_DAYS)
with self.connection() as connection:
cursor = connection.execute(
"""
@@ -644,10 +647,22 @@ class Database:
)
def expired_ready_jobs(self) -> list[dict[str, Any]]:
now = timestamp()
now_datetime = utcnow()
now = timestamp(now_datetime)
maximum_completion_age = timestamp(
now_datetime - timedelta(days=MAX_ARCHIVE_RETENTION_DAYS)
)
with self.connection() as connection:
rows = connection.execute(
"SELECT * FROM jobs WHERE state = 'ready' AND expires_at <= ?", (now,)
"""
SELECT * FROM jobs
WHERE state = 'ready'
AND (
expires_at <= ?
OR COALESCE(completed_at, created_at) <= ?
)
""",
(now, maximum_completion_age),
).fetchall()
return [self._row_to_job(row) for row in rows]
app/models.py
+5 -1
View File
@@ -4,6 +4,8 @@ from typing import Literal
from pydantic import BaseModel, Field
from app.config import MAX_ARCHIVE_RETENTION_DAYS
AssetType = Literal["styles", "scripts", "images", "fonts", "media", "other"]
SUPPORTED_ASSET_TYPES: tuple[AssetType, ...] = (
"styles",
@@ -28,4 +30,6 @@ class CaptureRequest(BaseModel):
max_depth: int = Field(default=5, ge=0)
max_bytes_mb: int = Field(default=100, ge=1)
max_duration_seconds: int = Field(default=300, ge=10)
retention_days: int = Field(default=7, ge=1)
retention_days: int = Field(
default=MAX_ARCHIVE_RETENTION_DAYS, ge=1, le=MAX_ARCHIVE_RETENTION_DAYS
)
app/static/app.css
+267 -59
View File
@@ -77,75 +77,156 @@ a:focus-visible {
margin: 0 auto;
}
.topbar {
display: flex;
.site-header {
position: relative;
display: grid;
grid-template-columns: minmax(220px, 1fr) auto minmax(180px, 1fr);
align-items: center;
justify-content: space-between;
min-height: 92px;
gap: 28px;
min-height: 108px;
border-bottom: 1px solid rgba(255, 255, 255, 0.1);
}
.site-header::after {
position: absolute;
bottom: -1px;
left: 0;
width: 118px;
height: 2px;
content: "";
background: var(--mint);
}
.brand {
display: inline-flex;
display: inline-grid;
grid-template-columns: auto 1fr;
align-items: center;
gap: 11px;
justify-self: start;
gap: 12px;
color: var(--ink);
font-size: 1.06rem;
font-weight: 780;
letter-spacing: -0.045em;
text-decoration: none;
}
.brand-mark {
position: relative;
display: block;
width: 24px;
height: 24px;
transform: rotate(45deg);
border: 1px solid rgba(152, 242, 187, 0.74);
border-radius: 7px;
background: rgba(79, 212, 138, 0.1);
display: grid;
width: 38px;
height: 38px;
place-items: center;
overflow: hidden;
border: 1px solid rgba(152, 242, 187, 0.48);
border-radius: 11px;
background:
linear-gradient(135deg, rgba(142, 234, 184, 0.22), rgba(6, 59, 44, 0.45)),
#0a2118;
box-shadow: inset 0 1px 0 rgba(238, 255, 244, 0.16);
}
.brand-mark::before {
position: absolute;
width: 23px;
height: 23px;
border: 1px solid rgba(197, 255, 218, 0.28);
border-radius: 50%;
content: "";
}
.brand-mark i {
position: absolute;
display: block;
width: 5px;
height: 5px;
z-index: 1;
width: 4px;
height: 4px;
border-radius: 50%;
background: var(--mint);
box-shadow: 0 0 8px rgba(142, 234, 184, 0.7);
}
.brand-mark i:nth-child(1) {
top: 4px;
left: 4px;
top: 10px;
left: 10px;
}
.brand-mark i:nth-child(2) {
right: 4px;
bottom: 4px;
right: 10px;
bottom: 10px;
opacity: 0.75;
}
.brand-mark i:nth-child(3) {
top: 4px;
right: 4px;
top: 10px;
right: 10px;
width: 3px;
height: 3px;
opacity: 0.55;
}
.topbar-meta,
.brand-copy {
display: grid;
gap: 3px;
}
.brand-kicker,
.service-panel-label,
.footer-kicker,
.footer-index {
color: var(--quiet);
font-size: 0.58rem;
font-weight: 780;
letter-spacing: 0.14em;
}
.brand strong {
color: var(--paper);
font-size: 1.16rem;
font-weight: 760;
letter-spacing: -0.05em;
}
.site-nav {
display: flex;
align-items: center;
gap: 4px;
padding: 5px;
border: 1px solid rgba(235, 246, 235, 0.1);
border-radius: 999px;
background: rgba(1, 4, 2, 0.3);
}
.site-nav a {
padding: 7px 10px;
border-radius: 999px;
color: #aeb6ad;
font-size: 0.68rem;
font-weight: 650;
text-decoration: none;
transition: color 160ms ease, background 160ms ease;
}
.site-nav a:hover {
color: var(--mint-bright);
background: rgba(142, 234, 184, 0.1);
}
.service-panel {
display: grid;
justify-items: end;
gap: 6px;
}
.service-status {
display: flex;
align-items: center;
gap: 8px;
color: #c3cbc2;
font-size: 0.72rem;
}
.topbar-meta {
gap: 20px;
.service-status[data-online="false"] > i {
background: var(--danger);
box-shadow: 0 0 0 4px rgba(240, 151, 142, 0.08), 0 0 13px rgba(240, 151, 142, 0.55);
}
.public-label,
.eyebrow {
color: var(--mint);
font-size: 0.65rem;
@@ -153,16 +234,6 @@ a:focus-visible {
letter-spacing: 0.16em;
}
.public-label {
color: var(--quiet);
}
.service-status {
gap: 8px;
color: #b7bdb7;
font-size: 0.72rem;
}
.service-status > i,
.pulse-dot {
display: inline-block;
@@ -1129,19 +1200,89 @@ h3 {
opacity: 0.56;
}
.footer-note {
display: flex;
justify-content: space-between;
gap: 22px;
padding: 24px 0 31px;
.site-footer {
margin-top: 7px;
border-top: 1px solid rgba(255, 255, 255, 0.1);
color: #7c827b;
font-size: 0.68rem;
line-height: 1.48;
}
.footer-note span:last-child {
max-width: 535px;
.footer-top {
display: grid;
grid-template-columns: minmax(0, 0.86fr) minmax(0, 1.54fr);
gap: clamp(34px, 6vw, 88px);
padding: 43px 0 37px;
}
.footer-kicker {
margin: 0 0 13px;
color: var(--mint);
}
.footer-intro h2 {
max-width: 390px;
margin: 0 0 14px;
font-size: clamp(1.55rem, 2.6vw, 2.2rem);
line-height: 1.03;
}
.footer-intro > p:last-child {
max-width: 420px;
margin: 0;
color: #99a19a;
font-size: 0.78rem;
line-height: 1.62;
}
.footer-points {
display: grid;
grid-template-columns: repeat(3, minmax(0, 1fr));
}
.footer-point {
min-width: 0;
padding: 0 20px;
}
.footer-point:first-child {
padding-left: 0;
}
.footer-point + .footer-point {
border-left: 1px solid var(--line);
}
.footer-index {
display: block;
margin-bottom: 15px;
color: #768177;
font-family: ui-monospace, SFMono-Regular, Menlo, Consolas, monospace;
}
.footer-point h3 {
margin: 0 0 8px;
color: #e4e9e2;
font-size: 0.87rem;
letter-spacing: -0.025em;
}
.footer-point p {
margin: 0;
color: #838b83;
font-size: 0.7rem;
line-height: 1.58;
}
.footer-bottom {
display: flex;
justify-content: space-between;
gap: 24px;
padding: 16px 0 27px;
border-top: 1px solid var(--line);
color: #687168;
font-size: 0.64rem;
line-height: 1.5;
}
.footer-bottom span:last-child {
text-align: right;
}
@@ -1261,6 +1402,15 @@ h3 {
}
@media (max-width: 930px) {
.site-header {
grid-template-columns: minmax(0, 1fr) auto;
min-height: 90px;
}
.site-nav {
display: none;
}
.workbench {
grid-template-columns: 1fr;
}
@@ -1272,6 +1422,11 @@ h3 {
.capture-console {
min-height: 0;
}
.footer-top {
grid-template-columns: 1fr;
gap: 33px;
}
}
@media (max-width: 680px) {
@@ -1279,18 +1434,40 @@ h3 {
width: min(100% - 28px, 1220px);
}
.topbar {
min-height: 72px;
.site-header {
min-height: 80px;
}
.public-label {
.brand-mark {
width: 33px;
height: 33px;
}
.brand-mark::before {
width: 19px;
height: 19px;
}
.brand-mark i:nth-child(1) {
top: 8px;
left: 8px;
}
.brand-mark i:nth-child(2) {
right: 8px;
bottom: 8px;
}
.brand-mark i:nth-child(3) {
top: 8px;
right: 8px;
}
.brand-kicker,
.service-panel-label {
display: none;
}
.topbar-meta {
gap: 0;
}
.service-status {
font-size: 0.66rem;
}
@@ -1338,14 +1515,12 @@ h3 {
}
.console-actions,
.section-heading,
.footer-note {
.section-heading {
align-items: flex-start;
flex-direction: column;
}
.console-actions p,
.footer-note span:last-child {
.console-actions p {
max-width: none;
text-align: left;
}
@@ -1365,6 +1540,39 @@ h3 {
.report-summary {
grid-template-columns: repeat(2, minmax(0, 1fr));
}
.footer-top {
padding: 34px 0 30px;
}
.footer-points {
grid-template-columns: 1fr;
gap: 0;
}
.footer-point,
.footer-point:first-child {
padding: 18px 0;
}
.footer-point:first-child {
padding-top: 0;
}
.footer-point + .footer-point {
border-top: 1px solid var(--line);
border-left: 0;
}
.footer-bottom {
align-items: flex-start;
flex-direction: column;
gap: 5px;
}
.footer-bottom span:last-child {
text-align: left;
}
}
@media (max-width: 390px) {
app/static/app.js
+1 -1
View File
@@ -608,7 +608,7 @@ form.addEventListener("submit", async (event) => {
max_depth: readNumber(controls.maxDepth, 5),
max_bytes_mb: readNumber(controls.maxBytes, 100),
max_duration_seconds: readNumber(controls.maxDuration, 300),
retention_days: readNumber(controls.retentionDays, 7),
retention_days: readNumber(controls.retentionDays, 2),
}),
});
upsertJob(job);
app/templates/index.html
+80 -24
View File
@@ -4,38 +4,65 @@
<meta charset="utf-8">
<meta name="viewport" content="width=device-width, initial-scale=1">
<meta name="theme-color" content="#151715">
<meta name="application-name" content="SiteHarbor">
<meta name="robots" content="index, follow">
<meta
name="description"
content="Create browser-owned offline archives of public websites with SiteHarbor."
content="Download public websites as private offline ZIP archives. SiteHarbor captures pages, static assets, and supported local links for offline browsing."
>
<title>{{ app_name }} - Public web capture</title>
<meta property="og:site_name" content="SiteHarbor">
<meta property="og:type" content="website">
<meta property="og:title" content="Website Downloader &amp; Offline Archive Tool | SiteHarbor">
<meta
property="og:description"
content="Create bounded, private offline ZIP archives of public websites with SiteHarbor."
>
<script type="application/ld+json">
{
"@context": "https://schema.org",
"@type": "WebApplication",
"name": "SiteHarbor",
"applicationCategory": "UtilitiesApplication",
"operatingSystem": "Web",
"description": "A local website downloader for creating bounded offline archives of public websites."
}
</script>
<title>Website Downloader &amp; Offline Archive Tool | {{ app_name }}</title>
<link rel="stylesheet" href="{{ url_for('static', path='app.css') }}">
<script defer src="{{ url_for('static', path='app.js') }}"></script>
</head>
<body>
<div class="site-shell">
<header class="topbar">
<header class="site-header">
<a class="brand" href="/" aria-label="SiteHarbor home">
<span class="brand-mark" aria-hidden="true"><i></i><i></i><i></i></span>
<span>siteharbor</span>
<span class="brand-copy">
<span class="brand-kicker">LOCAL WEB ARCHIVER</span>
<strong>SiteHarbor</strong>
</span>
</a>
<div class="topbar-meta">
<span class="public-label">PUBLIC WEB CAPTURE</span>
<nav class="site-nav" aria-label="Primary navigation">
<a href="#capture-workspace">Archive a site</a>
<a href="#recent-captures">Your captures</a>
<a href="#how-it-works">How it works</a>
</nav>
<div class="service-panel">
<span class="service-panel-label">LOCAL SERVICE</span>
<span class="service-status" id="service-status" aria-live="polite">
<i></i><span>Checking service</span>
</span>
</div>
</header>
<main>
<section class="workbench" aria-label="Website capture workspace">
<main id="main-content">
<section class="workbench" id="capture-workspace" aria-label="Website capture workspace">
<div class="observatory">
<div class="observatory-copy">
<p class="eyebrow">LIVE NETWORK SIGNAL</p>
<h1>Copy the web.<br><em>Keep it close.</em></h1>
<p class="eyebrow">OFFLINE WEBSITE DOWNLOADER</p>
<h1>Archive the web.<br><em>Keep it offline.</em></h1>
<p class="lede">
Build an offline-ready snapshot of a public site. The service runs the crawl;
this browser keeps the key to its own sessions.
Create a private, offline-ready ZIP archive of a public website. SiteHarbor captures
supported pages and assets, then prepares local links for offline browsing.
</p>
</div>
@@ -67,7 +94,7 @@
<form class="capture-console" id="capture-form" novalidate>
<div class="console-head">
<div>
<p class="eyebrow">NEW SESSION</p>
<p class="eyebrow">NEW WEBSITE ARCHIVE</p>
<h2>Set your crawl.</h2>
</div>
<span class="console-step">01 / 01</span>
@@ -135,8 +162,8 @@
<input id="max-duration" type="number" min="10" step="10" value="300">
</label>
<label>
<span>Keep (days)</span>
<input id="retention-days" type="number" min="1" step="1" value="7">
<span>Keep (days, max 2)</span>
<input id="retention-days" type="number" min="1" max="2" step="1" value="2">
</label>
</div>
</details>
@@ -197,30 +224,59 @@
</form>
</section>
<section class="activity-section" aria-labelledby="activity-title">
<section class="activity-section" id="recent-captures" aria-labelledby="activity-title">
<div class="section-heading">
<div>
<p class="eyebrow">THIS BROWSER</p>
<h2 id="activity-title">Your local sessions</h2>
<p>Session records stay in this browser's local storage. Reopen this page to reconnect to an active crawl.</p>
<h2 id="activity-title">Your website archives</h2>
<p>Private session records stay in this browser. Return to monitor a crawl or download an available offline archive.</p>
</div>
<button class="quiet-button" id="refresh-jobs" type="button">Refresh sessions</button>
<button class="quiet-button" id="refresh-jobs" type="button">Refresh archives</button>
</div>
<div class="empty-state" id="empty-state">
<span class="empty-mark" aria-hidden="true">+</span>
<div>
<h3>No local sessions yet</h3>
<p>Start a crawl above. Its status and archive will remain available from this browser after a reconnect.</p>
<h3>No website archives yet</h3>
<p>Start a crawl above. This browser can reconnect to its status and download it when ready.</p>
</div>
</div>
<div class="job-grid" id="job-grid" aria-live="polite"></div>
</section>
</main>
<footer class="footer-note">
<span>SiteHarbor captures browser-delivered static content.</span>
<span>Dynamic APIs, logins, service workers, and server behavior may not work offline.</span>
<footer class="site-footer" id="how-it-works">
<div class="footer-top">
<section class="footer-intro" aria-labelledby="footer-title">
<p class="footer-kicker">SITEHARBOR / OFFLINE WEB ARCHIVES</p>
<h2 id="footer-title">A practical website downloader for offline review.</h2>
<p>
Save a bounded, browser-owned copy of a public website without sending your archive
to a third-party storage service.
</p>
</section>
<div class="footer-points">
<section class="footer-point">
<span class="footer-index">01</span>
<h3>Bounded crawl</h3>
<p>Set page, depth, size, and time limits before SiteHarbor begins.</p>
</section>
<section class="footer-point">
<span class="footer-index">02</span>
<h3>Portable ZIP</h3>
<p>Download captured pages, supported assets, and a manifest for offline browsing.</p>
</section>
<section class="footer-point">
<span class="footer-index">03</span>
<h3>Private by default</h3>
<p>Archives stay scoped to this browser session and are removed after two days.</p>
</section>
</div>
</div>
<div class="footer-bottom">
<span>Local-first web archiving with FastAPI, Uvicorn, and SQLite.</span>
<span>Archive only websites you are authorized to copy.</span>
</div>
</footer>
</div>