diff --git a/app/capture.py b/app/capture.py index c495b88..1e35bdc 100644 --- a/app/capture.py +++ b/app/capture.py @@ -184,12 +184,17 @@ class CaptureService: ) def update( - phase: str, message: str, stats: dict[str, int], warnings: list[str] + phase: str, + message: str, + stats: dict[str, int], + warnings: list[str], + detail: dict[str, object] | None = None, ) -> None: nonlocal last_progress_at, last_progress_phase now = time.monotonic() if ( - phase == last_progress_phase + detail is None + and phase == last_progress_phase and now - last_progress_at < PROGRESS_WRITE_INTERVAL_SECONDS ): return @@ -203,6 +208,7 @@ class CaptureService: message=message, stats=stats, warnings=warnings, + detail=detail, ) if not renewed: raise CaptureCancelled("Capture lease was reassigned") diff --git a/app/crawler.py b/app/crawler.py index 9f6df22..f811e59 100644 --- a/app/crawler.py +++ b/app/crawler.py @@ -227,7 +227,9 @@ class SiteCrawler: settings: Settings, options: CaptureOptions, cancelled: Callable[[], bool], - on_progress: Callable[[str, str, dict[str, int], list[str]], None], + on_progress: Callable[ + [str, str, dict[str, int], list[str], dict[str, object] | None], None + ], ) -> None: self.settings = settings self.options = options @@ -273,6 +275,8 @@ class SiteCrawler: self._root_error: str | None = None self._limit_reached = False self._site_dir: Path | None = None + self._inflight_pages = 0 + self._inflight_assets = 0 self._address_book = PinnedAddressBook() limits = httpx.Limits( max_connections=max(4, self.parallel_connections * 2), @@ -307,6 +311,12 @@ class SiteCrawler: self.pending.popleft() for _ in range(min(len(self.pending), self.parallel_connections)) ] + self._inflight_pages = sum( + request.kind == "document" for request in batch + ) + self._inflight_assets = len(batch) - self._inflight_pages + for request in batch: + self._emit_page_started(request) pending_futures = {executor.submit(self._fetch, request) for request in batch} while pending_futures: completed, pending_futures = wait( @@ -316,7 +326,12 @@ class SiteCrawler: ) self._ensure_not_cancelled() for future in completed: - self._handle_fetch_result(future.result()) + result = future.result() + if result.request.kind == "document": + self._inflight_pages -= 1 + else: + self._inflight_assets -= 1 + self._handle_fetch_result(result) self._emit_progress("Capturing", "Discovering pages and assets") finally: @@ -473,6 +488,9 @@ class SiteCrawler: if is_root: self._root_error = result.message or "The starting page could not be fetched" self._record_problem(result) + self._emit_page_progress( + result, "skipped" if result.outcome == "skipped" else "failed" + ) return local_path = self.mapper.assign(result.final_url, result.content_type) @@ -491,6 +509,7 @@ class SiteCrawler: if is_root: self._root_error = failed_result.message self._record_problem(failed_result) + self._emit_page_progress(failed_result, "failed") return record = ResourceRecord( @@ -526,6 +545,7 @@ class SiteCrawler: self._discover_css(text, record.final_url, result.request.depth) elif self._is_javascript(record): self._discover_javascript(text, record.final_url, result.request.depth) + self._emit_page_progress(result, "captured") def _record_problem(self, result: FetchResult) -> None: if result.outcome == "skipped" and result.message: @@ -960,8 +980,46 @@ class SiteCrawler: ) (self._site_dir / "README.txt").write_text(readme, encoding="utf-8") - def _emit_progress(self, phase: str, message: str) -> None: - self.on_progress(phase, message, self.stats.copy(), self._visible_warnings()) + def _emit_page_started(self, request: CrawlRequest) -> None: + if request.kind != "document": + return + queued_pages = sum(request.kind == "document" for request in self.pending) + queued_assets = len(self.pending) - queued_pages + self._emit_progress( + "Capturing", + "Discovering pages and assets", + detail={ + "kind": "page", + "action": "fetching", + "url": request.url, + "depth": request.depth, + "pages_remaining": queued_pages + self._inflight_pages, + "assets_remaining": queued_assets + self._inflight_assets, + }, + ) + + def _emit_page_progress(self, result: FetchResult, action: str) -> None: + if result.request.kind != "document": + return + queued_pages = sum(request.kind == "document" for request in self.pending) + queued_assets = len(self.pending) - queued_pages + self._emit_progress( + "Capturing", + "Discovering pages and assets", + detail={ + "kind": "page", + "action": action, + "url": result.final_url or result.request.url, + "depth": result.request.depth, + "pages_remaining": queued_pages + self._inflight_pages, + "assets_remaining": queued_assets + self._inflight_assets, + }, + ) + + def _emit_progress( + self, phase: str, message: str, detail: dict[str, object] | None = None + ) -> None: + self.on_progress(phase, message, self.stats.copy(), self._visible_warnings(), detail) def _ensure_not_cancelled(self) -> None: if self.cancelled(): diff --git a/app/db.py b/app/db.py index 69106ee..f0a4c88 100644 --- a/app/db.py +++ b/app/db.py @@ -255,7 +255,7 @@ class Database: FROM job_events WHERE job_id = ? AND id > ? ORDER BY id ASC - LIMIT 100 + LIMIT 250 """, (job_id, after_id), ).fetchall() @@ -382,6 +382,7 @@ class Database: message: str, stats: dict[str, int], warnings: list[str] | None = None, + detail: dict[str, object] | None = None, ) -> bool: now = utcnow() values = { @@ -426,6 +427,7 @@ class Database: "message": message, "stats": stats, "warnings": warnings or [], + "detail": detail, }, ) return True diff --git a/app/main.py b/app/main.py index d4d88f8..f65db48 100644 --- a/app/main.py +++ b/app/main.py @@ -59,7 +59,8 @@ async def secure_headers(_: Request, call_next: Any) -> Response: response.headers.setdefault( "Content-Security-Policy", "default-src 'self'; base-uri 'none'; object-src 'none'; frame-ancestors 'none'; " - "form-action 'self'; connect-src 'self'; img-src 'self' data:; style-src 'self'; " + "form-action 'self'; connect-src 'self'; img-src 'self' data:; " + "style-src 'self' https://fonts.googleapis.com; font-src 'self' https://fonts.gstatic.com; " "script-src 'self'", ) return response @@ -224,8 +225,12 @@ async def _event_stream(request: Request, job_id: str, after_id: int) -> AsyncIt events = await asyncio.to_thread(database.get_events, job_id, last_id) for event in events: last_id = int(event["id"]) - payload = json.dumps(event["payload"], separators=(",", ":")) + payload = json.dumps( + {**event["payload"], "logged_at": event["created_at"]}, separators=(",", ":") + ) yield f"id: {last_id}\nevent: {event['kind']}\ndata: {payload}\n\n" + if events: + continue job = await asyncio.to_thread(database.get_job, job_id) if not job or job["state"] in terminal_states: diff --git a/app/static/app.css b/app/static/app.css index d4fedd8..b54572a 100644 --- a/app/static/app.css +++ b/app/static/app.css @@ -1,32 +1,23 @@ :root { - color-scheme: light; - --canvas: #f3f5f1; - --surface: #ffffff; - --surface-muted: #edf1ed; - --surface-strong: #e4ebe5; - --ink: #17201a; - --ink-soft: #334038; - --muted: #6b776f; - --quiet: #8a958d; - --line: #d8dfd8; - --line-strong: #c4cec5; - --night: #18221b; - --night-soft: #223128; - --night-line: rgba(239, 246, 239, 0.16); - --accent: #2168d8; - --accent-dark: #1855b4; - --accent-soft: #e6efff; - --success: #18714d; - --success-soft: #e3f4ea; - --danger: #b3473d; - --danger-soft: #fae9e6; - --warning: #996514; - --warning-soft: #fff4d6; - --radius-lg: 22px; - --radius-md: 14px; - --radius-sm: 9px; - --shadow: 0 18px 48px rgba(27, 42, 32, 0.08); - font-family: "Aptos", "Segoe UI", ui-sans-serif, system-ui, -apple-system, BlinkMacSystemFont, sans-serif; + color-scheme: dark; + --canvas: #171717; + --panel: #1d1d1d; + --panel-deep: #121212; + --panel-hover: #222222; + --ink: #e7e7e2; + --ink-soft: #c5c5be; + --muted: #898983; + --quiet: #60605d; + --line: #2c2c2a; + --line-strong: #3a3a37; + --blue: #89b4fa; + --green: #82c99b; + --yellow: #e3bd73; + --pink: #d99bd1; + --red: #ec9a91; + --mono: "Geist Mono", "Cascadia Mono", "SFMono-Regular", Consolas, monospace; + --sans: "Geist", "Segoe UI", ui-sans-serif, system-ui, sans-serif; + font-family: var(--sans); } * { @@ -64,22 +55,23 @@ button:focus-visible, input:focus-visible, summary:focus-visible, a:focus-visible { - outline: 3px solid rgba(33, 104, 216, 0.34); + outline: 1px solid var(--blue); outline-offset: 3px; } .skip-link { position: fixed; z-index: 20; - top: 12px; - left: 12px; - padding: 10px 13px; - border-radius: var(--radius-sm); - color: #fff; - background: var(--accent-dark); + top: 10px; + left: 10px; + padding: 8px 10px; + border: 1px solid var(--blue); + color: var(--ink); + background: var(--panel-deep); + font: 0.68rem var(--mono); text-decoration: none; - transform: translateY(-160%); - transition: transform 160ms ease; + transform: translateY(-180%); + transition: transform 120ms ease; } .skip-link:focus { @@ -87,36 +79,35 @@ a:focus-visible { } .site-shell { - width: min(1200px, calc(100% - 48px)); + width: min(1120px, calc(100% - 56px)); margin: 0 auto; } .site-header { display: grid; - grid-template-columns: minmax(210px, 1fr) auto minmax(250px, 1fr); + grid-template-columns: minmax(220px, 1fr) auto minmax(250px, 1fr); align-items: center; - gap: 28px; - min-height: 82px; - border-bottom: 1px solid var(--line); + gap: 20px; + min-height: 42px; + color: var(--muted); + font: 0.58rem var(--mono); } .brand { display: inline-flex; align-items: center; justify-self: start; - gap: 11px; + gap: 8px; + color: var(--muted); text-decoration: none; } .brand-mark { position: relative; - display: grid; - width: 31px; - height: 31px; - place-items: center; - overflow: hidden; - border-radius: 9px; - background: var(--night); + display: block; + width: 13px; + height: 13px; + border: 1px solid var(--ink-soft); } .brand-mark::before, @@ -126,88 +117,84 @@ a:focus-visible { } .brand-mark::before { - width: 13px; - height: 13px; - border: 2px solid #fff; - border-radius: 50%; + top: 2px; + right: 2px; + width: 3px; + height: 3px; + background: var(--blue); } .brand-mark::after { - right: 6px; - bottom: 6px; - width: 5px; - height: 5px; - border-radius: 50%; - background: #8db5ff; + bottom: 2px; + left: 2px; + width: 3px; + height: 3px; + background: var(--green); } .brand-copy { - display: grid; - gap: 2px; + display: flex; + align-items: baseline; + gap: 7px; } .brand-copy strong { - color: var(--ink); - font-size: 1rem; - font-weight: 760; - letter-spacing: -0.04em; + color: var(--ink-soft); + font: 600 0.63rem var(--mono); } .brand-copy span { color: var(--quiet); - font-size: 0.64rem; - font-weight: 650; + font-size: 0.54rem; } .site-nav { display: flex; align-items: center; - gap: 20px; + gap: 14px; } -.site-nav a { +.site-nav a, +.source-link { color: var(--muted); - font-size: 0.75rem; - font-weight: 680; text-decoration: none; - transition: color 150ms ease; + transition: color 120ms ease; } -.site-nav a:hover { - color: var(--accent-dark); +.site-nav a::before { + margin-right: 4px; + color: var(--quiet); + content: "/"; +} + +.site-nav a:hover, +.source-link:hover { + color: var(--blue); } .header-tools { display: flex; align-items: center; justify-content: flex-end; - gap: 17px; + gap: 14px; } .source-link { - padding: 7px 10px; - border: 1px solid var(--line-strong); - border-radius: 7px; color: var(--ink-soft); - font-size: 0.69rem; - font-weight: 700; - text-decoration: none; - transition: border-color 150ms ease, color 150ms ease, background 150ms ease; + font-weight: 500; } -.source-link:hover { - border-color: #9fbff4; - color: var(--accent-dark); - background: var(--accent-soft); +.source-link::before { + margin-right: 4px; + color: var(--green); + content: "<>"; } .service-status { display: inline-flex; align-items: center; - gap: 7px; - color: var(--muted); - font-size: 0.68rem; - font-weight: 620; + gap: 6px; + color: var(--quiet); white-space: nowrap; } @@ -215,76 +202,42 @@ a:focus-visible { .pulse-dot { display: inline-block; flex: 0 0 auto; - width: 7px; - height: 7px; + width: 5px; + height: 5px; border-radius: 50%; - background: var(--success); + background: var(--green); } .service-status[data-online="false"] > i { - background: var(--danger); + background: var(--red); } main { - padding: 48px 0 76px; + padding: 38px 0 66px; } .workbench { display: grid; - grid-template-columns: minmax(0, 0.88fr) minmax(460px, 1.12fr); - overflow: hidden; - min-height: 580px; - border: 1px solid var(--line); - border-radius: var(--radius-lg); - background: var(--surface); - box-shadow: var(--shadow); + grid-template-columns: minmax(0, 1fr) minmax(310px, 0.48fr); + gap: 34px; + padding: 0 0 30px; + border-bottom: 1px solid var(--line); } .observatory, .capture-console { - position: relative; min-width: 0; } .observatory { - display: flex; - flex-direction: column; - justify-content: space-between; - overflow: hidden; - padding: clamp(34px, 5vw, 62px); - color: #eaf0eb; - background: var(--night); -} - -.observatory::before { - position: absolute; - top: -126px; - right: -126px; - width: 290px; - height: 290px; - border: 1px solid rgba(230, 242, 233, 0.12); - border-radius: 50%; - content: ""; - box-shadow: 0 0 0 42px rgba(230, 242, 233, 0.025); -} - -.observatory-copy, -.metric-grid, -.observatory-foot { - position: relative; - z-index: 1; + padding: 10px 0 0; } .eyebrow { - margin: 0 0 11px; - color: var(--accent); - font-size: 0.63rem; - font-weight: 780; - letter-spacing: 0.13em; -} - -.observatory .eyebrow { - color: #a9c7ff; + margin: 0 0 13px; + color: var(--yellow); + font: 0.58rem var(--mono); + letter-spacing: 0.03em; } h1, @@ -298,62 +251,54 @@ h1, h2, h3 { color: var(--ink); - letter-spacing: -0.055em; } h1 { - max-width: 500px; - margin-bottom: 20px; - color: #f6f9f7; - font-size: clamp(3rem, 4.8vw, 4.8rem); - font-weight: 730; - line-height: 0.95; + margin-bottom: 18px; + font: 500 clamp(2rem, 4vw, 3.1rem) / 1.15 var(--mono); + letter-spacing: 0.12em; + text-transform: lowercase; } h1 em { - color: #a9c7ff; + color: var(--blue); font-style: normal; } h2 { margin-bottom: 0; - font-size: clamp(1.72rem, 2.5vw, 2.35rem); - font-weight: 730; - line-height: 1; + font: 600 1rem / 1.2 var(--mono); + letter-spacing: 0.02em; } h3 { margin-bottom: 6px; - font-size: 0.96rem; - font-weight: 740; + font: 600 0.77rem / 1.35 var(--mono); } .lede { - max-width: 430px; + max-width: 540px; margin-bottom: 0; - color: #bdc8c0; - font-size: 0.91rem; + color: var(--muted); + font-size: 0.78rem; line-height: 1.65; } .metric-grid { display: grid; - grid-template-columns: repeat(2, minmax(0, 1fr)); - gap: 10px; - margin: 35px 0; + grid-template-columns: repeat(4, minmax(0, 1fr)); + margin: 30px 0 24px; + border-top: 1px solid var(--line); + border-bottom: 1px solid var(--line); } .metric-card { - min-height: 88px; - padding: 15px; - border: 1px solid var(--night-line); - border-radius: var(--radius-sm); - background: rgba(255, 255, 255, 0.045); + min-width: 0; + padding: 13px 11px; } -.metric-card.metric-primary { - border-color: #b9cdf1; - background: #e9f0fc; +.metric-card + .metric-card { + border-left: 1px solid var(--line); } .metric-value, @@ -363,146 +308,134 @@ h3 { .metric-value { overflow: hidden; - color: #f4f8f5; - font-size: clamp(1.25rem, 1.9vw, 1.62rem); - font-weight: 760; - letter-spacing: -0.06em; + color: var(--ink-soft); + font: 600 clamp(1rem, 1.7vw, 1.35rem) / 1 var(--mono); text-overflow: ellipsis; white-space: nowrap; } -.metric-primary .metric-value { - color: #183e78; +.metric-card.metric-primary .metric-value { + color: var(--blue); } .metric-label { - margin-top: 6px; - color: #aab8af; - font-size: 0.59rem; - font-weight: 720; - letter-spacing: 0.08em; - text-transform: uppercase; -} - -.metric-primary .metric-label { - color: #58719a; + margin-top: 7px; + color: var(--quiet); + font: 0.54rem var(--mono); + letter-spacing: 0.03em; + text-transform: lowercase; } .observatory-foot { display: flex; align-items: flex-start; - gap: 10px; - max-width: 430px; - padding-top: 17px; - border-top: 1px solid var(--night-line); + gap: 8px; + max-width: 570px; + color: var(--quiet); } .observatory-foot .pulse-dot { - width: 6px; - height: 6px; - margin-top: 5px; - background: #a9c7ff; + width: 5px; + height: 5px; + margin-top: 4px; + background: var(--green); } .observatory-foot p { margin: 0; - color: #aebbb2; - font-size: 0.71rem; + font-size: 0.65rem; line-height: 1.55; } .capture-console { - padding: clamp(32px, 4.5vw, 54px); - background: var(--surface); + align-self: start; + padding: 18px; + border: 1px solid var(--line-strong); + background: var(--panel); } .console-head { display: flex; align-items: flex-start; justify-content: space-between; - gap: 18px; - margin-bottom: 31px; + gap: 14px; + margin-bottom: 23px; + padding-bottom: 14px; + border-bottom: 1px solid var(--line); +} + +.console-head .eyebrow { + margin-bottom: 8px; } .console-step { - padding: 5px 8px; - border: 1px solid var(--line); - border-radius: 6px; color: var(--quiet); - font-family: "Cascadia Mono", "SFMono-Regular", Consolas, monospace; - font-size: 0.6rem; - font-weight: 700; - letter-spacing: 0.06em; + font: 0.56rem var(--mono); } .field-label, .limit-grid label > span, .text-rule > span { display: block; - margin-bottom: 8px; + margin-bottom: 7px; color: var(--ink-soft); - font-size: 0.7rem; - font-weight: 720; + font-size: 0.66rem; + font-weight: 500; } .url-input-wrap { display: flex; align-items: center; border: 1px solid var(--line-strong); - border-radius: var(--radius-sm); - background: #fbfcfa; - transition: border-color 150ms ease, box-shadow 150ms ease; + background: var(--panel-deep); + transition: border-color 120ms ease; } .url-input-wrap:focus-within { - border-color: var(--accent); - box-shadow: 0 0 0 4px rgba(33, 104, 216, 0.1); + border-color: var(--blue); } .url-input-wrap > span { - padding-left: 15px; - color: var(--accent); - font-family: "Cascadia Mono", "SFMono-Regular", Consolas, monospace; - font-size: 0.95rem; - font-weight: 700; + padding-left: 11px; + color: var(--green); + font: 0.76rem var(--mono); } .url-input-wrap input { width: 100%; min-width: 0; - padding: 15px; + padding: 12px 11px; border: 0; outline: 0; color: var(--ink); background: transparent; - font-size: 0.93rem; + font: 0.75rem var(--mono); } .url-input-wrap input::placeholder, .text-rule input::placeholder { - color: #9aa49d; + color: var(--quiet); } .quick-settings { display: grid; - gap: 9px; - margin-top: 17px; + gap: 7px; + margin-top: 10px; } .switch-row, .connection-control { border: 1px solid var(--line); - border-radius: var(--radius-sm); - background: var(--surface-muted); + background: var(--panel-deep); } .switch-row { display: flex; align-items: center; justify-content: space-between; - gap: 18px; - min-height: 72px; - padding: 13px 15px; + gap: 14px; + min-height: 64px; + padding: 11px 12px; } .switch-row strong, @@ -516,15 +449,14 @@ h3 { .connection-control strong { margin-bottom: 3px; color: var(--ink-soft); - font-size: 0.79rem; - font-weight: 730; + font: 500 0.69rem var(--mono); } .switch-row small, .connection-control small { - max-width: 330px; - color: var(--muted); - font-size: 0.68rem; + max-width: 245px; + color: var(--quiet); + font-size: 0.6rem; line-height: 1.4; } @@ -542,92 +474,84 @@ h3 { .switch > span { position: relative; display: block; - width: 39px; - height: 22px; - border: 1px solid #b4c0b6; - border-radius: 999px; - background: #d6ddd7; - transition: border-color 150ms ease, background 150ms ease; + width: 32px; + height: 17px; + border: 1px solid var(--line-strong); + background: #282826; + transition: border-color 120ms ease, background 120ms ease; } .switch > span::after { position: absolute; top: 3px; left: 3px; - width: 14px; - height: 14px; - border-radius: 50%; - background: #fff; - box-shadow: 0 1px 3px rgba(23, 32, 26, 0.18); + width: 9px; + height: 9px; + background: var(--muted); content: ""; - transition: transform 150ms ease; + transition: transform 120ms ease, background 120ms ease; } .switch input:checked + span { - border-color: var(--accent); - background: var(--accent); + border-color: var(--green); + background: #1e3427; } .switch input:checked + span::after { - transform: translateX(16px); + transform: translateX(13px); + background: var(--green); } .switch input:disabled + span { - cursor: not-allowed; - opacity: 0.5; + opacity: 0.45; } .connection-control { display: grid; grid-template-columns: minmax(0, 1fr) auto; - gap: 11px 16px; - padding: 13px 15px; + gap: 10px 12px; + padding: 11px 12px; } .connection-value { align-self: center; - color: var(--muted); - font-family: "Cascadia Mono", "SFMono-Regular", Consolas, monospace; - font-size: 0.66rem; + color: var(--quiet); + font: 0.58rem var(--mono); } .connection-value output { - color: var(--accent-dark); - font-weight: 800; + color: var(--blue); } .connection-control input[type="range"] { width: 100%; - height: 4px; + height: 3px; grid-column: 1 / -1; - margin: 2px 0 0; - accent-color: var(--accent); - cursor: pointer; + margin: 1px 0 0; + accent-color: var(--blue); } .advanced-settings { display: grid; - gap: 8px; - margin-top: 10px; + gap: 6px; + margin-top: 7px; } .setting-panel { border: 1px solid var(--line); - border-radius: var(--radius-sm); - background: #fbfcfa; + background: var(--panel-deep); } .setting-panel summary { display: flex; align-items: center; justify-content: space-between; - gap: 14px; - min-height: 49px; - padding: 0 14px; + gap: 10px; + min-height: 40px; + padding: 0 11px; color: var(--ink-soft); cursor: pointer; - font-size: 0.75rem; - font-weight: 720; + font: 0.66rem var(--mono); list-style: none; } @@ -635,26 +559,25 @@ h3 { display: none; } -.setting-panel summary::after { - color: var(--muted); +.setting-panel summary::before { + margin-right: 5px; + color: var(--quiet); content: "+"; - font-size: 1rem; - font-weight: 400; } .setting-panel[open] summary { border-bottom: 1px solid var(--line); } -.setting-panel[open] summary::after { +.setting-panel[open] summary::before { content: "-"; } .summary-value { overflow: hidden; + margin-left: auto; color: var(--quiet); - font-size: 0.64rem; - font-weight: 560; + font-size: 0.56rem; text-align: right; text-overflow: ellipsis; white-space: nowrap; @@ -663,14 +586,14 @@ h3 { .limit-grid { display: grid; grid-template-columns: repeat(5, minmax(0, 1fr)); - gap: 8px; - padding: 14px; + gap: 6px; + padding: 11px; } .limit-grid label > span { - margin-bottom: 6px; - color: var(--muted); - font-size: 0.61rem; + margin-bottom: 5px; + color: var(--quiet); + font: 0.54rem var(--mono); } .limit-grid input, @@ -678,212 +601,201 @@ h3 { width: 100%; min-width: 0; border: 1px solid var(--line-strong); - border-radius: 6px; + border-radius: 0; outline: 0; color: var(--ink); - background: #fff; - transition: border-color 150ms ease, box-shadow 150ms ease; + background: #191919; + font: 0.64rem var(--mono); + transition: border-color 120ms ease; } .limit-grid input { - padding: 9px 7px; - font-size: 0.74rem; + padding: 8px 5px; } .limit-grid input:focus, .text-rule input:focus { - border-color: var(--accent); - box-shadow: 0 0 0 3px rgba(33, 104, 216, 0.1); + border-color: var(--blue); } .file-rules { - padding: 14px; + padding: 11px; } .file-rules > p { - margin-bottom: 12px; - color: var(--muted); - font-size: 0.68rem; + margin-bottom: 10px; + color: var(--quiet); + font-size: 0.6rem; line-height: 1.45; } .asset-choices { display: grid; grid-template-columns: repeat(3, minmax(0, 1fr)); - gap: 7px; + gap: 5px; } .asset-choices label { display: flex; align-items: center; - gap: 7px; - min-height: 33px; - padding: 0 8px; + gap: 5px; + min-height: 29px; + padding: 0 6px; border: 1px solid var(--line); - border-radius: 6px; - color: var(--ink-soft); - background: #fff; - font-size: 0.67rem; + color: var(--muted); + font: 0.57rem var(--mono); } .asset-choices input { - width: 13px; - height: 13px; + width: 11px; + height: 11px; margin: 0; - accent-color: var(--accent); + accent-color: var(--blue); } .text-rule { display: block; - margin-top: 13px; + margin-top: 10px; } .text-rule > span { - margin-bottom: 6px; - color: var(--muted); - font-size: 0.64rem; + margin-bottom: 5px; + color: var(--quiet); + font: 0.55rem var(--mono); } .text-rule input { - padding: 10px 11px; - font-size: 0.74rem; + padding: 9px; } .proxy-row { - min-height: 72px; - margin: 14px; + min-height: 64px; + margin: 11px; } .proxy-row.is-unavailable { - opacity: 0.6; + opacity: 0.5; } .form-message { - min-height: 1.25em; - margin: 13px 0 0; - color: var(--danger); - font-size: 0.73rem; + min-height: 1.2em; + margin: 10px 0 0; + color: var(--red); + font: 0.63rem var(--mono); line-height: 1.4; } .form-message.success { - color: var(--success); + color: var(--green); } .console-actions { display: flex; align-items: center; justify-content: space-between; - gap: 18px; - margin-top: 13px; + gap: 12px; + margin-top: 10px; } .capture-button { display: inline-flex; align-items: center; justify-content: space-between; - gap: 20px; - flex: 0 0 auto; - min-height: 43px; - padding: 0 14px 0 16px; - border: 1px solid var(--accent); - border-radius: 7px; - color: #fff; - background: var(--accent); - font-size: 0.75rem; - font-weight: 760; - transition: transform 150ms ease, background 150ms ease, box-shadow 150ms ease; + gap: 16px; + min-height: 35px; + padding: 0 10px; + border: 1px solid var(--ink-soft); + border-radius: 0; + color: #161616; + background: var(--ink-soft); + font: 600 0.64rem var(--mono); + transition: background 120ms ease, border-color 120ms ease, transform 120ms ease; } .capture-button:hover:not(:disabled) { - background: var(--accent-dark); - box-shadow: 0 8px 16px rgba(33, 104, 216, 0.22); + border-color: #fff; + background: #fff; transform: translateY(-1px); } .capture-button:disabled { cursor: wait; - opacity: 0.58; + opacity: 0.5; } .capture-button span:last-child { - font-family: "Cascadia Mono", "SFMono-Regular", Consolas, monospace; - font-size: 0.95rem; + color: var(--blue); } .console-actions p { - max-width: 270px; + max-width: 190px; margin: 0; - color: var(--muted); - font-size: 0.64rem; + color: var(--quiet); + font-size: 0.55rem; line-height: 1.45; text-align: right; } .activity-section { - padding-top: 72px; + padding-top: 31px; } .section-heading { display: flex; align-items: flex-end; justify-content: space-between; - gap: 22px; - margin-bottom: 20px; + gap: 20px; + padding-bottom: 16px; + border-bottom: 1px solid var(--line); } .section-heading .eyebrow { - margin-bottom: 9px; + margin-bottom: 8px; } .section-heading h2 { - margin-bottom: 8px; - font-size: clamp(1.65rem, 2.2vw, 2.1rem); + margin-bottom: 7px; } .section-heading p:not(.eyebrow) { max-width: 560px; margin: 0; color: var(--muted); - font-size: 0.77rem; - line-height: 1.55; + font-size: 0.7rem; + line-height: 1.5; } .quiet-button, .job-action, .icon-button { border: 1px solid var(--line-strong); - border-radius: 7px; - color: var(--ink-soft); - background: var(--surface); - font-size: 0.7rem; - font-weight: 700; - transition: border-color 150ms ease, color 150ms ease, background 150ms ease; + border-radius: 0; + color: var(--muted); + background: transparent; + font: 0.61rem var(--mono); + transition: color 120ms ease, border-color 120ms ease, background 120ms ease; } .quiet-button { - min-height: 34px; - padding: 0 12px; - white-space: nowrap; + min-height: 30px; + padding: 0 9px; } .quiet-button:hover, .job-action:hover, .icon-button:hover { - border-color: #9fbff4; - color: var(--accent-dark); - background: var(--accent-soft); + border-color: var(--blue); + color: var(--blue); + background: #1b2130; } .empty-state { display: flex; align-items: center; - gap: 15px; - min-height: 119px; - padding: 21px; - border: 1px dashed var(--line-strong); - border-radius: var(--radius-md); - background: rgba(255, 255, 255, 0.48); + gap: 11px; + min-height: 96px; + padding: 16px 0; + border-bottom: 1px solid var(--line); } .empty-state[hidden] { @@ -891,74 +803,38 @@ h3 { } .empty-state p { - max-width: 560px; + max-width: 520px; margin: 0; color: var(--muted); - font-size: 0.76rem; + font-size: 0.68rem; line-height: 1.5; } .empty-mark { display: grid; flex: 0 0 auto; - width: 36px; - height: 36px; + width: 22px; + height: 22px; place-items: center; - border-radius: 50%; - color: var(--accent-dark); - background: var(--accent-soft); - font-size: 1.2rem; + border: 1px solid var(--line-strong); + color: var(--blue); + font: 0.8rem var(--mono); } .job-grid { display: grid; - grid-template-columns: repeat(2, minmax(0, 1fr)); - gap: 13px; + grid-template-columns: 1fr; } .job-card { - position: relative; - contain-intrinsic-size: auto 270px; - content-visibility: auto; - overflow: hidden; - min-height: 252px; - padding: 19px; - border: 1px solid var(--line); - border-radius: var(--radius-md); - background: var(--surface); - box-shadow: 0 3px 10px rgba(27, 42, 32, 0.025); - transition: border-color 150ms ease, box-shadow 150ms ease, transform 150ms ease; + padding: 17px 0 18px; + border-bottom: 1px solid var(--line); + background: transparent; } -.job-card:hover { - border-color: #c5d0c6; - box-shadow: 0 10px 24px rgba(27, 42, 32, 0.07); - transform: translateY(-1px); -} - -.job-card::before { - position: absolute; - top: 0; - left: 0; - width: 100%; - height: 3px; - content: ""; - background: #bdc8bf; -} - -.job-card[data-state="running"]::before, -.job-card[data-state="queued"]::before { - background: var(--accent); -} - -.job-card[data-state="ready"]::before { - background: var(--success); -} - -.job-card[data-state="failed"]::before, -.job-card[data-state="cancelled"]::before, -.job-card[data-state="expired"]::before { - background: var(--danger); +.job-card.is-active { + border-left: 2px solid var(--blue); + padding-left: 14px; } .job-topline, @@ -972,99 +848,87 @@ h3 { .job-topline { align-items: center; justify-content: space-between; - gap: 12px; + gap: 14px; } .job-host { overflow: hidden; - color: var(--ink); - font-size: 0.91rem; - font-weight: 760; - letter-spacing: -0.03em; + color: var(--ink-soft); + font: 600 0.82rem var(--mono); text-overflow: ellipsis; white-space: nowrap; } .state-pill { flex: 0 0 auto; - padding: 5px 7px; + padding: 3px 5px; border: 1px solid var(--line-strong); - border-radius: 999px; color: var(--muted); - font-size: 0.57rem; - font-weight: 760; - letter-spacing: 0.07em; - text-transform: uppercase; + font: 0.53rem var(--mono); + text-transform: lowercase; } .state-pill.ready { - border-color: #a9d8bd; - color: var(--success); - background: var(--success-soft); + border-color: #416c50; + color: var(--green); } .state-pill.failed, .state-pill.cancelled, .state-pill.expired { - border-color: #edc0ba; - color: var(--danger); - background: var(--danger-soft); + border-color: #71443f; + color: var(--red); } .state-pill.running, .state-pill.queued { - border-color: #b9cff2; - color: var(--accent-dark); - background: var(--accent-soft); + border-color: #40557a; + color: var(--blue); } .job-url { overflow: hidden; - margin: 6px 0 16px; + margin: 6px 0 11px; color: var(--quiet); - font-family: "Cascadia Mono", "SFMono-Regular", Consolas, monospace; - font-size: 0.64rem; + font: 0.59rem var(--mono); text-overflow: ellipsis; white-space: nowrap; } .job-phase { - min-height: 35px; - margin: 0 0 12px; + margin: 0 0 10px; color: var(--ink-soft); - font-size: 0.75rem; + font-size: 0.7rem; line-height: 1.45; } .job-progress { overflow: hidden; - height: 4px; - margin: 0 0 15px; - border-radius: 999px; - background: var(--surface-strong); + height: 2px; + margin: 0 0 12px; + background: var(--line); } .job-progress > span { display: block; width: var(--progress, 0%); height: 100%; - border-radius: inherit; - background: var(--accent); - transition: width 220ms ease; + background: var(--blue); + transition: width 180ms ease; } .job-card[data-state="ready"] .job-progress > span { - background: var(--success); + background: var(--green); } .job-card[data-state="failed"] .job-progress > span, .job-card[data-state="cancelled"] .job-progress > span, .job-card[data-state="expired"] .job-progress > span { - background: var(--danger); + background: var(--red); } .job-stats { - margin-bottom: 13px; + margin-bottom: 10px; border-top: 1px solid var(--line); border-bottom: 1px solid var(--line); } @@ -1072,7 +936,7 @@ h3 { .job-stats div { flex: 1; min-width: 0; - padding: 10px 7px; + padding: 8px 7px; } .job-stats div + div { @@ -1087,8 +951,8 @@ h3 { .job-stats strong { overflow: hidden; - color: var(--ink); - font-size: 0.8rem; + color: var(--ink-soft); + font: 500 0.73rem var(--mono); text-overflow: ellipsis; white-space: nowrap; } @@ -1097,23 +961,20 @@ h3 { .job-meta span:first-child { margin-top: 3px; color: var(--quiet); - font-size: 0.57rem; - font-weight: 650; - letter-spacing: 0.05em; - text-transform: uppercase; + font: 0.51rem var(--mono); + text-transform: lowercase; } .job-meta { justify-content: space-between; gap: 10px; - min-height: 18px; margin-bottom: 9px; color: var(--muted); - font-size: 0.65rem; + font-size: 0.61rem; } .job-meta span:last-child { - max-width: 56%; + max-width: 58%; overflow: hidden; text-align: right; text-overflow: ellipsis; @@ -1123,73 +984,150 @@ h3 { .job-tags { flex-wrap: wrap; gap: 5px; - min-height: 21px; margin-bottom: 10px; } .job-tags span { - padding: 3px 6px; + padding: 2px 5px; border: 1px solid var(--line); - border-radius: 999px; - color: var(--muted); - font-size: 0.57rem; + color: var(--quiet); + font: 0.51rem var(--mono); } .job-warning { overflow: hidden; margin: 0 0 10px; - color: var(--warning); - font-size: 0.65rem; - line-height: 1.35; + color: var(--yellow); + font: 0.6rem var(--mono); text-overflow: ellipsis; white-space: nowrap; } +.job-log { + margin: 13px 0; + border: 1px solid var(--line-strong); + background: var(--panel-deep); +} + +.job-log-heading { + display: flex; + align-items: center; + justify-content: space-between; + padding: 7px 9px; + border-bottom: 1px solid var(--line); + color: var(--quiet); + font: 0.54rem var(--mono); + text-transform: lowercase; +} + +.job-log-title { + color: var(--ink-soft); +} + +.job-log-title::before { + margin-right: 5px; + color: var(--green); + content: ">"; +} + +.job-log-list { + max-height: 260px; + margin: 0; + padding: 0; + overflow: auto; + list-style: none; + scrollbar-color: var(--line-strong) var(--panel-deep); +} + +.job-log-entry { + display: grid; + grid-template-columns: 68px 88px minmax(0, 1fr) auto; + gap: 8px; + align-items: start; + padding: 6px 9px; + border-bottom: 1px solid #242422; + color: var(--muted); + font: 0.58rem/1.45 var(--mono); +} + +.job-log-entry:last-child { + border-bottom: 0; +} + +.job-log-entry[data-kind="page"] { + color: var(--ink-soft); +} + +.job-log-time { + color: var(--quiet); +} + +.job-log-action { + color: var(--blue); + text-transform: lowercase; +} + +.job-log-entry[data-kind="state"] .job-log-action { + color: var(--green); +} + +.job-log-url { + min-width: 0; + overflow-wrap: anywhere; +} + +.job-log-queue { + color: var(--quiet); + white-space: nowrap; +} + +.job-log-empty { + padding: 10px 9px; + color: var(--quiet); + font: 0.58rem var(--mono); +} + .job-actions { align-items: center; flex-wrap: wrap; - gap: 7px; + gap: 6px; } .job-action { display: inline-flex; align-items: center; - min-height: 30px; - padding: 0 9px; + min-height: 27px; + padding: 0 7px; } .job-action.primary { - border-color: var(--accent); - color: #fff; - background: var(--accent); + border-color: #4a689a; + color: var(--blue); } .job-action.primary:hover { - border-color: var(--accent-dark); - color: #fff; - background: var(--accent-dark); + border-color: var(--blue); + background: #1b2130; } .job-action.danger { - border-color: #e1b4ae; - color: var(--danger); + border-color: #67433e; + color: var(--red); } .job-action:disabled { cursor: wait; - opacity: 0.56; + opacity: 0.5; } .site-footer { display: flex; align-items: center; justify-content: space-between; - gap: 22px; - padding: 22px 0 30px; - border-top: 1px solid var(--line); - color: var(--muted); - font-size: 0.68rem; - line-height: 1.5; + gap: 20px; + padding: 23px 0 32px; + color: var(--quiet); + font: 0.56rem var(--mono); } .site-footer p { @@ -1199,18 +1137,16 @@ h3 { .footer-links { display: flex; align-items: center; - gap: 16px; - white-space: nowrap; + gap: 13px; } .footer-links a { - color: var(--accent-dark); - font-weight: 720; + color: var(--muted); text-decoration: none; } .footer-links a:hover { - text-decoration: underline; + color: var(--blue); } .report-dialog { @@ -1218,22 +1154,22 @@ h3 { max-height: min(760px, calc(100vh - 44px)); padding: 0; border: 1px solid var(--line-strong); - border-radius: var(--radius-md); + border-radius: 0; color: var(--ink); - background: var(--surface); - box-shadow: 0 24px 70px rgba(27, 42, 32, 0.2); + background: var(--panel); + box-shadow: 0 24px 72px rgba(0, 0, 0, 0.45); } .report-dialog::backdrop { - background: rgba(24, 34, 27, 0.36); + background: rgba(0, 0, 0, 0.72); } .dialog-head { display: flex; align-items: flex-start; justify-content: space-between; - gap: 20px; - padding: 23px 24px 18px; + gap: 18px; + padding: 19px; border-bottom: 1px solid var(--line); } @@ -1242,34 +1178,35 @@ h3 { } .dialog-head h2 { - margin: 0; - font-size: 1.65rem; + font-size: 0.98rem; } .icon-button { - width: 31px; - height: 31px; - font-size: 1rem; + width: 29px; + height: 29px; + font-size: 0.9rem; } .report-content { overflow: auto; - max-height: calc(min(760px, 100vh - 44px) - 102px); - padding: 22px 24px 28px; + max-height: calc(min(760px, 100vh - 44px) - 94px); + padding: 19px; } .report-summary { display: grid; grid-template-columns: repeat(4, minmax(0, 1fr)); - gap: 9px; - margin-bottom: 20px; + margin-bottom: 18px; + border: 1px solid var(--line); } .report-summary div { - padding: 12px; - border: 1px solid var(--line); - border-radius: 8px; - background: var(--surface-muted); + min-width: 0; + padding: 10px; +} + +.report-summary div + div { + border-left: 1px solid var(--line); } .report-summary strong, @@ -1279,38 +1216,36 @@ h3 { .report-summary strong { overflow: hidden; - color: var(--accent-dark); - font-size: 0.98rem; + color: var(--blue); + font: 500 0.76rem var(--mono); text-overflow: ellipsis; white-space: nowrap; } .report-summary span { margin-top: 4px; - color: var(--muted); - font-size: 0.59rem; - letter-spacing: 0.04em; - text-transform: uppercase; + color: var(--quiet); + font: 0.5rem var(--mono); + text-transform: lowercase; } .report-list { margin: 0; - padding-left: 18px; - color: var(--ink-soft); - font-size: 0.76rem; + padding-left: 17px; + color: var(--muted); + font-size: 0.69rem; line-height: 1.55; } .report-list li + li { - margin-top: 6px; + margin-top: 5px; } .report-subhead { - margin: 21px 0 9px; - color: var(--accent-dark); - font-size: 0.68rem; - letter-spacing: 0.08em; - text-transform: uppercase; + margin: 19px 0 8px; + color: var(--yellow); + font: 0.62rem var(--mono); + text-transform: lowercase; } @media (prefers-reduced-motion: reduce) { @@ -1322,7 +1257,7 @@ h3 { } } -@media (max-width: 980px) { +@media (max-width: 900px) { .site-header { grid-template-columns: minmax(0, 1fr) auto; } @@ -1333,21 +1268,21 @@ h3 { .workbench { grid-template-columns: 1fr; + gap: 26px; } - .observatory { - min-height: 460px; + .capture-console { + border-left: 0; } } @media (max-width: 680px) { .site-shell { - width: min(100% - 28px, 1200px); + width: min(100% - 28px, 1120px); } .site-header { - min-height: 72px; - gap: 14px; + min-height: 38px; } .brand-copy span, @@ -1356,45 +1291,37 @@ h3 { } .header-tools { - gap: 10px; - } - - .source-link { - padding: 6px 8px; + gap: 9px; } main { - padding: 28px 0 56px; + padding: 27px 0 48px; } .workbench { - border-radius: var(--radius-md); - } - - .observatory, - .capture-console { - padding: 28px 22px; - } - - .observatory { - min-height: 0; + gap: 22px; + padding-bottom: 23px; } h1 { - font-size: clamp(2.65rem, 13vw, 3.85rem); + font-size: clamp(1.6rem, 8vw, 2.4rem); + letter-spacing: 0.08em; } .metric-grid { - margin: 28px 0; + grid-template-columns: repeat(2, minmax(0, 1fr)); } - .metric-card { - min-height: 82px; - padding: 13px; + .metric-card:nth-child(3) { + border-left: 0; } - .console-head { - margin-bottom: 25px; + .metric-card:nth-child(-n + 2) { + border-bottom: 1px solid var(--line); + } + + .capture-console { + padding: 14px; } .limit-grid { @@ -1418,38 +1345,62 @@ h3 { } .activity-section { - padding-top: 56px; + padding-top: 26px; } .section-heading { - gap: 15px; + gap: 14px; } - .job-grid { - grid-template-columns: 1fr; + .job-stats div { + padding: 7px 5px; + } + + .job-log-entry { + grid-template-columns: 61px minmax(0, 1fr); + } + + .job-log-action { + grid-column: 2; + } + + .job-log-url { + grid-column: 2; + } + + .job-log-queue { + grid-column: 2; } .report-summary { grid-template-columns: repeat(2, minmax(0, 1fr)); } - .site-footer { - gap: 10px; + .report-summary div:nth-child(3) { + border-left: 0; + } + + .report-summary div:nth-child(-n + 2) { + border-bottom: 1px solid var(--line); } .footer-links { flex-wrap: wrap; - gap: 8px 14px; - white-space: normal; + gap: 8px 13px; } } -@media (max-width: 390px) { - .metric-grid { - grid-template-columns: 1fr; - } +/* Keep the interface readable in Geist Sans; reserve the mono face for live crawl output. */ +.skip-link, +.site-shell, +.site-shell button, +.site-shell input, +.report-dialog, +.report-dialog * { + font-family: var(--sans) !important; +} - .summary-value { - max-width: 122px; - } +.job-log, +.job-log * { + font-family: var(--mono) !important; } diff --git a/app/static/app.js b/app/static/app.js index 54c3777..51cdfc2 100644 --- a/app/static/app.js +++ b/app/static/app.js @@ -43,6 +43,9 @@ const TOKEN_PATTERN = /^[A-Za-z0-9_-]{24,200}$/; const jobs = new Map(); const streams = new Map(); +const jobLogs = new Map(); +const jobLogIds = new Map(); +const refreshTimers = new Map(); let storageAvailable = true; const ownerToken = getOwnerToken(); @@ -130,6 +133,8 @@ function rememberJob(job) { function forgetJob(jobId) { jobs.delete(jobId); closeEvents(jobId); + jobLogs.delete(jobId); + jobLogIds.delete(jobId); writeStoredSessions(readStoredSessions().filter((session) => session.id !== jobId)); renderJobs(); } @@ -161,6 +166,14 @@ function formatTime(value) { : date.toLocaleString([], { dateStyle: "medium", timeStyle: "short" }); } +function formatLogTime(value) { + if (!value) return "--:--:--"; + const date = new Date(value); + return Number.isNaN(date.getTime()) + ? "--:--:--" + : date.toLocaleTimeString([], { hour: "2-digit", minute: "2-digit", second: "2-digit" }); +} + function hostname(url) { try { return new URL(url).hostname; @@ -311,6 +324,8 @@ function jobProgress(job) { function createJobCard(job) { const card = createElement("article", "job-card"); card.dataset.state = job.state; + card.dataset.jobId = job.id; + card.classList.toggle("is-active", isActive(job)); const topLine = createElement("div", "job-topline"); topLine.append( @@ -326,11 +341,19 @@ function createJobCard(job) { progress.firstElementChild.style.setProperty("--progress", `${jobProgress(job)}%`); card.append(progress); + const latestPage = latestPageLog(job.id); + const pagesFound = Number(job.stats?.pages_found) || 0; + const pagesFetched = Number(job.stats?.pages_fetched) || 0; + const queuedPages = Number(latestPage?.detail?.pages_remaining); + const pagesRemaining = Number.isFinite(queuedPages) + ? queuedPages + : Math.max(0, pagesFound - pagesFetched); const stats = createElement("div", "job-stats"); const statItems = [ - [job.stats?.pages_fetched || 0, "pages"], - [job.stats?.files_written || 0, "files"], - [formatBytes(job.stats?.bytes_downloaded || 0), "collected"], + [pagesFetched, "pages fetched"], + [pagesFound, "pages found"], + [pagesRemaining, "pages remaining"], + [formatBytes(job.stats?.bytes_downloaded || 0), "downloaded"], ]; statItems.forEach(([value, label]) => { const item = document.createElement("div"); @@ -355,6 +378,11 @@ function createJobCard(job) { const tags = createElement("div", "job-tags"); const parallel = job.options?.parallel_connections; if (parallel) tags.append(createElement("span", "", `${parallel} connections`)); + const assetsFound = Number(job.stats?.assets_found) || 0; + const assetsFetched = Number(job.stats?.assets_fetched) || 0; + if (assetsFound || assetsFetched) { + tags.append(createElement("span", "", `${assetsFetched}/${assetsFound} assets`)); + } tags.append( createElement( "span", @@ -369,6 +397,8 @@ function createJobCard(job) { if (job.warnings?.length) card.append(createElement("p", "job-warning", job.warnings[0])); + if (isActive(job)) card.append(createJobLog(job.id)); + const actions = createElement("div", "job-actions"); if (job.archive?.available) { const download = createElement("button", "job-action primary", "Download ZIP"); @@ -403,15 +433,106 @@ function createJobCard(job) { return card; } +function latestPageLog(jobId) { + const entries = jobLogs.get(jobId) || []; + return [...entries].reverse().find((entry) => entry.detail?.kind === "page") || null; +} + +function createJobLog(jobId) { + const section = createElement("section", "job-log"); + const heading = createElement("div", "job-log-heading"); + heading.append( + createElement("span", "job-log-title", "Session log"), + createElement("span", "job-log-count", `${(jobLogs.get(jobId) || []).length} events`), + ); + const list = createElement("ol", "job-log-list"); + const entries = jobLogs.get(jobId) || []; + if (entries.length) entries.forEach((entry) => list.append(createLogEntry(entry))); + else list.append(createElement("li", "job-log-empty", "Waiting for worker activity...")); + section.append(heading, list); + return section; +} + +function createLogEntry(entry) { + const item = createElement("li", "job-log-entry"); + const detail = entry.detail || {}; + item.dataset.kind = detail.kind || entry.kind; + item.append(createElement("time", "job-log-time", formatLogTime(entry.logged_at))); + + if (detail.kind === "page") { + const action = createElement("span", "job-log-action", `page ${detail.action || "seen"}`); + const url = createElement("span", "job-log-url", String(detail.url || "unknown page")); + const queue = createElement( + "span", + "job-log-queue", + `queue ${detail.pages_remaining ?? 0}p / ${detail.assets_remaining ?? 0}a`, + ); + item.append(action, url, queue); + return item; + } + + item.append( + createElement("span", "job-log-action", String(entry.phase || entry.kind)), + createElement("span", "job-log-url", String(entry.message || "Worker update")), + ); + return item; +} + +function recordStreamEvent(jobId, kind, event) { + let payload; + try { + payload = JSON.parse(event.data); + } catch { + return; + } + if (kind === "progress" && !payload.detail) return; + + const eventId = event.lastEventId || `${kind}:${payload.logged_at}:${payload.message}`; + const seenIds = jobLogIds.get(jobId) || new Set(); + if (seenIds.has(eventId)) return; + seenIds.add(eventId); + jobLogIds.set(jobId, seenIds); + + const entries = jobLogs.get(jobId) || []; + const entry = { id: eventId, kind, ...payload }; + entries.push(entry); + jobLogs.set(jobId, entries); + appendLogEntry(jobId, entry, entries.length); +} + +function appendLogEntry(jobId, entry, count) { + const card = jobGrid.querySelector(`[data-job-id="${jobId}"]`); + const list = card?.querySelector(".job-log-list"); + if (!list) return; + const followTail = list.scrollTop + list.clientHeight >= list.scrollHeight - 16; + list.querySelector(".job-log-empty")?.remove(); + list.append(createLogEntry(entry)); + const counter = card.querySelector(".job-log-count"); + if (counter) counter.textContent = `${count} events`; + if (followTail) list.scrollTop = list.scrollHeight; +} + +function scheduleJobRefresh(jobId) { + if (refreshTimers.has(jobId)) return; + const timer = window.setTimeout(() => { + refreshTimers.delete(jobId); + void refreshJob(jobId); + }, 350); + refreshTimers.set(jobId, timer); +} + function connectEvents(jobId) { if (streams.has(jobId)) return; const eventUrl = `/api/captures/${encodeURIComponent(jobId)}/events?session=${encodeURIComponent(ownerToken)}`; const stream = new EventSource(eventUrl); streams.set(jobId, stream); - ["state", "progress"].forEach((eventName) => { - stream.addEventListener(eventName, () => { - void refreshJob(jobId); - }); + stream.addEventListener("state", (event) => { + recordStreamEvent(jobId, "state", event); + void refreshJob(jobId); + }); + stream.addEventListener("progress", (event) => { + recordStreamEvent(jobId, "progress", event); + scheduleJobRefresh(jobId); }); stream.addEventListener("terminal", () => { closeEvents(jobId); @@ -429,6 +550,9 @@ function closeEvents(jobId) { const stream = streams.get(jobId); if (stream) stream.close(); streams.delete(jobId); + const timer = refreshTimers.get(jobId); + if (timer) window.clearTimeout(timer); + refreshTimers.delete(jobId); } async function refreshJob(jobId) { diff --git a/app/templates/index.html b/app/templates/index.html index a21931b..15498f3 100644 --- a/app/templates/index.html +++ b/app/templates/index.html @@ -3,7 +3,7 @@ - + Website Downloader & Offline Archive Tool | {{ app_name }} + + + @@ -40,7 +46,7 @@ SiteHarbor - Private website archiver + v0.1 / private archive utility