__init__

app/static/app.js

@@ -0,0 +1,679 @@
+const form = document.querySelector("#capture-form");
+const urlInput = document.querySelector("#capture-url");
+const submitButton = document.querySelector("#capture-submit");
+const formMessage = document.querySelector("#form-message");
+const jobGrid = document.querySelector("#job-grid");
+const emptyState = document.querySelector("#empty-state");
+const refreshButton = document.querySelector("#refresh-jobs");
+const serviceStatus = document.querySelector("#service-status");
+const reportDialog = document.querySelector("#report-dialog");
+const reportContent = document.querySelector("#report-content");
+const closeReportButton = document.querySelector("#close-report");
+
+const controls = {
+  externalAssets: document.querySelector("#external-assets"),
+  parallelConnections: document.querySelector("#parallel-connections"),
+  parallelValue: document.querySelector("#parallel-value"),
+  maxPages: document.querySelector("#max-pages"),
+  maxDepth: document.querySelector("#max-depth"),
+  maxBytes: document.querySelector("#max-bytes"),
+  maxDuration: document.querySelector("#max-duration"),
+  retentionDays: document.querySelector("#retention-days"),
+  budgetSummary: document.querySelector("#budget-summary"),
+  assetTypes: [...document.querySelectorAll("[data-asset-type]")],
+  skipFileTypes: document.querySelector("#skip-file-types"),
+  useProxy: document.querySelector("#use-proxy"),
+  proxySummary: document.querySelector("#proxy-summary"),
+  proxyDetail: document.querySelector("#proxy-detail"),
+  proxyRow: document.querySelector(".proxy-row"),
+};
+
+const publicStats = {
+  websites: document.querySelector("#stat-websites"),
+  data: document.querySelector("#stat-data"),
+  active: document.querySelector("#stat-active"),
+  files: document.querySelector("#stat-files"),
+};
+
+const OWNER_STORAGE_KEY = "siteharbor.owner-token.v1";
+const SESSION_STORAGE_KEY = "siteharbor.sessions.v1";
+const OWNER_HEADER = "X-SiteHarbor-Session";
+const MAX_STORED_SESSIONS = 40;
+const TOKEN_PATTERN = /^[A-Za-z0-9_-]{24,200}$/;
+
+const jobs = new Map();
+const streams = new Map();
+let storageAvailable = true;
+const ownerToken = getOwnerToken();
+
+function createOwnerToken() {
+  if (window.crypto?.randomUUID) return window.crypto.randomUUID();
+  if (window.crypto?.getRandomValues) {
+    const bytes = new Uint8Array(24);
+    window.crypto.getRandomValues(bytes);
+    return [...bytes].map((byte) => byte.toString(16).padStart(2, "0")).join("");
+  }
+  return `browser-${Date.now()}-${Math.random().toString(36).slice(2).padEnd(16, "0")}`;
+}
+
+function getOwnerToken() {
+  try {
+    const existing = window.localStorage.getItem(OWNER_STORAGE_KEY);
+    if (existing && TOKEN_PATTERN.test(existing)) return existing;
+    const token = createOwnerToken();
+    window.localStorage.setItem(OWNER_STORAGE_KEY, token);
+    return token;
+  } catch {
+    storageAvailable = false;
+    return createOwnerToken();
+  }
+}
+
+function readStoredSessions() {
+  if (!storageAvailable) return [];
+  try {
+    const raw = window.localStorage.getItem(SESSION_STORAGE_KEY);
+    if (!raw) return [];
+    const parsed = JSON.parse(raw);
+    const sessions = Array.isArray(parsed) ? parsed : parsed.sessions;
+    if (!Array.isArray(sessions)) return [];
+    return sessions.filter((session) => typeof session?.id === "string" && session.id.length > 0);
+  } catch {
+    storageAvailable = false;
+    return [];
+  }
+}
+
+function writeStoredSessions(sessions) {
+  if (!storageAvailable) return;
+  try {
+    window.localStorage.setItem(
+      SESSION_STORAGE_KEY,
+      JSON.stringify({ version: 1, sessions: sessions.slice(0, MAX_STORED_SESSIONS) }),
+    );
+  } catch {
+    storageAvailable = false;
+  }
+}
+
+function snapshotJob(job) {
+  return {
+    id: job.id,
+    url: job.url,
+    state: job.state,
+    phase: job.phase,
+    message: job.message,
+    error_message: job.error_message,
+    warnings: Array.isArray(job.warnings) ? job.warnings.slice(0, 5) : [],
+    created_at: job.created_at,
+    updated_at: job.updated_at,
+    started_at: job.started_at,
+    completed_at: job.completed_at,
+    expires_at: job.expires_at,
+    cancel_requested: job.cancel_requested,
+    stats: job.stats || {},
+    options: job.options || {},
+    archive: job.archive || {},
+  };
+}
+
+function rememberJob(job) {
+  const sessions = readStoredSessions();
+  const record = { id: job.id, created_at: job.created_at, job: snapshotJob(job) };
+  const existingIndex = sessions.findIndex((session) => session.id === job.id);
+  if (existingIndex >= 0) sessions[existingIndex] = { ...sessions[existingIndex], ...record };
+  else sessions.unshift(record);
+  sessions.sort((left, right) => sortByCreated(right.job || right, left.job || left));
+  writeStoredSessions(sessions);
+}
+
+function forgetJob(jobId) {
+  jobs.delete(jobId);
+  closeEvents(jobId);
+  writeStoredSessions(readStoredSessions().filter((session) => session.id !== jobId));
+  renderJobs();
+}
+
+function sortByCreated(left, right) {
+  return String(left.created_at || "").localeCompare(String(right.created_at || ""));
+}
+
+function formatBytes(value) {
+  const amount = Number(value) || 0;
+  if (!amount) return "0 B";
+  const units = ["B", "KB", "MB", "GB", "TB"];
+  const index = Math.min(Math.floor(Math.log(amount) / Math.log(1024)), units.length - 1);
+  const rounded = amount / (1024 ** index);
+  return `${rounded >= 10 || index === 0 ? rounded.toFixed(0) : rounded.toFixed(1)} ${units[index]}`;
+}
+
+function formatCount(value) {
+  return new Intl.NumberFormat(undefined, { notation: "compact", maximumFractionDigits: 1 }).format(
+    Number(value) || 0,
+  );
+}
+
+function formatTime(value) {
+  if (!value) return "--";
+  const date = new Date(value);
+  return Number.isNaN(date.getTime())
+    ? "--"
+    : date.toLocaleString([], { dateStyle: "medium", timeStyle: "short" });
+}
+
+function hostname(url) {
+  try {
+    return new URL(url).hostname;
+  } catch {
+    return url;
+  }
+}
+
+function setMessage(message = "", kind = "") {
+  formMessage.textContent = message;
+  formMessage.className = `form-message ${kind}`.trim();
+}
+
+function setServiceStatus(message, online = false) {
+  serviceStatus.querySelector("span").textContent = message;
+  serviceStatus.dataset.online = String(online);
+}
+
+function readNumber(control, fallback) {
+  const value = Number.parseInt(control.value, 10);
+  return Number.isFinite(value) ? value : fallback;
+}
+
+function formatDuration(seconds) {
+  const value = Number(seconds) || 0;
+  if (value >= 60 && value % 60 === 0) return `${value / 60} min`;
+  return `${value} sec`;
+}
+
+function updateBudgetSummary() {
+  controls.budgetSummary.textContent = [
+    `${readNumber(controls.maxPages, 150)} pages`,
+    `${readNumber(controls.maxBytes, 100)} MB`,
+    formatDuration(readNumber(controls.maxDuration, 300)),
+  ].join(" / ");
+}
+
+function updateParallelValue() {
+  controls.parallelValue.value = controls.parallelConnections.value;
+  controls.parallelValue.textContent = controls.parallelConnections.value;
+}
+
+function updateProxySummary() {
+  if (controls.useProxy.disabled) {
+    controls.proxySummary.textContent = "Not configured";
+  } else if (controls.useProxy.checked) {
+    controls.proxySummary.textContent = "Approved proxy";
+  } else {
+    controls.proxySummary.textContent = "Direct connection";
+  }
+}
+
+function setControlLimits(capabilities) {
+  const defaults = capabilities.defaults || {};
+  const limits = capabilities.limits || {};
+  const bindings = [
+    [controls.maxPages, "max_pages"],
+    [controls.maxDepth, "max_depth"],
+    [controls.maxBytes, "max_bytes_mb"],
+    [controls.maxDuration, "max_duration_seconds"],
+    [controls.retentionDays, "max_retention_days"],
+  ];
+  bindings.forEach(([control, key]) => {
+    if (limits[key] !== undefined) control.max = limits[key];
+  });
+  if (limits.max_parallel_connections !== undefined) {
+    controls.parallelConnections.max = limits.max_parallel_connections;
+  }
+
+  controls.maxPages.value = defaults.max_pages ?? controls.maxPages.value;
+  controls.maxDepth.value = defaults.max_depth ?? controls.maxDepth.value;
+  controls.maxBytes.value = defaults.max_bytes_mb ?? controls.maxBytes.value;
+  controls.maxDuration.value = defaults.max_duration_seconds ?? controls.maxDuration.value;
+  controls.retentionDays.value = defaults.retention_days ?? controls.retentionDays.value;
+  controls.externalAssets.checked = defaults.include_external_assets ?? true;
+  controls.parallelConnections.value = defaults.parallel_connections ?? controls.parallelConnections.value;
+
+  const selectedTypes = new Set(defaults.download_asset_types || []);
+  controls.assetTypes.forEach((control) => {
+    control.checked = selectedTypes.has(control.dataset.assetType);
+  });
+  controls.skipFileTypes.value = (defaults.skip_file_types || []).join(", ");
+  controls.useProxy.checked = Boolean(defaults.use_proxy);
+  controls.useProxy.disabled = !capabilities.proxy_available;
+  controls.proxyRow.classList.toggle("is-unavailable", !capabilities.proxy_available);
+  controls.proxyDetail.textContent = capabilities.proxy_available
+    ? "Use the operator-approved route for this capture."
+    : "No operator-approved proxy route is available on this service.";
+
+  updateBudgetSummary();
+  updateParallelValue();
+  updateProxySummary();
+}
+
+async function request(path, options = {}, { session = true } = {}) {
+  const headers = new Headers(options.headers || {});
+  if (session) headers.set(OWNER_HEADER, ownerToken);
+  if (options.body && !headers.has("Content-Type")) headers.set("Content-Type", "application/json");
+
+  const response = await fetch(path, { cache: "no-store", ...options, headers });
+  const body = response.status === 204 ? null : await response.json().catch(() => null);
+  if (!response.ok) {
+    const error = new Error(body?.detail || "SiteHarbor could not complete that request.");
+    error.status = response.status;
+    throw error;
+  }
+  return body;
+}
+
+function isActive(job) {
+  return ["queued", "running"].includes(job.state);
+}
+
+function upsertJob(job, { persist = true, connect = true } = {}) {
+  jobs.set(job.id, job);
+  if (persist) rememberJob(job);
+  renderJobs();
+  if (connect && isActive(job)) connectEvents(job.id);
+  else if (!isActive(job)) closeEvents(job.id);
+}
+
+function renderJobs() {
+  const orderedJobs = [...jobs.values()].sort((left, right) => sortByCreated(right, left));
+  emptyState.hidden = orderedJobs.length > 0;
+  jobGrid.replaceChildren(...orderedJobs.map(createJobCard));
+}
+
+function createElement(tag, className, text) {
+  const element = document.createElement(tag);
+  if (className) element.className = className;
+  if (text !== undefined) element.textContent = text;
+  return element;
+}
+
+function jobProgress(job) {
+  if (job.state === "ready") return 100;
+  if (["failed", "cancelled", "expired"].includes(job.state)) return 100;
+  const phase = String(job.phase || "").toLowerCase();
+  if (phase.includes("packag")) return 94;
+  if (phase.includes("rewrit")) return 88;
+  if (phase.includes("validat") || phase.includes("prepar")) return 6;
+  const maximum = Number(job.options?.max_pages) || 0;
+  const fetched = Number(job.stats?.pages_fetched) || 0;
+  if (!maximum) return 16;
+  return Math.max(10, Math.min(82, Math.round((fetched / maximum) * 82)));
+}
+
+function createJobCard(job) {
+  const card = createElement("article", "job-card");
+  card.dataset.state = job.state;
+
+  const topLine = createElement("div", "job-topline");
+  topLine.append(
+    createElement("div", "job-host", hostname(job.url)),
+    createElement("span", `state-pill ${job.state}`, job.phase || job.state),
+  );
+  card.append(topLine);
+  card.append(createElement("p", "job-url", job.url));
+  card.append(createElement("p", "job-phase", job.error_message || job.message || "Waiting for status"));
+
+  const progress = createElement("div", "job-progress");
+  progress.append(createElement("span"));
+  progress.firstElementChild.style.setProperty("--progress", `${jobProgress(job)}%`);
+  card.append(progress);
+
+  const stats = createElement("div", "job-stats");
+  const statItems = [
+    [job.stats?.pages_fetched || 0, "pages"],
+    [job.stats?.files_written || 0, "files"],
+    [formatBytes(job.stats?.bytes_downloaded || 0), "collected"],
+  ];
+  statItems.forEach(([value, label]) => {
+    const item = document.createElement("div");
+    item.append(createElement("strong", "", String(value)), createElement("span", "", label));
+    stats.append(item);
+  });
+  card.append(stats);
+
+  const meta = createElement("div", "job-meta");
+  meta.append(
+    createElement("span", "", `Created ${formatTime(job.created_at)}`),
+    createElement(
+      "span",
+      "",
+      job.archive?.available
+        ? `Archive ${formatBytes(job.archive.size_bytes)}`
+        : `${job.options?.max_pages || "--"} page budget`,
+    ),
+  );
+  card.append(meta);
+
+  const tags = createElement("div", "job-tags");
+  const parallel = job.options?.parallel_connections;
+  if (parallel) tags.append(createElement("span", "", `${parallel} connections`));
+  tags.append(
+    createElement(
+      "span",
+      "",
+      job.options?.include_external_assets ? "external assets" : "first-party only",
+    ),
+  );
+  if (job.options?.skip_file_types?.length) {
+    tags.append(createElement("span", "", `skip .${job.options.skip_file_types[0]}`));
+  }
+  card.append(tags);
+
+  if (job.warnings?.length) card.append(createElement("p", "job-warning", job.warnings[0]));
+
+  const actions = createElement("div", "job-actions");
+  if (job.archive?.available) {
+    const download = createElement("button", "job-action primary", "Download ZIP");
+    download.type = "button";
+    download.addEventListener("click", () => downloadArchive(job, download));
+    actions.append(download);
+
+    const report = createElement("button", "job-action", "View report");
+    report.type = "button";
+    report.addEventListener("click", () => showReport(job.id));
+    actions.append(report);
+
+    const remove = createElement("button", "job-action danger", "Delete archive");
+    remove.type = "button";
+    remove.addEventListener("click", () => deleteJob(job.id));
+    actions.append(remove);
+  }
+  if (isActive(job)) {
+    const cancel = createElement("button", "job-action danger", job.cancel_requested ? "Cancelling" : "Cancel");
+    cancel.type = "button";
+    cancel.disabled = Boolean(job.cancel_requested);
+    cancel.addEventListener("click", () => cancelJob(job.id));
+    actions.append(cancel);
+  }
+  if (!isActive(job) && !job.archive?.available) {
+    const forget = createElement("button", "job-action", "Forget session");
+    forget.type = "button";
+    forget.addEventListener("click", () => forgetJob(job.id));
+    actions.append(forget);
+  }
+  card.append(actions);
+  return card;
+}
+
+function connectEvents(jobId) {
+  if (streams.has(jobId)) return;
+  const eventUrl = `/api/captures/${encodeURIComponent(jobId)}/events?session=${encodeURIComponent(ownerToken)}`;
+  const stream = new EventSource(eventUrl);
+  streams.set(jobId, stream);
+  ["state", "progress"].forEach((eventName) => {
+    stream.addEventListener(eventName, () => {
+      void refreshJob(jobId);
+    });
+  });
+  stream.addEventListener("terminal", () => {
+    closeEvents(jobId);
+    void refreshJob(jobId);
+    void loadPublicStats();
+  });
+  stream.onerror = () => {
+    window.setTimeout(() => {
+      if (streams.get(jobId) === stream) void refreshJob(jobId);
+    }, 1_000);
+  };
+}
+
+function closeEvents(jobId) {
+  const stream = streams.get(jobId);
+  if (stream) stream.close();
+  streams.delete(jobId);
+}
+
+async function refreshJob(jobId) {
+  try {
+    upsertJob(await request(`/api/captures/${encodeURIComponent(jobId)}`));
+  } catch (error) {
+    closeEvents(jobId);
+    if (error.status === 404) forgetJob(jobId);
+  }
+}
+
+async function refreshKnownJobs() {
+  const storedIds = readStoredSessions().map((session) => session.id);
+  const knownIds = [...new Set([...storedIds, ...jobs.keys()])];
+  await Promise.all(knownIds.map((jobId) => refreshJob(jobId)));
+}
+
+async function loadStoredJobs() {
+  const sessions = readStoredSessions();
+  sessions.forEach((session) => {
+    if (session.job?.id && !jobs.has(session.id)) jobs.set(session.id, session.job);
+  });
+  renderJobs();
+  await refreshKnownJobs();
+}
+
+async function loadPublicStats() {
+  try {
+    const stats = await request("/api/stats", {}, { session: false });
+    publicStats.websites.textContent = formatCount(stats.websites_cloned);
+    publicStats.data.textContent = formatBytes(stats.bytes_scraped);
+    publicStats.active.textContent = formatCount(stats.active_crawls);
+    publicStats.files.textContent = formatCount(stats.files_archived);
+    setServiceStatus("Capture service online", true);
+  } catch {
+    setServiceStatus("Service unavailable", false);
+  }
+}
+
+async function cancelJob(jobId) {
+  try {
+    upsertJob(await request(`/api/captures/${encodeURIComponent(jobId)}/cancel`, { method: "POST" }));
+    void loadPublicStats();
+  } catch (error) {
+    setMessage(error.message);
+  }
+}
+
+async function deleteJob(jobId) {
+  try {
+    const updated = await request(`/api/captures/${encodeURIComponent(jobId)}`, { method: "DELETE" });
+    if (updated.state === "expired") forgetJob(jobId);
+    else upsertJob(updated);
+    void loadPublicStats();
+  } catch (error) {
+    setMessage(error.message);
+  }
+}
+
+async function downloadArchive(job, button) {
+  if (!job.archive?.download_url) return;
+  const originalText = button.textContent;
+  button.disabled = true;
+  button.textContent = "Preparing download";
+  try {
+    const response = await fetch(job.archive.download_url, {
+      headers: { [OWNER_HEADER]: ownerToken },
+    });
+    if (!response.ok) {
+      const body = await response.json().catch(() => null);
+      const error = new Error(body?.detail || "The archive could not be downloaded.");
+      error.status = response.status;
+      throw error;
+    }
+    const objectUrl = URL.createObjectURL(await response.blob());
+    const anchor = document.createElement("a");
+    anchor.href = objectUrl;
+    anchor.download = `siteharbor-${job.id}.zip`;
+    document.body.append(anchor);
+    anchor.click();
+    anchor.remove();
+    window.setTimeout(() => URL.revokeObjectURL(objectUrl), 1_000);
+  } catch (error) {
+    setMessage(error.message);
+  } finally {
+    button.disabled = false;
+    button.textContent = originalText;
+  }
+}
+
+async function showReport(jobId) {
+  try {
+    const report = await request(`/api/captures/${encodeURIComponent(jobId)}/report`);
+    renderReport(report);
+    reportDialog.showModal();
+  } catch (error) {
+    setMessage(error.message);
+  }
+}
+
+function renderReport(report) {
+  reportContent.replaceChildren();
+  const summary = createElement("div", "report-summary");
+  const stats = report.stats || {};
+  [
+    [stats.pages_fetched || 0, "pages"],
+    [stats.assets_fetched || 0, "assets"],
+    [formatBytes(stats.bytes_downloaded || 0), "downloaded"],
+    [report.quality || "unknown", "capture quality"],
+  ].forEach(([value, label]) => {
+    const item = document.createElement("div");
+    item.append(createElement("strong", "", String(value)), createElement("span", "", label));
+    summary.append(item);
+  });
+  reportContent.append(summary);
+
+  const details = createElement("ul", "report-list");
+  [
+    `Entry point: ${report.entry_point || "not available"}`,
+    `Capture duration: ${report.elapsed_seconds || 0} seconds`,
+    `Captured resources: ${(report.resources || []).length}`,
+  ].forEach((line) => details.append(createElement("li", "", line)));
+  reportContent.append(details);
+
+  if (report.warnings?.length) {
+    reportContent.append(createElement("h3", "report-subhead", "Warnings"));
+    const warnings = createElement("ul", "report-list");
+    report.warnings.forEach((warning) => warnings.append(createElement("li", "", warning)));
+    reportContent.append(warnings);
+  }
+  if (report.errors?.length) {
+    reportContent.append(createElement("h3", "report-subhead", "Resource failures"));
+    const errors = createElement("ul", "report-list");
+    report.errors.slice(0, 20).forEach((error) => {
+      errors.append(createElement("li", "", `${error.url}: ${error.message}`));
+    });
+    reportContent.append(errors);
+  }
+}
+
+function selectedAssetTypes() {
+  return controls.assetTypes
+    .filter((control) => control.checked)
+    .map((control) => control.dataset.assetType);
+}
+
+function skippedFileTypes() {
+  return controls.skipFileTypes.value
+    .split(",")
+    .map((item) => item.trim())
+    .filter(Boolean);
+}
+
+form.addEventListener("submit", async (event) => {
+  event.preventDefault();
+  const url = urlInput.value.trim();
+  if (!url) {
+    setMessage("Enter a public website URL to begin.");
+    urlInput.focus();
+    return;
+  }
+
+  setMessage("");
+  submitButton.disabled = true;
+  submitButton.querySelector("span").textContent = "Creating crawl";
+  try {
+    const job = await request("/api/captures", {
+      method: "POST",
+      body: JSON.stringify({
+        url,
+        include_external_assets: controls.externalAssets.checked,
+        parallel_connections: readNumber(controls.parallelConnections, 6),
+        download_asset_types: selectedAssetTypes(),
+        skip_file_types: skippedFileTypes(),
+        use_proxy: controls.useProxy.checked && !controls.useProxy.disabled,
+        max_pages: readNumber(controls.maxPages, 150),
+        max_depth: readNumber(controls.maxDepth, 5),
+        max_bytes_mb: readNumber(controls.maxBytes, 100),
+        max_duration_seconds: readNumber(controls.maxDuration, 300),
+        retention_days: readNumber(controls.retentionDays, 7),
+      }),
+    });
+    upsertJob(job);
+    setMessage("Crawl queued. This browser will reconnect to it automatically.", "success");
+    urlInput.value = "";
+    void loadPublicStats();
+  } catch (error) {
+    setMessage(error.message);
+  } finally {
+    submitButton.disabled = false;
+    submitButton.querySelector("span").textContent = "Start private crawl";
+  }
+});
+
+[
+  controls.maxPages,
+  controls.maxDepth,
+  controls.maxBytes,
+  controls.maxDuration,
+  controls.retentionDays,
+].forEach((control) => control.addEventListener("input", updateBudgetSummary));
+controls.parallelConnections.addEventListener("input", updateParallelValue);
+controls.useProxy.addEventListener("change", updateProxySummary);
+
+refreshButton.addEventListener("click", async () => {
+  refreshButton.disabled = true;
+  try {
+    await Promise.all([refreshKnownJobs(), loadPublicStats()]);
+  } finally {
+    refreshButton.disabled = false;
+  }
+});
+closeReportButton.addEventListener("click", () => reportDialog.close());
+reportDialog.addEventListener("click", (event) => {
+  if (event.target === reportDialog) reportDialog.close();
+});
+
+document.addEventListener("visibilitychange", () => {
+  if (!document.hidden) {
+    void refreshKnownJobs();
+    void loadPublicStats();
+  }
+});
+window.addEventListener("online", () => {
+  void refreshKnownJobs();
+  void loadPublicStats();
+});
+window.addEventListener("beforeunload", () => {
+  streams.forEach((stream) => stream.close());
+});
+
+async function initialize() {
+  if (!storageAvailable) {
+    setMessage("Browser storage is unavailable, so this session cannot be restored after this tab closes.");
+  }
+  try {
+    setControlLimits(await request("/api/capabilities", {}, { session: false }));
+  } catch (error) {
+    setMessage(error.message);
+  }
+  await Promise.all([loadStoredJobs(), loadPublicStats()]);
+  window.setInterval(() => {
+    void refreshKnownJobs();
+    void loadPublicStats();
+  }, 15_000);
+}
+
+void initialize();