kstyagi23
2059c275c2
- Implement job logging functionality to track progress and events for each job. - Add a new section in the job card to display session logs with detailed entries. - Update job statistics to include pages fetched, pages found, and assets processed. - Modify the UI theme and text for better clarity and aesthetics. - Adjust tests to validate new logging features and ensure proper event handling.
817 lines
28 KiB
JavaScript
817 lines
28 KiB
JavaScript
const form = document.querySelector("#capture-form");
|
|
const urlInput = document.querySelector("#capture-url");
|
|
const submitButton = document.querySelector("#capture-submit");
|
|
const formMessage = document.querySelector("#form-message");
|
|
const jobGrid = document.querySelector("#job-grid");
|
|
const emptyState = document.querySelector("#empty-state");
|
|
const refreshButton = document.querySelector("#refresh-jobs");
|
|
const serviceStatus = document.querySelector("#service-status");
|
|
const reportDialog = document.querySelector("#report-dialog");
|
|
const reportContent = document.querySelector("#report-content");
|
|
const closeReportButton = document.querySelector("#close-report");
|
|
|
|
const controls = {
|
|
externalAssets: document.querySelector("#external-assets"),
|
|
parallelConnections: document.querySelector("#parallel-connections"),
|
|
parallelValue: document.querySelector("#parallel-value"),
|
|
maxPages: document.querySelector("#max-pages"),
|
|
maxDepth: document.querySelector("#max-depth"),
|
|
maxBytes: document.querySelector("#max-bytes"),
|
|
maxDuration: document.querySelector("#max-duration"),
|
|
retentionDays: document.querySelector("#retention-days"),
|
|
budgetSummary: document.querySelector("#budget-summary"),
|
|
assetTypes: [...document.querySelectorAll("[data-asset-type]")],
|
|
skipFileTypes: document.querySelector("#skip-file-types"),
|
|
useProxy: document.querySelector("#use-proxy"),
|
|
proxySummary: document.querySelector("#proxy-summary"),
|
|
proxyDetail: document.querySelector("#proxy-detail"),
|
|
proxyRow: document.querySelector(".proxy-row"),
|
|
};
|
|
|
|
const publicStats = {
|
|
websites: document.querySelector("#stat-websites"),
|
|
data: document.querySelector("#stat-data"),
|
|
active: document.querySelector("#stat-active"),
|
|
files: document.querySelector("#stat-files"),
|
|
};
|
|
|
|
const OWNER_STORAGE_KEY = "siteharbor.owner-token.v1";
|
|
const SESSION_STORAGE_KEY = "siteharbor.sessions.v1";
|
|
const OWNER_HEADER = "X-SiteHarbor-Session";
|
|
const MAX_STORED_SESSIONS = 40;
|
|
const TOKEN_PATTERN = /^[A-Za-z0-9_-]{24,200}$/;
|
|
|
|
const jobs = new Map();
|
|
const streams = new Map();
|
|
const jobLogs = new Map();
|
|
const jobLogIds = new Map();
|
|
const refreshTimers = new Map();
|
|
let storageAvailable = true;
|
|
const ownerToken = getOwnerToken();
|
|
|
|
function createOwnerToken() {
|
|
if (window.crypto?.randomUUID) return window.crypto.randomUUID();
|
|
if (window.crypto?.getRandomValues) {
|
|
const bytes = new Uint8Array(24);
|
|
window.crypto.getRandomValues(bytes);
|
|
return [...bytes].map((byte) => byte.toString(16).padStart(2, "0")).join("");
|
|
}
|
|
return `browser-${Date.now()}-${Math.random().toString(36).slice(2).padEnd(16, "0")}`;
|
|
}
|
|
|
|
function getOwnerToken() {
|
|
try {
|
|
const existing = window.localStorage.getItem(OWNER_STORAGE_KEY);
|
|
if (existing && TOKEN_PATTERN.test(existing)) return existing;
|
|
const token = createOwnerToken();
|
|
window.localStorage.setItem(OWNER_STORAGE_KEY, token);
|
|
return token;
|
|
} catch {
|
|
storageAvailable = false;
|
|
return createOwnerToken();
|
|
}
|
|
}
|
|
|
|
function readStoredSessions() {
|
|
if (!storageAvailable) return [];
|
|
try {
|
|
const raw = window.localStorage.getItem(SESSION_STORAGE_KEY);
|
|
if (!raw) return [];
|
|
const parsed = JSON.parse(raw);
|
|
const sessions = Array.isArray(parsed) ? parsed : parsed.sessions;
|
|
if (!Array.isArray(sessions)) return [];
|
|
return sessions.filter((session) => typeof session?.id === "string" && session.id.length > 0);
|
|
} catch {
|
|
storageAvailable = false;
|
|
return [];
|
|
}
|
|
}
|
|
|
|
function writeStoredSessions(sessions) {
|
|
if (!storageAvailable) return;
|
|
try {
|
|
window.localStorage.setItem(
|
|
SESSION_STORAGE_KEY,
|
|
JSON.stringify({ version: 1, sessions: sessions.slice(0, MAX_STORED_SESSIONS) }),
|
|
);
|
|
} catch {
|
|
storageAvailable = false;
|
|
}
|
|
}
|
|
|
|
function snapshotJob(job) {
|
|
return {
|
|
id: job.id,
|
|
url: job.url,
|
|
state: job.state,
|
|
phase: job.phase,
|
|
message: job.message,
|
|
error_message: job.error_message,
|
|
warnings: Array.isArray(job.warnings) ? job.warnings.slice(0, 5) : [],
|
|
created_at: job.created_at,
|
|
updated_at: job.updated_at,
|
|
started_at: job.started_at,
|
|
completed_at: job.completed_at,
|
|
expires_at: job.expires_at,
|
|
cancel_requested: job.cancel_requested,
|
|
stats: job.stats || {},
|
|
options: job.options || {},
|
|
archive: job.archive || {},
|
|
};
|
|
}
|
|
|
|
function rememberJob(job) {
|
|
const sessions = readStoredSessions();
|
|
const record = { id: job.id, created_at: job.created_at, job: snapshotJob(job) };
|
|
const existingIndex = sessions.findIndex((session) => session.id === job.id);
|
|
if (existingIndex >= 0) sessions[existingIndex] = { ...sessions[existingIndex], ...record };
|
|
else sessions.unshift(record);
|
|
sessions.sort((left, right) => sortByCreated(right.job || right, left.job || left));
|
|
writeStoredSessions(sessions);
|
|
}
|
|
|
|
function forgetJob(jobId) {
|
|
jobs.delete(jobId);
|
|
closeEvents(jobId);
|
|
jobLogs.delete(jobId);
|
|
jobLogIds.delete(jobId);
|
|
writeStoredSessions(readStoredSessions().filter((session) => session.id !== jobId));
|
|
renderJobs();
|
|
}
|
|
|
|
function sortByCreated(left, right) {
|
|
return String(left.created_at || "").localeCompare(String(right.created_at || ""));
|
|
}
|
|
|
|
function formatBytes(value) {
|
|
const amount = Number(value) || 0;
|
|
if (!amount) return "0 B";
|
|
const units = ["B", "KB", "MB", "GB", "TB"];
|
|
const index = Math.min(Math.floor(Math.log(amount) / Math.log(1024)), units.length - 1);
|
|
const rounded = amount / (1024 ** index);
|
|
return `${rounded >= 10 || index === 0 ? rounded.toFixed(0) : rounded.toFixed(1)} ${units[index]}`;
|
|
}
|
|
|
|
function formatCount(value) {
|
|
return new Intl.NumberFormat(undefined, { notation: "compact", maximumFractionDigits: 1 }).format(
|
|
Number(value) || 0,
|
|
);
|
|
}
|
|
|
|
function formatTime(value) {
|
|
if (!value) return "--";
|
|
const date = new Date(value);
|
|
return Number.isNaN(date.getTime())
|
|
? "--"
|
|
: date.toLocaleString([], { dateStyle: "medium", timeStyle: "short" });
|
|
}
|
|
|
|
function formatLogTime(value) {
|
|
if (!value) return "--:--:--";
|
|
const date = new Date(value);
|
|
return Number.isNaN(date.getTime())
|
|
? "--:--:--"
|
|
: date.toLocaleTimeString([], { hour: "2-digit", minute: "2-digit", second: "2-digit" });
|
|
}
|
|
|
|
function hostname(url) {
|
|
try {
|
|
return new URL(url).hostname;
|
|
} catch {
|
|
return url;
|
|
}
|
|
}
|
|
|
|
function setMessage(message = "", kind = "") {
|
|
formMessage.textContent = message;
|
|
formMessage.className = `form-message ${kind}`.trim();
|
|
}
|
|
|
|
function setServiceStatus(message, online = false) {
|
|
serviceStatus.querySelector("span").textContent = message;
|
|
serviceStatus.dataset.online = String(online);
|
|
}
|
|
|
|
function readNumber(control, fallback) {
|
|
const value = Number.parseInt(control.value, 10);
|
|
return Number.isFinite(value) ? value : fallback;
|
|
}
|
|
|
|
function formatDuration(seconds) {
|
|
const value = Number(seconds) || 0;
|
|
if (value >= 60 && value % 60 === 0) return `${value / 60} min`;
|
|
return `${value} sec`;
|
|
}
|
|
|
|
function updateBudgetSummary() {
|
|
controls.budgetSummary.textContent = [
|
|
`${readNumber(controls.maxPages, 150)} pages`,
|
|
`${readNumber(controls.maxBytes, 100)} MB`,
|
|
formatDuration(readNumber(controls.maxDuration, 300)),
|
|
].join(" / ");
|
|
}
|
|
|
|
function updateParallelValue() {
|
|
controls.parallelValue.value = controls.parallelConnections.value;
|
|
controls.parallelValue.textContent = controls.parallelConnections.value;
|
|
}
|
|
|
|
function updateProxySummary() {
|
|
if (controls.useProxy.disabled) {
|
|
controls.proxySummary.textContent = "Not configured";
|
|
} else if (controls.useProxy.checked) {
|
|
controls.proxySummary.textContent = "Approved proxy";
|
|
} else {
|
|
controls.proxySummary.textContent = "Direct connection";
|
|
}
|
|
}
|
|
|
|
function setControlLimits(capabilities) {
|
|
const defaults = capabilities.defaults || {};
|
|
const limits = capabilities.limits || {};
|
|
const bindings = [
|
|
[controls.maxPages, "max_pages"],
|
|
[controls.maxDepth, "max_depth"],
|
|
[controls.maxBytes, "max_bytes_mb"],
|
|
[controls.maxDuration, "max_duration_seconds"],
|
|
[controls.retentionDays, "max_retention_days"],
|
|
];
|
|
bindings.forEach(([control, key]) => {
|
|
if (limits[key] !== undefined) control.max = limits[key];
|
|
});
|
|
if (limits.max_parallel_connections !== undefined) {
|
|
controls.parallelConnections.max = limits.max_parallel_connections;
|
|
}
|
|
|
|
controls.maxPages.value = defaults.max_pages ?? controls.maxPages.value;
|
|
controls.maxDepth.value = defaults.max_depth ?? controls.maxDepth.value;
|
|
controls.maxBytes.value = defaults.max_bytes_mb ?? controls.maxBytes.value;
|
|
controls.maxDuration.value = defaults.max_duration_seconds ?? controls.maxDuration.value;
|
|
controls.retentionDays.value = defaults.retention_days ?? controls.retentionDays.value;
|
|
controls.externalAssets.checked = defaults.include_external_assets ?? true;
|
|
controls.parallelConnections.value = defaults.parallel_connections ?? controls.parallelConnections.value;
|
|
|
|
const selectedTypes = new Set(defaults.download_asset_types || []);
|
|
controls.assetTypes.forEach((control) => {
|
|
control.checked = selectedTypes.has(control.dataset.assetType);
|
|
});
|
|
controls.skipFileTypes.value = (defaults.skip_file_types || []).join(", ");
|
|
controls.useProxy.checked = Boolean(defaults.use_proxy);
|
|
controls.useProxy.disabled = !capabilities.proxy_available;
|
|
controls.proxyRow.classList.toggle("is-unavailable", !capabilities.proxy_available);
|
|
controls.proxyDetail.textContent = capabilities.proxy_available
|
|
? "Use the operator-approved route for this capture."
|
|
: "No operator-approved proxy route is available on this service.";
|
|
|
|
updateBudgetSummary();
|
|
updateParallelValue();
|
|
updateProxySummary();
|
|
}
|
|
|
|
async function request(path, options = {}, { session = true } = {}) {
|
|
const headers = new Headers(options.headers || {});
|
|
if (session) headers.set(OWNER_HEADER, ownerToken);
|
|
if (options.body && !headers.has("Content-Type")) headers.set("Content-Type", "application/json");
|
|
|
|
const response = await fetch(path, { cache: "no-store", ...options, headers });
|
|
const body = response.status === 204 ? null : await response.json().catch(() => null);
|
|
if (!response.ok) {
|
|
const error = new Error(body?.detail || "SiteHarbor could not complete that request.");
|
|
error.status = response.status;
|
|
throw error;
|
|
}
|
|
return body;
|
|
}
|
|
|
|
function isActive(job) {
|
|
return ["queued", "running"].includes(job.state);
|
|
}
|
|
|
|
function upsertJob(job, { persist = true, connect = true } = {}) {
|
|
jobs.set(job.id, job);
|
|
if (persist) rememberJob(job);
|
|
renderJobs();
|
|
if (connect && isActive(job)) connectEvents(job.id);
|
|
else if (!isActive(job)) closeEvents(job.id);
|
|
}
|
|
|
|
function renderJobs() {
|
|
const orderedJobs = [...jobs.values()].sort((left, right) => sortByCreated(right, left));
|
|
emptyState.hidden = orderedJobs.length > 0;
|
|
jobGrid.replaceChildren(...orderedJobs.map(createJobCard));
|
|
}
|
|
|
|
function createElement(tag, className, text) {
|
|
const element = document.createElement(tag);
|
|
if (className) element.className = className;
|
|
if (text !== undefined) element.textContent = text;
|
|
return element;
|
|
}
|
|
|
|
function jobProgress(job) {
|
|
if (job.state === "ready") return 100;
|
|
if (["failed", "cancelled", "expired"].includes(job.state)) return 100;
|
|
const phase = String(job.phase || "").toLowerCase();
|
|
if (phase.includes("packag")) return 94;
|
|
if (phase.includes("rewrit")) return 88;
|
|
if (phase.includes("validat") || phase.includes("prepar")) return 6;
|
|
const maximum = Number(job.options?.max_pages) || 0;
|
|
const fetched = Number(job.stats?.pages_fetched) || 0;
|
|
if (!maximum) return 16;
|
|
return Math.max(10, Math.min(82, Math.round((fetched / maximum) * 82)));
|
|
}
|
|
|
|
function createJobCard(job) {
|
|
const card = createElement("article", "job-card");
|
|
card.dataset.state = job.state;
|
|
card.dataset.jobId = job.id;
|
|
card.classList.toggle("is-active", isActive(job));
|
|
|
|
const topLine = createElement("div", "job-topline");
|
|
topLine.append(
|
|
createElement("div", "job-host", hostname(job.url)),
|
|
createElement("span", `state-pill ${job.state}`, job.phase || job.state),
|
|
);
|
|
card.append(topLine);
|
|
card.append(createElement("p", "job-url", job.url));
|
|
card.append(createElement("p", "job-phase", job.error_message || job.message || "Waiting for status"));
|
|
|
|
const progress = createElement("div", "job-progress");
|
|
progress.append(createElement("span"));
|
|
progress.firstElementChild.style.setProperty("--progress", `${jobProgress(job)}%`);
|
|
card.append(progress);
|
|
|
|
const latestPage = latestPageLog(job.id);
|
|
const pagesFound = Number(job.stats?.pages_found) || 0;
|
|
const pagesFetched = Number(job.stats?.pages_fetched) || 0;
|
|
const queuedPages = Number(latestPage?.detail?.pages_remaining);
|
|
const pagesRemaining = Number.isFinite(queuedPages)
|
|
? queuedPages
|
|
: Math.max(0, pagesFound - pagesFetched);
|
|
const stats = createElement("div", "job-stats");
|
|
const statItems = [
|
|
[pagesFetched, "pages fetched"],
|
|
[pagesFound, "pages found"],
|
|
[pagesRemaining, "pages remaining"],
|
|
[formatBytes(job.stats?.bytes_downloaded || 0), "downloaded"],
|
|
];
|
|
statItems.forEach(([value, label]) => {
|
|
const item = document.createElement("div");
|
|
item.append(createElement("strong", "", String(value)), createElement("span", "", label));
|
|
stats.append(item);
|
|
});
|
|
card.append(stats);
|
|
|
|
const meta = createElement("div", "job-meta");
|
|
meta.append(
|
|
createElement("span", "", `Created ${formatTime(job.created_at)}`),
|
|
createElement(
|
|
"span",
|
|
"",
|
|
job.archive?.available
|
|
? `Archive ${formatBytes(job.archive.size_bytes)}`
|
|
: `${job.options?.max_pages || "--"} page budget`,
|
|
),
|
|
);
|
|
card.append(meta);
|
|
|
|
const tags = createElement("div", "job-tags");
|
|
const parallel = job.options?.parallel_connections;
|
|
if (parallel) tags.append(createElement("span", "", `${parallel} connections`));
|
|
const assetsFound = Number(job.stats?.assets_found) || 0;
|
|
const assetsFetched = Number(job.stats?.assets_fetched) || 0;
|
|
if (assetsFound || assetsFetched) {
|
|
tags.append(createElement("span", "", `${assetsFetched}/${assetsFound} assets`));
|
|
}
|
|
tags.append(
|
|
createElement(
|
|
"span",
|
|
"",
|
|
job.options?.include_external_assets ? "external assets" : "first-party only",
|
|
),
|
|
);
|
|
if (job.options?.skip_file_types?.length) {
|
|
tags.append(createElement("span", "", `skip .${job.options.skip_file_types[0]}`));
|
|
}
|
|
card.append(tags);
|
|
|
|
if (job.warnings?.length) card.append(createElement("p", "job-warning", job.warnings[0]));
|
|
|
|
if (isActive(job)) card.append(createJobLog(job.id));
|
|
|
|
const actions = createElement("div", "job-actions");
|
|
if (job.archive?.available) {
|
|
const download = createElement("button", "job-action primary", "Download ZIP");
|
|
download.type = "button";
|
|
download.addEventListener("click", () => downloadArchive(job, download));
|
|
actions.append(download);
|
|
|
|
const report = createElement("button", "job-action", "View report");
|
|
report.type = "button";
|
|
report.addEventListener("click", () => showReport(job.id));
|
|
actions.append(report);
|
|
|
|
const remove = createElement("button", "job-action danger", "Delete archive");
|
|
remove.type = "button";
|
|
remove.addEventListener("click", () => deleteJob(job.id));
|
|
actions.append(remove);
|
|
}
|
|
if (isActive(job)) {
|
|
const cancel = createElement("button", "job-action danger", job.cancel_requested ? "Cancelling" : "Cancel");
|
|
cancel.type = "button";
|
|
cancel.disabled = Boolean(job.cancel_requested);
|
|
cancel.addEventListener("click", () => cancelJob(job.id));
|
|
actions.append(cancel);
|
|
}
|
|
if (!isActive(job) && !job.archive?.available) {
|
|
const forget = createElement("button", "job-action", "Forget session");
|
|
forget.type = "button";
|
|
forget.addEventListener("click", () => forgetJob(job.id));
|
|
actions.append(forget);
|
|
}
|
|
card.append(actions);
|
|
return card;
|
|
}
|
|
|
|
function latestPageLog(jobId) {
|
|
const entries = jobLogs.get(jobId) || [];
|
|
return [...entries].reverse().find((entry) => entry.detail?.kind === "page") || null;
|
|
}
|
|
|
|
function createJobLog(jobId) {
|
|
const section = createElement("section", "job-log");
|
|
const heading = createElement("div", "job-log-heading");
|
|
heading.append(
|
|
createElement("span", "job-log-title", "Session log"),
|
|
createElement("span", "job-log-count", `${(jobLogs.get(jobId) || []).length} events`),
|
|
);
|
|
const list = createElement("ol", "job-log-list");
|
|
const entries = jobLogs.get(jobId) || [];
|
|
if (entries.length) entries.forEach((entry) => list.append(createLogEntry(entry)));
|
|
else list.append(createElement("li", "job-log-empty", "Waiting for worker activity..."));
|
|
section.append(heading, list);
|
|
return section;
|
|
}
|
|
|
|
function createLogEntry(entry) {
|
|
const item = createElement("li", "job-log-entry");
|
|
const detail = entry.detail || {};
|
|
item.dataset.kind = detail.kind || entry.kind;
|
|
item.append(createElement("time", "job-log-time", formatLogTime(entry.logged_at)));
|
|
|
|
if (detail.kind === "page") {
|
|
const action = createElement("span", "job-log-action", `page ${detail.action || "seen"}`);
|
|
const url = createElement("span", "job-log-url", String(detail.url || "unknown page"));
|
|
const queue = createElement(
|
|
"span",
|
|
"job-log-queue",
|
|
`queue ${detail.pages_remaining ?? 0}p / ${detail.assets_remaining ?? 0}a`,
|
|
);
|
|
item.append(action, url, queue);
|
|
return item;
|
|
}
|
|
|
|
item.append(
|
|
createElement("span", "job-log-action", String(entry.phase || entry.kind)),
|
|
createElement("span", "job-log-url", String(entry.message || "Worker update")),
|
|
);
|
|
return item;
|
|
}
|
|
|
|
function recordStreamEvent(jobId, kind, event) {
|
|
let payload;
|
|
try {
|
|
payload = JSON.parse(event.data);
|
|
} catch {
|
|
return;
|
|
}
|
|
if (kind === "progress" && !payload.detail) return;
|
|
|
|
const eventId = event.lastEventId || `${kind}:${payload.logged_at}:${payload.message}`;
|
|
const seenIds = jobLogIds.get(jobId) || new Set();
|
|
if (seenIds.has(eventId)) return;
|
|
seenIds.add(eventId);
|
|
jobLogIds.set(jobId, seenIds);
|
|
|
|
const entries = jobLogs.get(jobId) || [];
|
|
const entry = { id: eventId, kind, ...payload };
|
|
entries.push(entry);
|
|
jobLogs.set(jobId, entries);
|
|
appendLogEntry(jobId, entry, entries.length);
|
|
}
|
|
|
|
function appendLogEntry(jobId, entry, count) {
|
|
const card = jobGrid.querySelector(`[data-job-id="${jobId}"]`);
|
|
const list = card?.querySelector(".job-log-list");
|
|
if (!list) return;
|
|
const followTail = list.scrollTop + list.clientHeight >= list.scrollHeight - 16;
|
|
list.querySelector(".job-log-empty")?.remove();
|
|
list.append(createLogEntry(entry));
|
|
const counter = card.querySelector(".job-log-count");
|
|
if (counter) counter.textContent = `${count} events`;
|
|
if (followTail) list.scrollTop = list.scrollHeight;
|
|
}
|
|
|
|
function scheduleJobRefresh(jobId) {
|
|
if (refreshTimers.has(jobId)) return;
|
|
const timer = window.setTimeout(() => {
|
|
refreshTimers.delete(jobId);
|
|
void refreshJob(jobId);
|
|
}, 350);
|
|
refreshTimers.set(jobId, timer);
|
|
}
|
|
|
|
function connectEvents(jobId) {
|
|
if (streams.has(jobId)) return;
|
|
const eventUrl = `/api/captures/${encodeURIComponent(jobId)}/events?session=${encodeURIComponent(ownerToken)}`;
|
|
const stream = new EventSource(eventUrl);
|
|
streams.set(jobId, stream);
|
|
stream.addEventListener("state", (event) => {
|
|
recordStreamEvent(jobId, "state", event);
|
|
void refreshJob(jobId);
|
|
});
|
|
stream.addEventListener("progress", (event) => {
|
|
recordStreamEvent(jobId, "progress", event);
|
|
scheduleJobRefresh(jobId);
|
|
});
|
|
stream.addEventListener("terminal", () => {
|
|
closeEvents(jobId);
|
|
void refreshJob(jobId);
|
|
void loadPublicStats();
|
|
});
|
|
stream.onerror = () => {
|
|
window.setTimeout(() => {
|
|
if (streams.get(jobId) === stream) void refreshJob(jobId);
|
|
}, 1_000);
|
|
};
|
|
}
|
|
|
|
function closeEvents(jobId) {
|
|
const stream = streams.get(jobId);
|
|
if (stream) stream.close();
|
|
streams.delete(jobId);
|
|
const timer = refreshTimers.get(jobId);
|
|
if (timer) window.clearTimeout(timer);
|
|
refreshTimers.delete(jobId);
|
|
}
|
|
|
|
async function refreshJob(jobId) {
|
|
try {
|
|
upsertJob(await request(`/api/captures/${encodeURIComponent(jobId)}`));
|
|
} catch (error) {
|
|
closeEvents(jobId);
|
|
if (error.status === 404) forgetJob(jobId);
|
|
}
|
|
}
|
|
|
|
async function refreshKnownJobs() {
|
|
const storedIds = readStoredSessions().map((session) => session.id);
|
|
const knownIds = [...new Set([...storedIds, ...jobs.keys()])];
|
|
await Promise.all(knownIds.map((jobId) => refreshJob(jobId)));
|
|
}
|
|
|
|
async function loadStoredJobs() {
|
|
const sessions = readStoredSessions();
|
|
sessions.forEach((session) => {
|
|
if (session.job?.id && !jobs.has(session.id)) jobs.set(session.id, session.job);
|
|
});
|
|
renderJobs();
|
|
await refreshKnownJobs();
|
|
}
|
|
|
|
async function loadPublicStats() {
|
|
try {
|
|
const stats = await request("/api/stats", {}, { session: false });
|
|
publicStats.websites.textContent = formatCount(stats.websites_cloned);
|
|
publicStats.data.textContent = formatBytes(stats.bytes_scraped);
|
|
publicStats.active.textContent = formatCount(stats.active_crawls);
|
|
publicStats.files.textContent = formatCount(stats.files_archived);
|
|
setServiceStatus("Capture service online", true);
|
|
} catch {
|
|
setServiceStatus("Service unavailable", false);
|
|
}
|
|
}
|
|
|
|
async function cancelJob(jobId) {
|
|
const current = jobs.get(jobId);
|
|
if (current && !current.cancel_requested) {
|
|
upsertJob(
|
|
{
|
|
...current,
|
|
cancel_requested: true,
|
|
phase: "Cancelling",
|
|
message: "Cancellation requested",
|
|
},
|
|
{ persist: false, connect: false },
|
|
);
|
|
}
|
|
try {
|
|
upsertJob(await request(`/api/captures/${encodeURIComponent(jobId)}/cancel`, { method: "POST" }));
|
|
void loadPublicStats();
|
|
} catch (error) {
|
|
if (current) upsertJob(current);
|
|
setMessage(error.message);
|
|
}
|
|
}
|
|
|
|
async function deleteJob(jobId) {
|
|
try {
|
|
const updated = await request(`/api/captures/${encodeURIComponent(jobId)}`, { method: "DELETE" });
|
|
if (updated.state === "expired") forgetJob(jobId);
|
|
else upsertJob(updated);
|
|
void loadPublicStats();
|
|
} catch (error) {
|
|
setMessage(error.message);
|
|
}
|
|
}
|
|
|
|
async function downloadArchive(job, button) {
|
|
if (!job.archive?.download_url) return;
|
|
const originalText = button.textContent;
|
|
button.disabled = true;
|
|
button.textContent = "Preparing download";
|
|
try {
|
|
const response = await fetch(job.archive.download_url, {
|
|
headers: { [OWNER_HEADER]: ownerToken },
|
|
});
|
|
if (!response.ok) {
|
|
const body = await response.json().catch(() => null);
|
|
const error = new Error(body?.detail || "The archive could not be downloaded.");
|
|
error.status = response.status;
|
|
throw error;
|
|
}
|
|
const objectUrl = URL.createObjectURL(await response.blob());
|
|
const anchor = document.createElement("a");
|
|
anchor.href = objectUrl;
|
|
anchor.download = `siteharbor-${job.id}.zip`;
|
|
document.body.append(anchor);
|
|
anchor.click();
|
|
anchor.remove();
|
|
window.setTimeout(() => URL.revokeObjectURL(objectUrl), 1_000);
|
|
} catch (error) {
|
|
setMessage(error.message);
|
|
} finally {
|
|
button.disabled = false;
|
|
button.textContent = originalText;
|
|
}
|
|
}
|
|
|
|
async function showReport(jobId) {
|
|
try {
|
|
const report = await request(`/api/captures/${encodeURIComponent(jobId)}/report`);
|
|
renderReport(report);
|
|
reportDialog.showModal();
|
|
} catch (error) {
|
|
setMessage(error.message);
|
|
}
|
|
}
|
|
|
|
function renderReport(report) {
|
|
reportContent.replaceChildren();
|
|
const summary = createElement("div", "report-summary");
|
|
const stats = report.stats || {};
|
|
[
|
|
[stats.pages_fetched || 0, "pages"],
|
|
[stats.assets_fetched || 0, "assets"],
|
|
[formatBytes(stats.bytes_downloaded || 0), "downloaded"],
|
|
[report.quality || "unknown", "capture quality"],
|
|
].forEach(([value, label]) => {
|
|
const item = document.createElement("div");
|
|
item.append(createElement("strong", "", String(value)), createElement("span", "", label));
|
|
summary.append(item);
|
|
});
|
|
reportContent.append(summary);
|
|
|
|
const details = createElement("ul", "report-list");
|
|
[
|
|
`Entry point: ${report.entry_point || "not available"}`,
|
|
`Capture duration: ${report.elapsed_seconds || 0} seconds`,
|
|
`Captured resources: ${(report.resources || []).length}`,
|
|
].forEach((line) => details.append(createElement("li", "", line)));
|
|
reportContent.append(details);
|
|
|
|
if (report.warnings?.length) {
|
|
reportContent.append(createElement("h3", "report-subhead", "Warnings"));
|
|
const warnings = createElement("ul", "report-list");
|
|
report.warnings.forEach((warning) => warnings.append(createElement("li", "", warning)));
|
|
reportContent.append(warnings);
|
|
}
|
|
if (report.errors?.length) {
|
|
reportContent.append(createElement("h3", "report-subhead", "Resource failures"));
|
|
const errors = createElement("ul", "report-list");
|
|
report.errors.slice(0, 20).forEach((error) => {
|
|
errors.append(createElement("li", "", `${error.url}: ${error.message}`));
|
|
});
|
|
reportContent.append(errors);
|
|
}
|
|
}
|
|
|
|
function selectedAssetTypes() {
|
|
return controls.assetTypes
|
|
.filter((control) => control.checked)
|
|
.map((control) => control.dataset.assetType);
|
|
}
|
|
|
|
function skippedFileTypes() {
|
|
return controls.skipFileTypes.value
|
|
.split(",")
|
|
.map((item) => item.trim())
|
|
.filter(Boolean);
|
|
}
|
|
|
|
form.addEventListener("submit", async (event) => {
|
|
event.preventDefault();
|
|
const url = urlInput.value.trim();
|
|
if (!url) {
|
|
setMessage("Enter a public website URL to begin.");
|
|
urlInput.focus();
|
|
return;
|
|
}
|
|
|
|
setMessage("");
|
|
submitButton.disabled = true;
|
|
submitButton.querySelector("span").textContent = "Preparing archive";
|
|
try {
|
|
const job = await request("/api/captures", {
|
|
method: "POST",
|
|
body: JSON.stringify({
|
|
url,
|
|
include_external_assets: controls.externalAssets.checked,
|
|
parallel_connections: readNumber(controls.parallelConnections, 6),
|
|
download_asset_types: selectedAssetTypes(),
|
|
skip_file_types: skippedFileTypes(),
|
|
use_proxy: controls.useProxy.checked && !controls.useProxy.disabled,
|
|
max_pages: readNumber(controls.maxPages, 150),
|
|
max_depth: readNumber(controls.maxDepth, 5),
|
|
max_bytes_mb: readNumber(controls.maxBytes, 100),
|
|
max_duration_seconds: readNumber(controls.maxDuration, 300),
|
|
retention_days: readNumber(controls.retentionDays, 2),
|
|
}),
|
|
});
|
|
upsertJob(job);
|
|
setMessage("Archive queued. This browser will reconnect to it automatically.", "success");
|
|
urlInput.value = "";
|
|
void loadPublicStats();
|
|
} catch (error) {
|
|
setMessage(error.message);
|
|
} finally {
|
|
submitButton.disabled = false;
|
|
submitButton.querySelector("span").textContent = "Start archive";
|
|
}
|
|
});
|
|
|
|
[
|
|
controls.maxPages,
|
|
controls.maxDepth,
|
|
controls.maxBytes,
|
|
controls.maxDuration,
|
|
controls.retentionDays,
|
|
].forEach((control) => control.addEventListener("input", updateBudgetSummary));
|
|
controls.parallelConnections.addEventListener("input", updateParallelValue);
|
|
controls.useProxy.addEventListener("change", updateProxySummary);
|
|
|
|
refreshButton.addEventListener("click", async () => {
|
|
refreshButton.disabled = true;
|
|
try {
|
|
await Promise.all([refreshKnownJobs(), loadPublicStats()]);
|
|
} finally {
|
|
refreshButton.disabled = false;
|
|
}
|
|
});
|
|
closeReportButton.addEventListener("click", () => reportDialog.close());
|
|
reportDialog.addEventListener("click", (event) => {
|
|
if (event.target === reportDialog) reportDialog.close();
|
|
});
|
|
|
|
document.addEventListener("visibilitychange", () => {
|
|
if (!document.hidden) {
|
|
void refreshKnownJobs();
|
|
void loadPublicStats();
|
|
}
|
|
});
|
|
window.addEventListener("online", () => {
|
|
void refreshKnownJobs();
|
|
void loadPublicStats();
|
|
});
|
|
window.addEventListener("beforeunload", () => {
|
|
streams.forEach((stream) => stream.close());
|
|
});
|
|
|
|
async function initialize() {
|
|
if (!storageAvailable) {
|
|
setMessage("Browser storage is unavailable, so this session cannot be restored after this tab closes.");
|
|
}
|
|
try {
|
|
setControlLimits(await request("/api/capabilities", {}, { session: false }));
|
|
} catch (error) {
|
|
setMessage(error.message);
|
|
}
|
|
await Promise.all([loadStoredJobs(), loadPublicStats()]);
|
|
window.setInterval(() => {
|
|
void refreshKnownJobs();
|
|
void loadPublicStats();
|
|
}, 15_000);
|
|
}
|
|
|
|
void initialize();
|