Enhance job logging and UI updates

- Implement job logging functionality to track progress and events for each job. - Add a new section in the job card to display session logs with detailed entries. - Update job statistics to include pages fetched, pages found, and assets processed. - Modify the UI theme and text for better clarity and aesthetics. - Adjust tests to validate new logging features and ensure proper event handling.
2026-09-05 07:36:34 +05:30
parent 8f9210d6be
commit 2059c275c2
10 changed files with 762 additions and 593 deletions
@@ -43,6 +43,9 @@ const TOKEN_PATTERN = /^[A-Za-z0-9_-]{24,200}$/;

 const jobs = new Map();
 const streams = new Map();
+const jobLogs = new Map();
+const jobLogIds = new Map();
+const refreshTimers = new Map();
 let storageAvailable = true;
 const ownerToken = getOwnerToken();

@@ -130,6 +133,8 @@ function rememberJob(job) {
 function forgetJob(jobId) {
  jobs.delete(jobId);
  closeEvents(jobId);
+  jobLogs.delete(jobId);
+  jobLogIds.delete(jobId);
  writeStoredSessions(readStoredSessions().filter((session) => session.id !== jobId));
  renderJobs();
 }
@@ -161,6 +166,14 @@ function formatTime(value) {
    : date.toLocaleString([], { dateStyle: "medium", timeStyle: "short" });
 }

+function formatLogTime(value) {
+  if (!value) return "--:--:--";
+  const date = new Date(value);
+  return Number.isNaN(date.getTime())
+    ? "--:--:--"
+    : date.toLocaleTimeString([], { hour: "2-digit", minute: "2-digit", second: "2-digit" });
+}
+
 function hostname(url) {
  try {
    return new URL(url).hostname;
@@ -311,6 +324,8 @@ function jobProgress(job) {
 function createJobCard(job) {
  const card = createElement("article", "job-card");
  card.dataset.state = job.state;
+  card.dataset.jobId = job.id;
+  card.classList.toggle("is-active", isActive(job));

  const topLine = createElement("div", "job-topline");
  topLine.append(
@@ -326,11 +341,19 @@ function createJobCard(job) {
  progress.firstElementChild.style.setProperty("--progress", `${jobProgress(job)}%`);
  card.append(progress);

+  const latestPage = latestPageLog(job.id);
+  const pagesFound = Number(job.stats?.pages_found) || 0;
+  const pagesFetched = Number(job.stats?.pages_fetched) || 0;
+  const queuedPages = Number(latestPage?.detail?.pages_remaining);
+  const pagesRemaining = Number.isFinite(queuedPages)
+    ? queuedPages
+    : Math.max(0, pagesFound - pagesFetched);
  const stats = createElement("div", "job-stats");
  const statItems = [
-    [job.stats?.pages_fetched || 0, "pages"],
-    [job.stats?.files_written || 0, "files"],
-    [formatBytes(job.stats?.bytes_downloaded || 0), "collected"],
+    [pagesFetched, "pages fetched"],
+    [pagesFound, "pages found"],
+    [pagesRemaining, "pages remaining"],
+    [formatBytes(job.stats?.bytes_downloaded || 0), "downloaded"],
  ];
  statItems.forEach(([value, label]) => {
    const item = document.createElement("div");
@@ -355,6 +378,11 @@ function createJobCard(job) {
  const tags = createElement("div", "job-tags");
  const parallel = job.options?.parallel_connections;
  if (parallel) tags.append(createElement("span", "", `${parallel} connections`));
+  const assetsFound = Number(job.stats?.assets_found) || 0;
+  const assetsFetched = Number(job.stats?.assets_fetched) || 0;
+  if (assetsFound || assetsFetched) {
+    tags.append(createElement("span", "", `${assetsFetched}/${assetsFound} assets`));
+  }
  tags.append(
    createElement(
      "span",
@@ -369,6 +397,8 @@ function createJobCard(job) {

  if (job.warnings?.length) card.append(createElement("p", "job-warning", job.warnings[0]));

+  if (isActive(job)) card.append(createJobLog(job.id));
+
  const actions = createElement("div", "job-actions");
  if (job.archive?.available) {
    const download = createElement("button", "job-action primary", "Download ZIP");
@@ -403,15 +433,106 @@ function createJobCard(job) {
  return card;
 }

+function latestPageLog(jobId) {
+  const entries = jobLogs.get(jobId) || [];
+  return [...entries].reverse().find((entry) => entry.detail?.kind === "page") || null;
+}
+
+function createJobLog(jobId) {
+  const section = createElement("section", "job-log");
+  const heading = createElement("div", "job-log-heading");
+  heading.append(
+    createElement("span", "job-log-title", "Session log"),
+    createElement("span", "job-log-count", `${(jobLogs.get(jobId) || []).length} events`),
+  );
+  const list = createElement("ol", "job-log-list");
+  const entries = jobLogs.get(jobId) || [];
+  if (entries.length) entries.forEach((entry) => list.append(createLogEntry(entry)));
+  else list.append(createElement("li", "job-log-empty", "Waiting for worker activity..."));
+  section.append(heading, list);
+  return section;
+}
+
+function createLogEntry(entry) {
+  const item = createElement("li", "job-log-entry");
+  const detail = entry.detail || {};
+  item.dataset.kind = detail.kind || entry.kind;
+  item.append(createElement("time", "job-log-time", formatLogTime(entry.logged_at)));
+
+  if (detail.kind === "page") {
+    const action = createElement("span", "job-log-action", `page ${detail.action || "seen"}`);
+    const url = createElement("span", "job-log-url", String(detail.url || "unknown page"));
+    const queue = createElement(
+      "span",
+      "job-log-queue",
+      `queue ${detail.pages_remaining ?? 0}p / ${detail.assets_remaining ?? 0}a`,
+    );
+    item.append(action, url, queue);
+    return item;
+  }
+
+  item.append(
+    createElement("span", "job-log-action", String(entry.phase || entry.kind)),
+    createElement("span", "job-log-url", String(entry.message || "Worker update")),
+  );
+  return item;
+}
+
+function recordStreamEvent(jobId, kind, event) {
+  let payload;
+  try {
+    payload = JSON.parse(event.data);
+  } catch {
+    return;
+  }
+  if (kind === "progress" && !payload.detail) return;
+
+  const eventId = event.lastEventId || `${kind}:${payload.logged_at}:${payload.message}`;
+  const seenIds = jobLogIds.get(jobId) || new Set();
+  if (seenIds.has(eventId)) return;
+  seenIds.add(eventId);
+  jobLogIds.set(jobId, seenIds);
+
+  const entries = jobLogs.get(jobId) || [];
+  const entry = { id: eventId, kind, ...payload };
+  entries.push(entry);
+  jobLogs.set(jobId, entries);
+  appendLogEntry(jobId, entry, entries.length);
+}
+
+function appendLogEntry(jobId, entry, count) {
+  const card = jobGrid.querySelector(`[data-job-id="${jobId}"]`);
+  const list = card?.querySelector(".job-log-list");
+  if (!list) return;
+  const followTail = list.scrollTop + list.clientHeight >= list.scrollHeight - 16;
+  list.querySelector(".job-log-empty")?.remove();
+  list.append(createLogEntry(entry));
+  const counter = card.querySelector(".job-log-count");
+  if (counter) counter.textContent = `${count} events`;
+  if (followTail) list.scrollTop = list.scrollHeight;
+}
+
+function scheduleJobRefresh(jobId) {
+  if (refreshTimers.has(jobId)) return;
+  const timer = window.setTimeout(() => {
+    refreshTimers.delete(jobId);
+    void refreshJob(jobId);
+  }, 350);
+  refreshTimers.set(jobId, timer);
+}
+
 function connectEvents(jobId) {
  if (streams.has(jobId)) return;
  const eventUrl = `/api/captures/${encodeURIComponent(jobId)}/events?session=${encodeURIComponent(ownerToken)}`;
  const stream = new EventSource(eventUrl);
  streams.set(jobId, stream);
-  ["state", "progress"].forEach((eventName) => {
-    stream.addEventListener(eventName, () => {
-      void refreshJob(jobId);
-    });
+  stream.addEventListener("state", (event) => {
+    recordStreamEvent(jobId, "state", event);
+    void refreshJob(jobId);
+  });
+  stream.addEventListener("progress", (event) => {
+    recordStreamEvent(jobId, "progress", event);
+    scheduleJobRefresh(jobId);
  });
  stream.addEventListener("terminal", () => {
    closeEvents(jobId);
@@ -429,6 +550,9 @@ function closeEvents(jobId) {
  const stream = streams.get(jobId);
  if (stream) stream.close();
  streams.delete(jobId);
+  const timer = refreshTimers.get(jobId);
+  if (timer) window.clearTimeout(timer);
+  refreshTimers.delete(jobId);
 }

 async function refreshJob(jobId) {