Enhance job logging and UI updates
- Implement job logging functionality to track progress and events for each job. - Add a new section in the job card to display session logs with detailed entries. - Update job statistics to include pages fetched, pages found, and assets processed. - Modify the UI theme and text for better clarity and aesthetics. - Adjust tests to validate new logging features and ensure proper event handling.
This commit is contained in:
kstyagi23
+131
-7
@@ -43,6 +43,9 @@ const TOKEN_PATTERN = /^[A-Za-z0-9_-]{24,200}$/;
|
||||
|
||||
const jobs = new Map();
|
||||
const streams = new Map();
|
||||
const jobLogs = new Map();
|
||||
const jobLogIds = new Map();
|
||||
const refreshTimers = new Map();
|
||||
let storageAvailable = true;
|
||||
const ownerToken = getOwnerToken();
|
||||
|
||||
@@ -130,6 +133,8 @@ function rememberJob(job) {
|
||||
function forgetJob(jobId) {
|
||||
jobs.delete(jobId);
|
||||
closeEvents(jobId);
|
||||
jobLogs.delete(jobId);
|
||||
jobLogIds.delete(jobId);
|
||||
writeStoredSessions(readStoredSessions().filter((session) => session.id !== jobId));
|
||||
renderJobs();
|
||||
}
|
||||
@@ -161,6 +166,14 @@ function formatTime(value) {
|
||||
: date.toLocaleString([], { dateStyle: "medium", timeStyle: "short" });
|
||||
}
|
||||
|
||||
function formatLogTime(value) {
|
||||
if (!value) return "--:--:--";
|
||||
const date = new Date(value);
|
||||
return Number.isNaN(date.getTime())
|
||||
? "--:--:--"
|
||||
: date.toLocaleTimeString([], { hour: "2-digit", minute: "2-digit", second: "2-digit" });
|
||||
}
|
||||
|
||||
function hostname(url) {
|
||||
try {
|
||||
return new URL(url).hostname;
|
||||
@@ -311,6 +324,8 @@ function jobProgress(job) {
|
||||
function createJobCard(job) {
|
||||
const card = createElement("article", "job-card");
|
||||
card.dataset.state = job.state;
|
||||
card.dataset.jobId = job.id;
|
||||
card.classList.toggle("is-active", isActive(job));
|
||||
|
||||
const topLine = createElement("div", "job-topline");
|
||||
topLine.append(
|
||||
@@ -326,11 +341,19 @@ function createJobCard(job) {
|
||||
progress.firstElementChild.style.setProperty("--progress", `${jobProgress(job)}%`);
|
||||
card.append(progress);
|
||||
|
||||
const latestPage = latestPageLog(job.id);
|
||||
const pagesFound = Number(job.stats?.pages_found) || 0;
|
||||
const pagesFetched = Number(job.stats?.pages_fetched) || 0;
|
||||
const queuedPages = Number(latestPage?.detail?.pages_remaining);
|
||||
const pagesRemaining = Number.isFinite(queuedPages)
|
||||
? queuedPages
|
||||
: Math.max(0, pagesFound - pagesFetched);
|
||||
const stats = createElement("div", "job-stats");
|
||||
const statItems = [
|
||||
[job.stats?.pages_fetched || 0, "pages"],
|
||||
[job.stats?.files_written || 0, "files"],
|
||||
[formatBytes(job.stats?.bytes_downloaded || 0), "collected"],
|
||||
[pagesFetched, "pages fetched"],
|
||||
[pagesFound, "pages found"],
|
||||
[pagesRemaining, "pages remaining"],
|
||||
[formatBytes(job.stats?.bytes_downloaded || 0), "downloaded"],
|
||||
];
|
||||
statItems.forEach(([value, label]) => {
|
||||
const item = document.createElement("div");
|
||||
@@ -355,6 +378,11 @@ function createJobCard(job) {
|
||||
const tags = createElement("div", "job-tags");
|
||||
const parallel = job.options?.parallel_connections;
|
||||
if (parallel) tags.append(createElement("span", "", `${parallel} connections`));
|
||||
const assetsFound = Number(job.stats?.assets_found) || 0;
|
||||
const assetsFetched = Number(job.stats?.assets_fetched) || 0;
|
||||
if (assetsFound || assetsFetched) {
|
||||
tags.append(createElement("span", "", `${assetsFetched}/${assetsFound} assets`));
|
||||
}
|
||||
tags.append(
|
||||
createElement(
|
||||
"span",
|
||||
@@ -369,6 +397,8 @@ function createJobCard(job) {
|
||||
|
||||
if (job.warnings?.length) card.append(createElement("p", "job-warning", job.warnings[0]));
|
||||
|
||||
if (isActive(job)) card.append(createJobLog(job.id));
|
||||
|
||||
const actions = createElement("div", "job-actions");
|
||||
if (job.archive?.available) {
|
||||
const download = createElement("button", "job-action primary", "Download ZIP");
|
||||
@@ -403,15 +433,106 @@ function createJobCard(job) {
|
||||
return card;
|
||||
}
|
||||
|
||||
function latestPageLog(jobId) {
|
||||
const entries = jobLogs.get(jobId) || [];
|
||||
return [...entries].reverse().find((entry) => entry.detail?.kind === "page") || null;
|
||||
}
|
||||
|
||||
function createJobLog(jobId) {
|
||||
const section = createElement("section", "job-log");
|
||||
const heading = createElement("div", "job-log-heading");
|
||||
heading.append(
|
||||
createElement("span", "job-log-title", "Session log"),
|
||||
createElement("span", "job-log-count", `${(jobLogs.get(jobId) || []).length} events`),
|
||||
);
|
||||
const list = createElement("ol", "job-log-list");
|
||||
const entries = jobLogs.get(jobId) || [];
|
||||
if (entries.length) entries.forEach((entry) => list.append(createLogEntry(entry)));
|
||||
else list.append(createElement("li", "job-log-empty", "Waiting for worker activity..."));
|
||||
section.append(heading, list);
|
||||
return section;
|
||||
}
|
||||
|
||||
function createLogEntry(entry) {
|
||||
const item = createElement("li", "job-log-entry");
|
||||
const detail = entry.detail || {};
|
||||
item.dataset.kind = detail.kind || entry.kind;
|
||||
item.append(createElement("time", "job-log-time", formatLogTime(entry.logged_at)));
|
||||
|
||||
if (detail.kind === "page") {
|
||||
const action = createElement("span", "job-log-action", `page ${detail.action || "seen"}`);
|
||||
const url = createElement("span", "job-log-url", String(detail.url || "unknown page"));
|
||||
const queue = createElement(
|
||||
"span",
|
||||
"job-log-queue",
|
||||
`queue ${detail.pages_remaining ?? 0}p / ${detail.assets_remaining ?? 0}a`,
|
||||
);
|
||||
item.append(action, url, queue);
|
||||
return item;
|
||||
}
|
||||
|
||||
item.append(
|
||||
createElement("span", "job-log-action", String(entry.phase || entry.kind)),
|
||||
createElement("span", "job-log-url", String(entry.message || "Worker update")),
|
||||
);
|
||||
return item;
|
||||
}
|
||||
|
||||
function recordStreamEvent(jobId, kind, event) {
|
||||
let payload;
|
||||
try {
|
||||
payload = JSON.parse(event.data);
|
||||
} catch {
|
||||
return;
|
||||
}
|
||||
if (kind === "progress" && !payload.detail) return;
|
||||
|
||||
const eventId = event.lastEventId || `${kind}:${payload.logged_at}:${payload.message}`;
|
||||
const seenIds = jobLogIds.get(jobId) || new Set();
|
||||
if (seenIds.has(eventId)) return;
|
||||
seenIds.add(eventId);
|
||||
jobLogIds.set(jobId, seenIds);
|
||||
|
||||
const entries = jobLogs.get(jobId) || [];
|
||||
const entry = { id: eventId, kind, ...payload };
|
||||
entries.push(entry);
|
||||
jobLogs.set(jobId, entries);
|
||||
appendLogEntry(jobId, entry, entries.length);
|
||||
}
|
||||
|
||||
function appendLogEntry(jobId, entry, count) {
|
||||
const card = jobGrid.querySelector(`[data-job-id="${jobId}"]`);
|
||||
const list = card?.querySelector(".job-log-list");
|
||||
if (!list) return;
|
||||
const followTail = list.scrollTop + list.clientHeight >= list.scrollHeight - 16;
|
||||
list.querySelector(".job-log-empty")?.remove();
|
||||
list.append(createLogEntry(entry));
|
||||
const counter = card.querySelector(".job-log-count");
|
||||
if (counter) counter.textContent = `${count} events`;
|
||||
if (followTail) list.scrollTop = list.scrollHeight;
|
||||
}
|
||||
|
||||
function scheduleJobRefresh(jobId) {
|
||||
if (refreshTimers.has(jobId)) return;
|
||||
const timer = window.setTimeout(() => {
|
||||
refreshTimers.delete(jobId);
|
||||
void refreshJob(jobId);
|
||||
}, 350);
|
||||
refreshTimers.set(jobId, timer);
|
||||
}
|
||||
|
||||
function connectEvents(jobId) {
|
||||
if (streams.has(jobId)) return;
|
||||
const eventUrl = `/api/captures/${encodeURIComponent(jobId)}/events?session=${encodeURIComponent(ownerToken)}`;
|
||||
const stream = new EventSource(eventUrl);
|
||||
streams.set(jobId, stream);
|
||||
["state", "progress"].forEach((eventName) => {
|
||||
stream.addEventListener(eventName, () => {
|
||||
void refreshJob(jobId);
|
||||
});
|
||||
stream.addEventListener("state", (event) => {
|
||||
recordStreamEvent(jobId, "state", event);
|
||||
void refreshJob(jobId);
|
||||
});
|
||||
stream.addEventListener("progress", (event) => {
|
||||
recordStreamEvent(jobId, "progress", event);
|
||||
scheduleJobRefresh(jobId);
|
||||
});
|
||||
stream.addEventListener("terminal", () => {
|
||||
closeEvents(jobId);
|
||||
@@ -429,6 +550,9 @@ function closeEvents(jobId) {
|
||||
const stream = streams.get(jobId);
|
||||
if (stream) stream.close();
|
||||
streams.delete(jobId);
|
||||
const timer = refreshTimers.get(jobId);
|
||||
if (timer) window.clearTimeout(timer);
|
||||
refreshTimers.delete(jobId);
|
||||
}
|
||||
|
||||
async function refreshJob(jobId) {
|
||||
|
||||
Reference in New Issue
Block a user