From f0fc211a42ae5afdc65dcf3f300d19bf74aedb65 Mon Sep 17 00:00:00 2001
From: kstyagi23 <kstyagi@scribblex.net>
Date: Sat, 5 Sep 2026 06:58:24 +0530
Subject: [PATCH] add completed_successfully column to jobs table and update
 related statistics

---
 app/db.py              | 32 +++++++++++++++++++++++++++++---
 tests/test_database.py | 18 ++++++++++++++++++
 2 files changed, 47 insertions(+), 3 deletions(-)

diff --git a/app/db.py b/app/db.py
index 634bfc8..69106ee 100644
--- a/app/db.py
+++ b/app/db.py
@@ -65,6 +65,7 @@ class Database:
                     report_path TEXT,
                     archive_size INTEGER,
                     entry_point TEXT,
+                    completed_successfully INTEGER NOT NULL DEFAULT 0,
                     cancel_requested INTEGER NOT NULL DEFAULT 0,
                     worker_id TEXT,
                     lease_expires_at TEXT,
@@ -102,6 +103,24 @@ class Database:
                 connection.execute(
                     "ALTER TABLE jobs ADD COLUMN owner_token TEXT NOT NULL DEFAULT ''"
                 )
+            if "completed_successfully" not in columns:
+                try:
+                    connection.execute(
+                        "ALTER TABLE jobs ADD COLUMN completed_successfully "
+                        "INTEGER NOT NULL DEFAULT 0"
+                    )
+                except sqlite3.OperationalError as error:
+                    if "duplicate column name" not in str(error).lower():
+                        raise
+            connection.execute(
+                """
+                UPDATE jobs
+                SET completed_successfully = 1
+                WHERE completed_successfully = 0
+                  AND archive_size IS NOT NULL
+                  AND entry_point IS NOT NULL
+                """
+            )
             connection.execute(
                 "CREATE INDEX IF NOT EXISTS jobs_owner_created_idx "
                 "ON jobs(owner_token, created_at DESC)"
@@ -208,10 +227,16 @@ class Database:
             row = connection.execute(
                 """
                 SELECT
-                    COUNT(CASE WHEN state = 'ready' THEN 1 END) AS websites_cloned,
-                    COALESCE(SUM(bytes_downloaded), 0) AS bytes_scraped,
+                    COUNT(CASE WHEN completed_successfully = 1 THEN 1 END) AS websites_cloned,
+                    COALESCE(
+                        SUM(CASE WHEN completed_successfully = 1 THEN bytes_downloaded ELSE 0 END),
+                        0
+                    ) AS bytes_scraped,
                     COUNT(CASE WHEN state IN ('queued', 'running') THEN 1 END) AS active_crawls,
-                    COALESCE(SUM(files_written), 0) AS files_archived
+                    COALESCE(
+                        SUM(CASE WHEN completed_successfully = 1 THEN files_written ELSE 0 END),
+                        0
+                    ) AS files_archived
                 FROM jobs
                 """
             ).fetchone()
@@ -557,6 +582,7 @@ class Database:
                     artifact_path = ?, report_path = ?, archive_size = ?, entry_point = ?,
                     pages_found = ?, pages_fetched = ?, assets_found = ?, assets_fetched = ?,
                     bytes_downloaded = ?, files_written = ?, warnings_json = ?,
+                    completed_successfully = 1,
                     worker_id = NULL, lease_expires_at = NULL, expires_at = ?,
                     completed_at = ?, updated_at = ?
                 WHERE id = ?
diff --git a/tests/test_database.py b/tests/test_database.py
index b5d5902..05236fc 100644
--- a/tests/test_database.py
+++ b/tests/test_database.py
@@ -73,6 +73,24 @@ def test_sqlite_lease_claims_one_queued_job(tmp_path) -> None:
     expires_at = datetime.fromisoformat(ready["expires_at"])
     completed_at = datetime.fromisoformat(ready["completed_at"])
     assert expires_at - completed_at <= timedelta(days=2)
+    expected_stats = {
+        "websites_cloned": 1,
+        "bytes_scraped": 10,
+        "active_crawls": 0,
+        "files_archived": 1,
+    }
+    assert database.public_stats() == expected_stats
+
+    database.mark_deleted(job["id"])
+
+    assert database.public_stats() == expected_stats
+    with database.connection() as connection:
+        connection.execute(
+            "UPDATE jobs SET completed_successfully = 0 WHERE id = ?",
+            (job["id"],),
+        )
+    database.initialize()
+    assert database.public_stats() == expected_stats
 
 
 def test_worker_cannot_cancel_without_a_user_cancellation_request(tmp_path) -> None: