implement cancellation handling in CaptureService and Database, update job states accordingly

app/db.py

@@ -27,11 +27,11 @@ class Database:
         self.path = path
 
     @contextmanager
-    def connection(self) -> Iterator[sqlite3.Connection]:
-        connection = sqlite3.connect(self.path, timeout=15, isolation_level=None)
+    def connection(self, *, timeout: float = 15.0) -> Iterator[sqlite3.Connection]:
+        connection = sqlite3.connect(self.path, timeout=timeout, isolation_level=None)
         connection.row_factory = sqlite3.Row
         connection.execute("PRAGMA foreign_keys = ON")
-        connection.execute("PRAGMA busy_timeout = 15000")
+        connection.execute(f"PRAGMA busy_timeout = {int(timeout * 1_000)}")
         try:
             yield connection
         finally:
@@ -106,6 +106,32 @@ class Database:
                 "CREATE INDEX IF NOT EXISTS jobs_owner_created_idx "
                 "ON jobs(owner_token, created_at DESC)"
             )
+            cancelled_rows = connection.execute(
+                "SELECT id FROM jobs WHERE state = 'running' AND cancel_requested = 1"
+            ).fetchall()
+            if cancelled_rows:
+                now = timestamp()
+                connection.execute(
+                    """
+                    UPDATE jobs
+                    SET state = 'cancelled', phase = 'Cancelled',
+                        message = 'Cancellation requested',
+                        worker_id = NULL, lease_expires_at = NULL, completed_at = ?, updated_at = ?
+                    WHERE state = 'running' AND cancel_requested = 1
+                    """,
+                    (now, now),
+                )
+                for row in cancelled_rows:
+                    self._append_event(
+                        connection,
+                        str(row["id"]),
+                        "state",
+                        {
+                            "state": "cancelled",
+                            "phase": "Cancelled",
+                            "message": "Cancellation requested",
+                        },
+                    )
 
     def create_job(
         self,
@@ -429,6 +455,30 @@ class Database:
             and row["lease_expires_at"] >= now
         )
 
+    def worker_should_stop(self, job_id: str, worker_id: str) -> bool:
+        """Check cancellation and lease ownership without stalling a crawl on a busy database."""
+        now = timestamp()
+        try:
+            with self.connection(timeout=0.5) as connection:
+                row = connection.execute(
+                    """
+                    SELECT state, cancel_requested, worker_id, lease_expires_at
+                    FROM jobs
+                    WHERE id = ?
+                    """,
+                    (job_id,),
+                ).fetchone()
+        except sqlite3.OperationalError:
+            return False
+        return not bool(
+            row
+            and not row["cancel_requested"]
+            and row["state"] == "running"
+            and row["worker_id"] == worker_id
+            and row["lease_expires_at"]
+            and row["lease_expires_at"] >= now
+        )
+
     def request_cancel(self, job_id: str) -> dict[str, Any] | None:
         now = timestamp()
         with self.connection() as connection:
@@ -463,14 +513,16 @@ class Database:
                     cursor = connection.execute(
                         """
                         UPDATE jobs
-                        SET cancel_requested = 1, message = 'Cancellation requested', updated_at = ?
-                        WHERE id = ? AND state = 'running' AND cancel_requested = 0
+                        SET cancel_requested = 1, state = 'cancelled', phase = 'Cancelled',
+                            message = 'Cancellation requested', worker_id = NULL,
+                            lease_expires_at = NULL, completed_at = ?, updated_at = ?
+                        WHERE id = ? AND state = 'running'
                         """,
-                        (now, job_id),
+                        (now, now, job_id),
                     )
                     payload = {
-                        "state": "running",
-                        "phase": "Cancelling",
+                        "state": "cancelled",
+                        "phase": "Cancelled",
                         "message": "Cancellation requested",
                     }
                 if cursor.rowcount: