Implement guest tool installation and management system

- Added `guest_tools.py` for standalone tool installer functionality.
- Introduced `ToolJobStore` in `tool_store.py` to manage per-VM tool job records.
- Created `ToolService` in `tools.py` for background installation workers using the guest execution channel.
- Enhanced `LifecycleService` in `lifecycle.py` to support tool installation queuing and preparation.
- Updated `guest_exec.py` to handle connection shutdown gracefully.
- Added validation and normalization functions for tool specifications.
- Implemented logging and error handling for tool installation processes.
- Ensured compatibility with Ubuntu 22.04 and ARM64/x86_64 architectures.

tests/test_tool_catalog.py

@@ -0,0 +1,161 @@
+from __future__ import annotations
+
+import hashlib
+import io
+import os
+import shlex
+import tarfile
+import tempfile
+import unittest
+from pathlib import Path
+from types import SimpleNamespace
+from unittest.mock import Mock, patch
+
+from uvm import guest_tools
+from uvm.errors import ValidationError
+from uvm.tool_catalog import CATALOG, ToolSelection, all_selections, normalize_tools, recipe
+from uvm.tools import guest_command
+
+
+class ToolCatalogTests(unittest.TestCase):
+    def test_requested_catalog_and_every_guest_payload_fit_the_agent_protocol(self):
+        self.assertEqual({tool["id"] for tool in CATALOG},
+                         {"git", "nodejs", "python", "jdk", "wget", "curl", "htop", "php", "conda", "micromamba"})
+        for arch in ("x86_64", "aarch64"):
+            recipes = [recipe(item, arch) for item in all_selections()]
+            command = guest_command({"action": "probe", "tools": recipes, "username": "worker", "architecture": arch})
+            self.assertLess(len(command.encode()), 65536)
+            argv = shlex.split(command)
+            self.assertEqual(argv[:3], ["/usr/bin/python3", "-I", "-c"])
+            compile(argv[3], "guest-tools", "exec")
+            for item in recipes:
+                if "url" in item:
+                    self.assertTrue(item["url"].startswith("https://"))
+                    self.assertNotIn("/latest", item["url"])
+                    self.assertRegex(item["sha256"], r"^[0-9a-f]{64}$")
+        self.assertEqual(len(all_selections()), 12)
+
+    def test_selection_validation_is_allowlisted_strict_and_deduplicated(self):
+        expected = (ToolSelection("git"), ToolSelection("python", "3.12"))
+        self.assertEqual(normalize_tools([{"id": "python", "version": "3.12"}, {"id": "git"}, {"id": "git"}]), expected)
+        for value in (None, "git", [], ["git"], [{"id": "git; id"}], [{"id": "python"}],
+                      [{"id": "python", "version": "3.14"}], [{"id": "git", "version": "latest"}],
+                      [{"id": "git", "command": "id"}], [{"id": ["git"]}], [{"id": "git"}] * 13):
+            with self.subTest(value=value), self.assertRaises(ValidationError):
+                normalize_tools(value)
+        self.assertEqual(normalize_tools([], allow_empty=True), ())
+
+    def test_versioned_python_commands_and_jdk_compiler_are_verified(self):
+        for version in ("3.11", "3.12", "3.13"):
+            item = recipe(ToolSelection("python", version), "aarch64")
+            self.assertEqual(set(item["links"]), {"python" + version})
+            self.assertIn("venv", item["checks"][0][-1])
+            self.assertEqual(item["checks"][1][:3], ["python" + version, "-m", "pip"])
+        jdk = recipe(ToolSelection("jdk"), "aarch64")
+        self.assertEqual(jdk["checks"], [["javac", "-version"], ["java", "-version"]])
+        self.assertIn("arm64", jdk["java_home"])
+
+
+class GuestToolRecipeTests(unittest.TestCase):
+    def setUp(self):
+        self.temporary = self.enterContext(tempfile.TemporaryDirectory())
+        self.root = Path(self.temporary)
+
+    def test_download_mismatch_removes_artifact_and_https_redirect_is_checked(self):
+        target = self.root / "download"
+        response = io.BytesIO(b"publisher payload")
+        response.geturl = lambda: "https://publisher.test/tool"
+        with patch.object(guest_tools.urllib.request, "urlopen", return_value=response):
+            with self.assertRaisesRegex(guest_tools.ToolError, "checksum"):
+                guest_tools.download("https://publisher.test/tool", target, "0" * 64)
+        self.assertFalse(target.exists())
+        response = io.BytesIO(b"publisher payload")
+        response.geturl = lambda: "https://publisher.test/tool"
+        with patch.object(guest_tools.urllib.request, "urlopen", return_value=response):
+            guest_tools.download("https://publisher.test/tool", target, hashlib.sha256(b"publisher payload").hexdigest())
+        self.assertEqual(target.read_bytes(), b"publisher payload")
+        with self.assertRaisesRegex(guest_tools.ToolError, "HTTPS"):
+            guest_tools.download("http://publisher.test/tool", target)
+        response = io.BytesIO(b"publisher payload")
+        response.geturl = lambda: "http://publisher.test/tool"
+        with patch.object(guest_tools.urllib.request, "urlopen", return_value=response):
+            with self.assertRaisesRegex(guest_tools.ToolError, "HTTPS"):
+                guest_tools.download("https://publisher.test/tool", target)
+
+    def test_archive_traversal_links_and_devices_cannot_escape_staging(self):
+        cases = [("../outside", tarfile.REGTYPE, ""), ("/outside", tarfile.REGTYPE, ""),
+                 ("link", tarfile.SYMTYPE, "../../outside"), ("link", tarfile.LNKTYPE, "../outside"),
+                 ("device", tarfile.CHRTYPE, "")]
+        for name, kind, link in cases:
+            with self.subTest(name=name, kind=kind):
+                archive = self.root / "archive.tar"
+                with tarfile.open(archive, "w") as stream:
+                    item = tarfile.TarInfo(name)
+                    item.type = kind
+                    item.linkname = link
+                    stream.addfile(item)
+                output = self.root / "output"
+                output.mkdir(exist_ok=True)
+                with self.assertRaises(guest_tools.ToolError):
+                    guest_tools.extract_archive(archive, output)
+                self.assertFalse((self.root / "outside").exists())
+
+    def test_internal_archive_symlinks_are_supported(self):
+        archive = self.root / "archive.tar"
+        with tarfile.open(archive, "w") as stream:
+            item = tarfile.TarInfo("python/bin/python3.12")
+            item.size = 4
+            item.mode = 0o755
+            stream.addfile(item, io.BytesIO(b"test"))
+            link = tarfile.TarInfo("python/bin/python3")
+            link.type = tarfile.SYMTYPE
+            link.linkname = "python3.12"
+            stream.addfile(link)
+        output = self.root / "output"
+        output.mkdir()
+        guest_tools.extract_archive(archive, output)
+        self.assertEqual((output / "python/bin/python3").read_bytes(), b"test")
+
+    def test_probe_uses_guest_account_and_rejects_wrong_runtime_or_missing_companion(self):
+        node = recipe(ToolSelection("nodejs"), "x86_64")
+        account = SimpleNamespace(pw_dir="/home/worker", pw_uid=1001, pw_gid=1002)
+        with (patch.object(guest_tools.pwd, "getpwnam", return_value=account),
+              patch.object(guest_tools.os, "getgrouplist", return_value=[1002, 1003]),
+              patch.object(guest_tools.shutil, "which", return_value="/usr/local/bin/node"),
+              patch.object(guest_tools.subprocess, "run") as run):
+            run.return_value = SimpleNamespace(returncode=0, stdout=b"v22.10.0\n")
+            result = guest_tools.probe(node, "worker")
+            self.assertFalse(result["installed"])
+            self.assertEqual(result["detected_version"], "22.10.0")
+            self.assertEqual(run.call_args.kwargs["user"], 1001)
+            self.assertEqual(run.call_args.kwargs["env"]["HOME"], "/home/worker")
+            run.side_effect = [SimpleNamespace(returncode=0, stdout=b"v24.21.0\n"),
+                               SimpleNamespace(returncode=1, stdout=b"broken npm")]
+            self.assertFalse(guest_tools.probe(node, "worker")["installed"])
+
+    def test_managed_files_preserve_edits_and_can_finish_interrupted_publication(self):
+        installer = guest_tools.Installer("root")
+        path = self.root / "profile"
+        # Isolated directory fixtures stand in for the root-owned guest hierarchy.
+        with (patch.object(guest_tools, "ROOT", self.root),
+              patch.object(guest_tools, "safe_directory"),
+              patch.object(guest_tools, "regular_file")):
+            installer.write_managed(path, b"initial")
+            installer.write_managed(path, b"updated")
+            self.assertEqual(path.read_bytes(), b"updated")
+            path.write_bytes(b"guest edit")
+            with self.assertRaisesRegex(guest_tools.ToolError, "customized"):
+                installer.write_managed(path, b"next")
+            self.assertEqual(path.read_bytes(), b"guest edit")
+            path.write_bytes(b"updated")
+            installer.remember(path, hashlib.sha256(b"next").hexdigest())
+            installer.write_managed(path, b"next")
+            self.assertEqual(path.read_bytes(), b"next")
+
+    def test_log_capture_retains_a_bounded_tail(self):
+        installer = guest_tools.Installer("root")
+        installer.append(b"a" * 20000)
+        installer.append(b"last failure")
+        self.assertEqual(len(installer.log), guest_tools.LOG_LIMIT)
+        self.assertTrue(installer.truncated)
+        self.assertTrue(installer.log.endswith(b"last failure"))

tests/test_tool_jobs.py

@@ -0,0 +1,151 @@
+from __future__ import annotations
+
+import json
+import os
+import tempfile
+import threading
+import unittest
+from concurrent.futures import ThreadPoolExecutor
+from dataclasses import replace
+from pathlib import Path
+from unittest.mock import Mock, patch
+
+from tests.test_server import FakeLifecycle
+from uvm.config import Settings
+from uvm.errors import ConflictError, GuestExecutionError, NotFoundError, StateError
+from uvm.state import StateStore
+from uvm.tool_catalog import ToolSelection
+from uvm.tool_store import ToolJobStore
+from uvm.tools import ToolService
+
+
+class ToolJobTests(unittest.TestCase):
+    def setUp(self):
+        temporary = self.enterContext(tempfile.TemporaryDirectory())
+        self.settings = replace(Settings(), base=Path(temporary), tool_workers=1)
+        self.state = StateStore(self.settings)
+        self.vm = replace(FakeLifecycle().vm, runtime_owned=True, pid=123, process_start_time="1", boot_id="host-boot")
+        self.settings.vm_dir(self.vm.id).mkdir(parents=True, mode=0o700)
+        with self.state.transaction() as state:
+            state.vms[self.vm.id] = self.vm
+        self.store = ToolJobStore(self.settings, self.state)
+
+    def enqueue(self, selections=None, **options):
+        with self.state.operation_lock():
+            return self.store.enqueue_locked(self.vm.id, selections or [ToolSelection("git")], **options)
+
+    def test_duplicate_concurrent_submissions_have_one_durable_job(self):
+        with ThreadPoolExecutor(max_workers=4) as executor:
+            jobs = list(executor.map(lambda _: self.enqueue(request_id="request-1"), range(4)))
+        self.assertEqual(len({job["id"] for job in jobs}), 1)
+        self.assertEqual(len(self.store.read(self.vm.id)["jobs"]), 1)
+        path = self.settings.vm_dir(self.vm.id) / "tools.json"
+        self.assertEqual(path.stat().st_mode & 0o777, 0o600)
+        self.assertNotIn("guest-secret", path.read_text())
+        with self.assertRaises(ConflictError):
+            self.enqueue([ToolSelection("htop")])
+
+    def test_history_is_vm_scoped_and_deletion_is_never_undone(self):
+        job = self.enqueue()
+        with self.assertRaises(NotFoundError):
+            self.store.get_job(self.vm.name, job["id"])
+        with self.state.transaction() as state:
+            del state.vms[self.vm.id]
+        with self.assertRaises(NotFoundError):
+            self.store.update_job(self.vm.id, job["id"], lambda current, data: current.update(status="succeeded"))
+        self.assertNotIn(self.vm.id, self.state.load().vms)
+
+    def test_recovery_preserves_queued_work_and_marks_inflight_outcomes_unknown(self):
+        job = self.enqueue()
+        self.store.recover()
+        self.assertEqual(self.store.get_job(self.vm.id, job["id"])["status"], "queued")
+        self.store.claim()
+        self.store.update_job(self.vm.id, job["id"], lambda current, data: current["items"][0].update(status="installing"))
+        self.store.recover()
+        recovered = self.store.get_job(self.vm.id, job["id"])
+        self.assertEqual(recovered["status"], "interrupted")
+        self.assertTrue(recovered["outcome_unknown"])
+        self.assertEqual(recovered["items"][0]["status"], "interrupted")
+        self.assertIsNone(self.store.claim())
+
+    def test_symlinks_and_foreign_history_are_rejected(self):
+        outside = self.settings.base / "outside"
+        outside.write_text("private data")
+        path = self.settings.vm_dir(self.vm.id) / "tools.json"
+        path.symlink_to(outside)
+        with self.assertRaises((StateError, OSError)):
+            self.store.read(self.vm.id)
+        self.assertEqual(outside.read_text(), "private data")
+        path.unlink()
+        self.enqueue()
+        data = json.loads(path.read_text())
+        data["jobs"][0]["vm_id"] = "vm-other"
+        path.write_text(json.dumps(data))
+        with self.assertRaises(StateError):
+            self.store.read(self.vm.id)
+
+    def test_only_one_worker_can_own_this_registry_and_shutdown_releases_it(self):
+        first = ToolService(self.settings, self.state, Mock())
+        second = ToolService(self.settings, self.state, Mock())
+        self.addCleanup(first.shutdown)
+        self.addCleanup(second.shutdown)
+        first.start()
+        with self.assertRaises(ConflictError):
+            second.start()
+        first.shutdown()
+        second.start()
+
+    def make_service(self):
+        lifecycle = Mock()
+        lifecycle.prepare_tool_install.return_value = self.vm
+        service = ToolService(self.settings, self.state, lifecycle)
+        return service
+
+    @staticmethod
+    def observation(selection, installed=False):
+        return {**selection.to_dict(), "key": selection.key, "installed": installed,
+                "detected_version": "2.0.0" if installed else None, "usage": "tool --version", "error": None}
+
+    def test_worker_reports_partial_failure_and_retains_successful_tools(self):
+        selections = [ToolSelection("git"), ToolSelection("htop")]
+        job = self.enqueue(selections)
+        claimed = self.store.claim()
+        service = self.make_service()
+        results = [
+            {"ok": True, "busy": False, "boot_id": "guest-boot", "tools": [self.observation(item) for item in selections]},
+            {"ok": True, "status": "installed", "tool": self.observation(selections[0], True), "output": "git installed"},
+            {"ok": False, "error": "mirror unavailable", "output": "apt download failed"},
+        ]
+        with patch.object(service, "_execute", side_effect=results) as execute:
+            service._run_job(claimed)
+        result = self.store.get_job(self.vm.id, job["id"])
+        self.assertEqual(result["status"], "failed")
+        self.assertEqual([item["status"] for item in result["items"]], ["installed", "failed"])
+        self.assertEqual(result["items"][1]["output"], "apt download failed")
+        self.assertEqual(self.state.get(self.vm.id).status, "running")
+        self.assertEqual(execute.call_args.args[1]["boot_id"], "guest-boot")
+
+    def test_lost_result_is_never_replayed_and_remaining_tools_are_skipped(self):
+        selections = [ToolSelection("git"), ToolSelection("htop")]
+        job = self.enqueue(selections)
+        service = self.make_service()
+        with patch.object(service, "_execute", side_effect=[
+            {"ok": True, "busy": False, "boot_id": "boot", "tools": [self.observation(item) for item in selections]},
+            GuestExecutionError("connection lost", outcome_unknown=True),
+        ]) as execute:
+            service._run_job(self.store.claim())
+        result = self.store.get_job(self.vm.id, job["id"])
+        self.assertEqual(execute.call_count, 2)
+        self.assertEqual(result["status"], "interrupted")
+        self.assertTrue(result["outcome_unknown"])
+        self.assertEqual([item["status"] for item in result["items"]], ["interrupted", "skipped"])
+
+    def test_prior_guest_installation_blocks_a_retry_before_submission(self):
+        job = self.enqueue()
+        service = self.make_service()
+        with patch.object(service, "_execute", return_value={"ok": True, "busy": True}) as execute:
+            service._run_job(self.store.claim())
+        self.assertEqual(execute.call_count, 1)
+        result = self.store.get_job(self.vm.id, job["id"])
+        self.assertEqual(result["status"], "interrupted")
+        self.assertFalse(result["outcome_unknown"])

tests/test_tool_lifecycle.py

@@ -0,0 +1,57 @@
+from __future__ import annotations
+
+from unittest.mock import Mock
+
+from tests.test_lifecycle import LifecycleTestCase
+from uvm.errors import ConflictError, NotFoundError, UvmError, ValidationError
+from uvm.tool_catalog import ToolSelection
+
+
+class ToolLifecycleTests(LifecycleTestCase):
+    def test_create_persists_tool_intent_before_boot(self):
+        def configure(config):
+            vm_id = next(iter(self.store.load().vms))
+            jobs = self.service._tool_store._read(vm_id)["jobs"]
+            self.assertEqual(jobs[0]["selections"], [{"id": "python", "version": "3.12"}])
+            self.assertEqual(jobs[0]["status"], "queued")
+        original = self.service._client_factory
+        def client(socket, timeout):
+            result = original(socket, timeout)
+            result.configure_and_start = configure
+            return result
+        self.service._client_factory = client
+        vm = self.create_vm(tools=(ToolSelection("python", "3.12"),))
+        self.assertEqual(vm.status, "running")
+        self.assertEqual(self.service._tool_store.read(vm.id)["jobs"][0]["source"], "create")
+
+    def test_invalid_tools_fail_before_reserving_a_vm_and_failed_boot_removes_intent(self):
+        with self.assertRaises(ValidationError):
+            self.create_vm(tools=({"id": "invalid"},))
+        self.assertEqual(self.store.load().vms, {})
+        self.host.failures["client.configure", None] = UvmError("failed boot")
+        with self.assertRaises(UvmError):
+            self.create_vm(tools=(ToolSelection("git"),))
+        self.assertEqual(list(self.settings.vms_dir.iterdir()), [])
+
+    def test_stopped_installation_starts_only_when_requested_and_aliases_are_rejected(self):
+        vm = self.create_vm(name="worker")
+        self.service.stop(vm.id)
+        for alias in (vm.name, vm.guest_ip):
+            with self.assertRaises(NotFoundError):
+                self.service.queue_tools(alias, [ToolSelection("git")], start_if_stopped=True)
+        with self.assertRaises(ConflictError):
+            self.service.queue_tools(vm.id, [ToolSelection("git")])
+        job = self.service.queue_tools(vm.id, [ToolSelection("git")], start_if_stopped=True)
+        self.assertEqual(self.store.get(vm.id).status, "stopped")
+        ready = self.service.prepare_tool_install(vm.id, start_if_stopped=job["start_if_stopped"])
+        self.assertEqual(ready.status, "running")
+        self.assertEqual(ready.guest_ip, vm.guest_ip)
+
+    def test_a_tool_step_cannot_reconnect_to_a_restarted_vm(self):
+        vm = self.create_vm()
+        identity = vm.pid, vm.process_start_time, vm.boot_id
+        self.service.restart(vm.id)
+        self.service._guest_connector = Mock()
+        with self.assertRaisesRegex(ConflictError, "restarted"):
+            self.service.execute(vm.id, "id", expected_identity=identity)
+        self.service._guest_connector.assert_not_called()