feat(tools): Enhance guest tools installation and management

- Added new API endpoints for tool catalog and installation management.
- Updated usage documentation to include new tool installation features.
- Implemented asynchronous tool installation with job tracking.
- Introduced integration tests for tool installation in disposable VMs.
- Enhanced dashboard UI to reflect tool installation status and usage instructions.
- Improved error handling and logging for tool installation processes.
- Added support for multiple Python versions and Conda environments.
- Refactored tool management logic to ensure better state handling and job persistence.

tests/browser_tool_server.py

@@ -0,0 +1,130 @@
+"""Browser fixture with the real API/job workers and simulated guest execution."""
+
+from __future__ import annotations
+
+import argparse
+import json
+import shlex
+import threading
+from dataclasses import replace
+from pathlib import Path
+from types import SimpleNamespace
+from unittest.mock import patch
+
+import uvicorn
+
+from nest.server import create_app
+from tests.browser_session_server import BrowserLifecycle
+from uvm.config import Settings
+from uvm.domain import new_vm_id
+from uvm.errors import ConflictError, GuestExecutionError
+from uvm.guest_exec import ExecResult
+from uvm.state import StateStore
+from uvm.tool_store import ToolJobStore
+from uvm.tools import ToolService
+
+
+class FixtureConnection:
+    def __init__(self):
+        self.closed = threading.Event()
+
+    def close(self):
+        self.closed.set()
+
+
+class ToolBrowserLifecycle(BrowserLifecycle):
+    def __init__(self, settings, state, *, delay=1.2):
+        super().__init__()
+        self.settings, self.state, self.delay = settings, state, delay
+        self.tools = ToolJobStore(settings, state)
+        self.installed = {}
+        self.attempts = {}
+        for index, vm in enumerate(self.records.values()):
+            vm.runtime_owned = True
+            vm.pid, vm.process_start_time, vm.boot_id = index + 100, "1", "fixture-boot"
+            settings.vm_dir(vm.id).mkdir(parents=True, mode=0o700, exist_ok=True)
+        with state.transaction() as document:
+            document.vms.update(self.records)
+
+    def create(self, spec):
+        vm = replace(self.vm, id=new_vm_id(), name=spec.name, username=spec.username, cpu=spec.cpu,
+                     ram_mib=spec.ram_mib, storage_mib=spec.storage_mib, ports=spec.ports)
+        with self.state.operation_lock():
+            self.settings.vm_dir(vm.id).mkdir(mode=0o700)
+            self.records[vm.id] = vm
+            with self.state.transaction() as document:
+                document.vms[vm.id] = vm
+            if spec.tools:
+                self.tools.enqueue_locked(vm.id, spec.tools, source="create")
+        return vm
+
+    def queue_tools(self, vm_id, selections, **options):
+        self.state.get(vm_id)
+        with self.state.operation_lock():
+            return self.tools.enqueue_locked(vm_id, selections, **options)
+
+    def prepare_tool_install(self, vm_id, *, start_if_stopped=False, restart=False):
+        with self.state.operation_lock():
+            vm = self.records[vm_id]
+            if vm.status == "stopped" and not (start_if_stopped or restart):
+                raise ConflictError("start the VM first")
+            vm.status = "running"
+            with self.state.transaction() as document:
+                document.vms[vm.id] = vm
+            return vm
+
+    def execute(self, vm_id, command, *, expected_identity=None, on_connect=None, **_options):
+        self.state.get(vm_id)
+        payload = json.loads(shlex.split(command)[-1])
+        connection = FixtureConnection()
+        if on_connect:
+            on_connect(connection)
+        installed = self.installed.setdefault(vm_id, {"curl"})
+
+        def observation(tool):
+            versions = {"nodejs": "24.21.0", "jdk": "21.0.12", "php": "8.5.11", "conda": "26.5.3", "micromamba": "2.9.0"}
+            version = tool["version"] + ".15" if tool["id"] == "python" else versions.get(tool["id"], "2.0.0")
+            return {"key": tool["key"], "id": tool["id"], "version": tool["version"], "installed": tool["key"] in installed,
+                    "detected_version": version if tool["key"] in installed else None, "error": None, "usage": tool["usage"]}
+
+        if payload["action"] == "probe":
+            result = {"ok": True, "busy": False, "boot_id": "guest-fixture", "supported": True,
+                      "tools": [observation(tool) for tool in payload["tools"]]}
+        else:
+            if connection.closed.wait(self.delay):
+                raise GuestExecutionError("fixture connection closed", outcome_unknown=True)
+            tool = payload["tools"][0]
+            key = (vm_id, tool["key"])
+            self.attempts[key] = self.attempts.get(key, 0) + 1
+            if tool["id"] == "php" and self.attempts[key] == 1:
+                result = {"ok": False, "error": "Simulated mirror failure", "output": "<script>bad()</script>\nTry again later."}
+            else:
+                installed.add(tool["key"])
+                result = {"ok": True, "status": "installed", "tool": observation(tool),
+                          "output": f"Verified and installed {tool['key']} inside the guest."}
+        return ExecResult(0, json.dumps(result), "", False, False)
+
+
+def fixture_application(base, *, delay=1.2):
+    settings = replace(Settings(), base=base, api_token="browser-test-token", tool_workers=2)
+    state = StateStore(settings)
+    lifecycle = ToolBrowserLifecycle(settings, state, delay=delay)
+    return SimpleNamespace(settings=settings, state_store=state, lifecycle=lifecycle,
+                           tools=ToolService(settings, state, lifecycle))
+
+
+def main():
+    parser = argparse.ArgumentParser()
+    parser.add_argument("--base", type=Path, required=True)
+    parser.add_argument("--port", type=int, required=True)
+    args = parser.parse_args()
+    application = fixture_application(args.base)
+    with patch("uvm.tools.require_root"):
+        api = create_app(application, host="127.0.0.1")
+        server = uvicorn.Server(uvicorn.Config(api, host="127.0.0.1", port=args.port, log_level="warning"))
+        api.state.uvicorn_server = server
+        server.run()
+
+
+if __name__ == "__main__":
+    main()

tests/browser_tools.cjs

@@ -0,0 +1,106 @@
+// Optional browser integration: real API/job persistence, simulated guest commands.
+const assert = require('node:assert/strict');
+const { spawn } = require('node:child_process');
+const { mkdtemp, rm } = require('node:fs/promises');
+const os = require('node:os');
+const path = require('node:path');
+const { chromium } = require('playwright');
+
+(async () => {
+  const temporary = await mkdtemp(path.join(os.tmpdir(), 'nest-tool-browser-'));
+  const port = process.env.NEST_BROWSER_TEST_PORT || '18766';
+  const origin = `http://127.0.0.1:${port}`;
+  const server = spawn(process.env.NEST_TEST_PYTHON || 'python3',
+    ['-B', '-m', 'tests.browser_tool_server', '--base', temporary, '--port', port],
+    { cwd: path.resolve(__dirname, '..'), stdio: ['ignore', 'pipe', 'pipe'] });
+  let logs = '';
+  server.stdout.on('data', (data) => { logs = (logs + data).slice(-16000); });
+  server.stderr.on('data', (data) => { logs = (logs + data).slice(-16000); });
+  let browser;
+  let page;
+  try {
+    let ready = false;
+    for (let attempt = 0; attempt < 100; attempt++) {
+      if (server.exitCode !== null) throw new Error(`Fixture server exited: ${logs}`);
+      try { if ((await fetch(`${origin}/health`)).ok) { ready = true; break; } } catch {}
+      await new Promise((resolve) => setTimeout(resolve, 100));
+    }
+    assert.ok(ready, logs);
+    browser = await chromium.launch({ headless: true });
+    page = await browser.newPage({ viewport: { width: 1440, height: 1100 } });
+    const errors = [];
+    page.on('pageerror', (error) => errors.push(error.message));
+    const installed = (key) => page.waitForFunction((value) =>
+      document.querySelector(`[data-install-tool="${value}"]`)?.textContent === 'Installed', key);
+    await page.goto(`${origin}/dashboard#create`);
+    await page.locator('#token-input').fill('browser-test-token');
+    await page.locator('#auth-btn').click();
+    await page.locator('[data-create-tool="python-3.12"]').check();
+    await page.locator('[data-create-tool="git"]').check();
+    await page.locator('[data-create-tool="nodejs"]').check();
+    assert.equal(await page.locator('[data-create-tool]').count(), 12);
+    assert.match(await page.locator('#create-summary-details').textContent(), /Git, Node.js, Python 3.12/);
+    const layout = await page.evaluate(() => ({
+      parent: document.querySelector('.create-tools-section').parentElement.className,
+      children: document.querySelector('#create-form').children.length,
+    }));
+    assert.equal(layout.parent, 'create-fields');
+    assert.equal(layout.children, 2);
+    await page.locator('#create-name').fill('tool-workspace');
+    await page.locator('#create-submit').click();
+    await page.waitForURL((url) => /#vm\/vm-[a-f0-9]+\/tools$/.test(url.hash));
+    const createdUrl = page.url();
+    await page.locator('#tools-job').waitFor({ state: 'visible' });
+    // A reload can occur while the server-owned worker is still installing.
+    await page.reload();
+    await page.waitForFunction(() => document.querySelector('#tools-job-title').textContent === 'Complete');
+    await installed('curl');
+    assert.equal(await page.locator('.tool-installed').count(), 4); // Includes image-provided curl.
+    assert.ok(await page.locator('[data-install-tool="python-3.12"]').isDisabled());
+    await page.locator('#tools-job-items details').first().locator('summary').click();
+    assert.match(await page.locator('#tools-job-items').textContent(), /Verified and installed git/);
+    await page.locator('[data-install-tool="php"]').click();
+    await page.waitForFunction(() => document.querySelector('#tools-job-title').textContent === 'Failed');
+    await page.locator('#tools-job-items details[data-output="php"] summary').click();
+    assert.match(await page.locator('.tool-output').textContent(), /<script>bad/);
+    assert.equal(await page.locator('#tools-job-items script').count(), 0);
+    await page.locator('#tools-retry').click();
+    await page.waitForFunction(() => document.querySelector('#tools-job-title').textContent === 'Complete');
+    await installed('php');
+    assert.ok(await page.locator('[data-install-tool="php"]').isDisabled());
+    assert.equal(await page.locator('#tools-job-select option').count(), 3);
+
+    // The same click boots a stopped VM before installing its selected tool.
+    await page.goto(`${origin}/dashboard#vm/vm-other/tools`);
+    await page.locator('[data-install-tool="htop"]').waitFor();
+    assert.equal(await page.locator('[data-install-tool="htop"]').textContent(), 'Start & install');
+    await page.locator('[data-install-tool="htop"]').click();
+    await installed('htop');
+    await page.waitForFunction(() => document.querySelector('#tools-job-title').textContent === 'Complete');
+    assert.ok(await page.locator('[data-install-tool="htop"]').isDisabled());
+    assert.ok(!(await page.locator('#tools-job-items').textContent()).includes('python-3.12'));
+
+    await page.goto(createdUrl);
+    await page.locator('[data-tool-select="wget"]').check();
+    await page.locator('[data-tool-select="micromamba"]').check();
+    await page.locator('#tools-install-selected').click();
+    await installed('wget');
+    await installed('micromamba');
+    await page.waitForFunction(() => document.querySelector('#tools-job-title').textContent === 'Complete');
+    await page.setViewportSize({ width: 390, height: 844 });
+    await page.waitForTimeout(250);
+    assert.ok(await page.evaluate(() => document.documentElement.scrollWidth <= window.innerWidth), 'mobile layout overflows');
+    if (process.env.NEST_BROWSER_SCREENSHOT) await page.screenshot({ path: process.env.NEST_BROWSER_SCREENSHOT, fullPage: true });
+    assert.deepEqual(errors, []);
+    console.log('Browser tools passed: creation, background/reload, detection, failure/retry, safe output, history, Start & install, VM isolation, multi-select, and mobile layout.');
+  } catch (error) {
+    if (page) console.error('Tool UI:', await page.locator('#tools-error').textContent(), await page.locator('#tools-job-message').textContent());
+    console.error(logs);
+    throw error;
+  } finally {
+    await browser?.close();
+    server.kill('SIGTERM');
+    if (server.exitCode === null) await new Promise((resolve) => server.once('exit', resolve));
+    await rm(temporary, { recursive: true, force: true });
+  }
+})().catch((error) => { console.error(error); process.exitCode = 1; });

tests/integration/test_guest_tools.py

@@ -0,0 +1,96 @@
+"""Opt-in tool installation smoke test against a prepared, disposable microVM."""
+
+from __future__ import annotations
+
+import json
+import os
+import time
+import unittest
+import uuid
+from urllib.parse import quote
+from urllib.request import Request, urlopen
+
+
+@unittest.skipUnless(os.environ.get("UVM_TOOLS_INTEGRATION") == "1",
+                     "set UVM_TOOLS_INTEGRATION=1 on an isolated disposable VM host")
+class GuestToolsIntegrationTests(unittest.TestCase):
+    @classmethod
+    def setUpClass(cls):
+        cls.api_url = os.environ.get("UVM_API_URL", "http://127.0.0.1:8000").rstrip("/")
+        cls.token = os.environ["UVM_API_TOKEN"]
+        cls.vm_id = os.environ["UVM_TOOLS_VM_ID"]
+        cls.path = "/vms/" + quote(cls.vm_id, safe="")
+        cls.vm = cls.request("GET", cls.path)
+        if (cls.vm["id"] != cls.vm_id or cls.vm["observed_status"] != "running"
+                or cls.vm["username"] == "root" or cls.vm["ports"] != []):
+            raise RuntimeError("provide a disposable running Ubuntu 22.04 VM with a non-root user and no open TCP ports")
+
+    @classmethod
+    def request(cls, method, path, body=None, *, timeout=120):
+        request = Request(cls.api_url + path, method=method,
+                          data=json.dumps(body).encode() if body is not None else None,
+                          headers={"X-UVM-Token": cls.token, "Content-Type": "application/json"})
+        with urlopen(request, timeout=timeout) as response:
+            return json.load(response)
+
+    def test_install_catalog_and_use_runtimes_as_the_guest_user(self):
+        catalog = self.request("GET", "/tools")
+        selections = [{"id": tool["id"], "version": version} for tool in catalog for version in tool["versions"] or [None]]
+        body = {"tools": selections, "request_id": str(uuid.uuid4())}
+        job = self.request("POST", self.path + "/tools/install", body)
+        deadline = time.monotonic() + 3600
+        while job["status"] in {"queued", "starting", "waiting", "installing"} and time.monotonic() < deadline:
+            time.sleep(2)
+            job = self.request("GET", self.path + "/tools/jobs/" + job["id"])
+        self.assertEqual(job["status"], "succeeded", json.dumps(job, indent=2))
+        self.assertEqual(self.request("POST", self.path + "/tools/install", body)["id"], job["id"])
+        inventory = self.request("GET", self.path + "/tools?refresh=true")
+        self.assertEqual(len(inventory["inventory"]), 12)
+        self.assertTrue(all(item["installed"] for item in inventory["inventory"]), inventory)
+
+        script = r'''set -eu
+test "$(id -u)" != 0
+git --version
+node -e 'if (1 + 1 !== 2) process.exit(1)'
+npm --version
+wget --version >/dev/null
+curl --version >/dev/null
+htop --version
+php -r 'if (2 + 2 !== 4) exit(1);'
+work=$(mktemp -d "$HOME/nest-tools-smoke.XXXXXX")
+name=$(basename "$work")
+export MAMBA_ROOT_PREFIX="$HOME/micromamba"
+cleanup() {
+    conda env remove -n "$name" -y >/dev/null 2>&1 || true
+    micromamba env remove -n "$name" -y >/dev/null 2>&1 || true
+    rm -rf -- "$work"
+}
+trap cleanup EXIT
+printf 'class Hello { public static void main(String[] args) { System.out.println("ok"); } }\n' > "$work/Hello.java"
+javac "$work/Hello.java"
+test "$(java -cp "$work" Hello)" = ok
+for version in 3.11 3.12 3.13; do
+    "python$version" -m venv "$work/venv-$version"
+    "$work/venv-$version/bin/python" -c 'import ssl, sqlite3, sys; assert sys.prefix != sys.base_prefix'
+    "$work/venv-$version/bin/python" -m pip --version
+done
+conda create -y -n "$name" python=3.13
+conda run -n "$name" python -c 'import sys; assert sys.version_info[:2] == (3, 13)'
+eval "$(conda shell.posix hook)"
+conda activate "$name"
+python -c 'import sys; assert sys.version_info[:2] == (3, 13)'
+conda deactivate
+micromamba create -y -n "$name" -c conda-forge python=3.12
+micromamba run -n "$name" python -c 'import sys; assert sys.version_info[:2] == (3, 12)'
+eval "$(micromamba shell hook --shell posix)"
+micromamba activate "$name"
+python -c 'import sys; assert sys.version_info[:2] == (3, 12)'
+micromamba deactivate
+'''
+        result = self.request("POST", self.path + "/exec",
+                              {"command": script, "sudo": False, "timeout_seconds": 1200}, timeout=1220)
+        self.assertFalse(result["timed_out"], result)
+        self.assertEqual(result["exit_code"], 0, result)
+        after = self.request("GET", self.path)
+        for key in ("id", "guest_ip", "mac", "username", "ports", "cpu", "ram_mib", "storage_mib"):
+            self.assertEqual(after[key], self.vm[key])

tests/test_tool_api.py

@@ -11,7 +11,6 @@ from tests.test_server import FASTAPI_AVAILABLE, FakeLifecycle
 from uvm.config import Settings
 from uvm.server import create_api
 from uvm.state import StateStore
-from uvm.tool_store import ToolJobStore
 from uvm.tools import ToolService
 
 if FASTAPI_AVAILABLE:

tests/test_tool_catalog.py

@@ -1,15 +1,15 @@
 from __future__ import annotations
 
+import ast
 import hashlib
 import io
-import os
 import shlex
 import tarfile
 import tempfile
 import unittest
 from pathlib import Path
 from types import SimpleNamespace
-from unittest.mock import Mock, patch
+from unittest.mock import patch
 
 from uvm import guest_tools
 from uvm.errors import ValidationError
@@ -28,6 +28,7 @@ class ToolCatalogTests(unittest.TestCase):
             argv = shlex.split(command)
             self.assertEqual(argv[:3], ["/usr/bin/python3", "-I", "-c"])
             compile(argv[3], "guest-tools", "exec")
+            ast.parse(argv[3], feature_version=(3, 10))
             for item in recipes:
                 if "url" in item:
                     self.assertTrue(item["url"].startswith("https://"))

tests/test_tool_flow.py

@@ -0,0 +1,73 @@
+from __future__ import annotations
+
+import tempfile
+import time
+import unittest
+from pathlib import Path
+from unittest.mock import patch
+
+from tests.test_server import FASTAPI_AVAILABLE
+
+if FASTAPI_AVAILABLE:
+    from fastapi.testclient import TestClient
+    from tests.browser_tool_server import fixture_application
+    from uvm.server import create_api
+
+
+@unittest.skipUnless(FASTAPI_AVAILABLE, "FastAPI and httpx are required")
+class ToolFlowTests(unittest.TestCase):
+    def setUp(self):
+        temporary = self.enterContext(tempfile.TemporaryDirectory())
+        self.application = fixture_application(Path(temporary), delay=0.01)
+        self.enterContext(patch("uvm.tools.require_root"))
+        self.headers = {"X-UVM-Token": "browser-test-token"}
+
+    def wait_for(self, condition):
+        deadline = time.monotonic() + 5
+        while time.monotonic() < deadline:
+            if value := condition():
+                return value
+            time.sleep(0.01)
+        self.fail("background tool worker did not reach the expected state")
+
+    def test_create_and_later_install_use_real_workers_and_durable_job_responses(self):
+        with TestClient(create_api(self.application, host="127.0.0.1")) as client:
+            response = client.post("/vms", headers=self.headers, json={"name": "tools-vm", "tools": [
+                {"id": "git"}, {"id": "python", "version": "3.12"},
+            ]})
+            self.assertEqual(response.status_code, 201, response.text)
+            vm_id = response.json()["id"]
+            self.wait_for(lambda: self.application.tools.store.read(vm_id)["jobs"][-1]["status"] == "succeeded")
+            snapshot = client.get(f"/vms/{vm_id}/tools?refresh=true", headers=self.headers)
+            self.assertEqual(snapshot.status_code, 200, snapshot.text)
+            installed = {item["key"] for item in snapshot.json()["inventory"] if item["installed"]}
+            self.assertEqual(installed, {"git", "python-3.12", "curl"})
+            self.assertEqual(snapshot.json()["latest_job"]["source"], "create")
+            later = client.post("/vms/vm-other/tools/install", headers=self.headers,
+                                json={"tools": [{"id": "htop"}], "start_if_stopped": True})
+            self.assertEqual(later.status_code, 202, later.text)
+            job_id = later.json()["id"]
+            self.wait_for(lambda: self.application.tools.store.get_job("vm-other", job_id)["status"] == "succeeded")
+            self.assertEqual(self.application.state_store.get("vm-other").status, "running")
+            self.assertEqual(client.get(f"/vms/{vm_id}/tools/jobs/{job_id}", headers=self.headers).status_code, 404)
+
+    def test_queued_jobs_are_dispatched_on_server_start(self):
+        with self.application.state_store.operation_lock():
+            job = self.application.tools.store.enqueue_locked("vm-test", [{"id": "git"}])
+        with TestClient(create_api(self.application, host="127.0.0.1")):
+            self.wait_for(lambda: self.application.tools.store.get_job("vm-test", job["id"])["status"] == "succeeded")
+        self.assertEqual(self.application.lifecycle.attempts[("vm-test", "git")], 1)
+
+    def test_shutdown_wakes_command_waiter_and_keeps_an_interrupted_job(self):
+        self.application.lifecycle.delay = 30
+        with TestClient(create_api(self.application, host="127.0.0.1")) as client:
+            response = client.post("/vms/vm-test/tools/install", headers=self.headers, json={"tools": [{"id": "git"}]})
+            job_id = response.json()["id"]
+            self.wait_for(lambda: self.application.tools.store.get_job("vm-test", job_id)["status"] == "installing"
+                          and bool(self.application.tools._sessions))
+            started = time.monotonic()
+        self.assertLess(time.monotonic() - started, 3)
+        job = self.application.tools.store.get_job("vm-test", job_id)
+        self.assertEqual(job["status"], "interrupted")
+        self.assertTrue(job["outcome_unknown"])
+        self.assertTrue(all(not thread.is_alive() for thread in self.application.tools._threads))

tests/test_tool_jobs.py

@@ -1,9 +1,7 @@
 from __future__ import annotations
 
 import json
-import os
 import tempfile
-import threading
 import unittest
 from concurrent.futures import ThreadPoolExecutor
 from dataclasses import replace