Implement guest tool installation and management system

- Added `guest_tools.py` for standalone tool installer functionality.
- Introduced `ToolJobStore` in `tool_store.py` to manage per-VM tool job records.
- Created `ToolService` in `tools.py` for background installation workers using the guest execution channel.
- Enhanced `LifecycleService` in `lifecycle.py` to support tool installation queuing and preparation.
- Updated `guest_exec.py` to handle connection shutdown gracefully.
- Added validation and normalization functions for tool specifications.
- Implemented logging and error handling for tool installation processes.
- Ensured compatibility with Ubuntu 22.04 and ARM64/x86_64 architectures.

uvm/lifecycle.py

@@ -26,6 +26,8 @@ from .known_hosts import KnownHostsManager
 from .network import NetworkManager
 from .state import StateStore
 from .system import check_kvm, ensure_data_directories, require_root
+from .tool_catalog import normalize_tools
+from .tool_store import ToolJobStore
 from .validation import parse_name, parse_password, parse_ports, parse_timeout, parse_username
 
 
@@ -75,6 +77,7 @@ class LifecycleService:
         self._guest_connector = guest_connector
         self._terminal_connector = terminal_connector
         self._clock = clock
+        self._tool_store = ToolJobStore(settings, state_store, clock=clock)
 
     def create(self, spec: VmSpec) -> VmRecord:
         require_root()
@@ -84,6 +87,7 @@ class LifecycleService:
         if spec.name is not None:
             parse_name(spec.name)
         ports = self._validated_ports(spec.ports)
+        tools = normalize_tools(spec.tools, allow_empty=True)
         ensure_data_directories(self._settings)
         with self._state_store.operation_lock():
             self._check_boot()
@@ -100,6 +104,8 @@ class LifecycleService:
                 self._images.provision_guest_agent(disk, vm.id, vm.username)
                 vm.disk_ready = True
                 self._save(vm)
+                if tools:
+                    self._tool_store.enqueue_locked(vm.id, tools, source="create")
                 self._boot(vm, assets.kernel)
                 return vm
             except BaseException as error:
@@ -506,6 +512,8 @@ class LifecycleService:
     def execute(
         self, vm_id: str, command: str, *, sudo: bool = False,
         timeout_seconds: float = DEFAULT_TIMEOUT_SECONDS,
+        expected_identity: tuple | None = None,
+        on_connect: Callable[[GuestSession], None] | None = None,
     ) -> ExecResult:
         """Pin a running VM's guest connection under the lock, then wait unlocked."""
         require_root()
@@ -516,14 +524,52 @@ class LifecycleService:
             if (vm.status != "running" or vm.operation is not None or vm.pending_resize is not None
                     or self._process.observe(vm) != "alive"):
                 raise ConflictError("guest execution requires a running VM with no pending lifecycle operation")
+            if expected_identity is not None and expected_identity != (vm.pid, vm.process_start_time, vm.boot_id):
+                raise ConflictError("the VM restarted during tool installation; retry on the current boot")
             session = self._guest_connector(self._settings.guest_socket_path(vm.id), vm.id, vm.username, vm.pid)
         try:
+            if on_connect is not None:
+                on_connect(session)
             # This session stays attached to that VMM even if the VM is stopped,
             # deleted, or restarted concurrently. Never reconnect/retry a command.
             return session.execute(command, sudo=sudo, timeout_seconds=timeout_seconds)
         finally:
             session.close()
 
+    def _validate_tool_target(self, vm: VmRecord, *, start_if_stopped: bool, restart: bool) -> str:
+        self._not_deleting(vm)
+        observation = self._process.observe(vm)
+        if vm.operation is not None or vm.pending_resize is not None or not vm.disk_ready:
+            raise ConflictError("finish pending lifecycle operations before installing tools")
+        if vm.status == "running" and observation == "alive":
+            return "running"
+        if vm.status == "stopped" and observation == "gone" and (start_if_stopped or restart):
+            self._validate_resume(vm)
+            return "stopped"
+        raise ConflictError("tool installation requires a running VM; use Start & install for a stopped VM")
+
+    def queue_tools(self, vm_id: str, selections, *, start_if_stopped=False, restart=False, request_id=None) -> dict:
+        require_root()
+        selections = normalize_tools(selections)
+        with self._state_store.operation_lock():
+            vm = self._load_for_operation(vm_id, exact_id=True)
+            self._validate_tool_target(vm, start_if_stopped=start_if_stopped, restart=restart)
+            return self._tool_store.enqueue_locked(vm.id, selections, start_if_stopped=start_if_stopped,
+                                                   restart=restart, request_id=request_id)
+
+    def prepare_tool_install(self, vm_id: str, *, start_if_stopped=False, restart=False) -> VmRecord:
+        """Perform requested startup/agent refresh atomically before a tool worker connects."""
+        require_root()
+        with self._state_store.operation_lock():
+            vm = self._load_for_operation(vm_id, exact_id=True)
+            status = self._validate_tool_target(vm, start_if_stopped=start_if_stopped, restart=restart)
+            if status == "running" and restart:
+                self._stop_locked(vm)
+            if status == "stopped" or restart:
+                # The normal retained-disk boot also provisions the current agent.
+                vm = self._start_locked(vm)
+            return vm
+
     def open_terminal(self, vm_id: str, *, rows: int = 24, cols: int = 80) -> tuple[GuestTerminal, str]:
         """Connect to the exact running VMM under the lock, then open its guest PTY."""
         require_root()