"""Serialized lifecycle and private-port policy changes with durable intent.""" from __future__ import annotations import re import shutil import time from collections.abc import Callable, Sequence from dataclasses import dataclass, replace from ipaddress import IPv4Network from pathlib import Path from .config import Settings from .domain import PortRule, VmRecord, VmSpec, new_vm_id from .errors import ConflictError, NotFoundError, UvmError, ValidationError from .firecracker.api import FirecrackerClient from .firecracker.config import build_config, write_config from .firecracker.process import FirecrackerProcessManager, ProcessInfo from .images import ImageStore from .integrity import load_manifest, verify_file from .network import NetworkManager from .state import StateStore from .system import check_kvm, ensure_data_directories, require_root from .validation import parse_cpu, parse_name, parse_password, parse_ports, parse_username @dataclass(frozen=True, slots=True) class ListedVm: vm: VmRecord observed_status: str policy_status: str = "unknown" @dataclass(frozen=True, slots=True) class OperationResult: id: str action: str outcome: str error: str | None = None class LifecycleService: """All mutating entry points use operation lock -> short state transaction.""" def __init__( self, settings: Settings, state_store: StateStore, images: ImageStore, network: NetworkManager, process: FirecrackerProcessManager, *, client_factory: Callable[[Path, float], FirecrackerClient] = FirecrackerClient, clock: Callable[[], float] = time.time, ) -> None: self._settings = settings self._state_store = state_store self._images = images self._network = network self._process = process self._client_factory = client_factory self._clock = clock def create(self, spec: VmSpec) -> VmRecord: require_root() parse_cpu(str(spec.cpu)) if type(spec.ram_mib) is not int or spec.ram_mib < 128: raise ValidationError("RAM must be an integer of at least 128 MiB") parse_username(spec.username) parse_password(spec.password) if spec.name is not None: parse_name(spec.name) ports = self._validated_ports(spec.ports) ensure_data_directories(self._settings) with self._state_store.operation_lock(): self._check_boot() assets = self._images.installed_assets() vm = self._reserve_vm(replace(spec, ports=ports)) try: runtime = self._settings.vm_dir(vm.id) runtime.mkdir(mode=0o700, exist_ok=False) vm.runtime_owned = True self._save(vm) disk = self._images.create_vm_disk(assets.rootfs, Path(vm.disk)) self._images.provision_credentials(disk, vm.username, vm.password) vm.disk_ready = True self._save(vm) self._boot(vm, assets.kernel) return vm except BaseException as error: cleanup_error = None try: if vm.operation != "preparing": self._cleanup_runtime(vm) self._remove_runtime(vm) self._remove_record(vm.id) except BaseException as cleanup: cleanup_error = cleanup message = f"failed to create {vm.id}: {error}" if cleanup_error is not None: message += f"; cleanup incomplete: {cleanup_error}" self._record_failure(vm, message) if isinstance(error, (KeyboardInterrupt, SystemExit)): raise raise UvmError(message) from error def _check_boot(self) -> None: if not any(self._settings.network.subnet_of(IPv4Network(network)) for network in ("10.0.0.0/8", "172.16.0.0/12", "192.168.0.0/16")): raise ValidationError("private ports require a network inside an RFC 1918 IPv4 range") check_kvm() self._verify_firecracker() self._network.firewall.check_available() def _verify_firecracker(self) -> None: manifest = load_manifest(self._settings.integrity_manifest_path) checksum = self._settings.firecracker_binary_sha256 if manifest.verified or self._settings.allow_unverified_downloads: checksum = checksum or manifest.checksums.get("firecracker") verify_file(self._settings.firecracker_binary, checksum, "Firecracker binary", "UVM_FIRECRACKER_BINARY_SHA256", allow_unverified=self._settings.allow_unverified_downloads) def _boot(self, vm: VmRecord, kernel: Path) -> None: vm.status, vm.operation = "starting", "network" vm.pid = vm.process_start_time = None vm.spawn_pending = False self._save(vm) self._network.ensure_bridge() self._network.prepare_tap(vm) self._save(vm) self._network.firewall.apply(vm) self._network.activate_tap(vm) config = build_config(self._settings, vm, kernel, Path(vm.disk)) write_config(Path(vm.config), config) # Persist the crash window before spawning, but give start() the already # verified process-free record. A restored pending spawn is never guessed gone. vm.operation = "boot" vm.boot_id = self._process.host_boot_id() self._save(replace(vm, operation="spawning", spawn_pending=True)) def spawned(info: ProcessInfo) -> None: vm.pid, vm.process_start_time, vm.boot_id = info.pid, info.start_time, info.boot_id vm.spawn_pending = False vm.operation = "configuring" self._save(vm) self._process.start(vm, on_spawn=spawned) client = self._client_factory(Path(vm.socket), self._settings.api_timeout_s) client.configure_and_start(config) if self._network.firewall.status(vm) != "enforced": raise UvmError("private-port enforcement could not be confirmed after boot") vm.status, vm.operation, vm.last_error = "running", None, None self._save(vm) def start(self, identifier: str) -> VmRecord: require_root() with self._state_store.operation_lock(): return self._start_locked(self._load_for_operation(identifier)) def _start_locked(self, vm: VmRecord) -> VmRecord: self._not_deleting(vm) if vm.ports is None: raise ConflictError("legacy VM has no port policy; stop it, use ports set, then start") observation = self._process.observe(vm) if observation == "unknown": raise ConflictError(f"cannot start {vm.id}: process identity is unknown; inspect before retrying") if observation == "alive": if vm.status != "running" or vm.operation not in {None, "policy"}: raise ConflictError(f"{vm.id} has an interrupted boot/stop; stop it before starting") return self._sync_ports_locked(vm) if not vm.disk_ready: raise ConflictError(f"{vm.id} has no completely provisioned private disk") disk = Path(vm.disk) if not disk.is_file() or disk.is_symlink() or disk.stat().st_nlink != 1: raise UvmError(f"private VM disk is missing or unsafe: {disk}") if (vm.gateway != str(self._settings.gateway) or vm.network not in {None, str(self._settings.network)} or vm.bridge not in {None, self._settings.bridge}): raise ConflictError("configured network differs from the VM's retained network") self._check_boot() kernel = self._images.installed_kernel() try: self._cleanup_runtime(vm) vm.network, vm.bridge = str(self._settings.network), self._settings.bridge self._boot(vm, kernel) return vm except BaseException as error: message = f"failed to start {vm.id}: {error}" try: self._cleanup_runtime(vm) except BaseException as cleanup: message += f"; cleanup incomplete: {cleanup}" self._record_failure(vm, message) if isinstance(error, (KeyboardInterrupt, SystemExit)): raise raise UvmError(message) from error def stop(self, identifier: str, *, timeout: float | None = None) -> VmRecord: require_root() with self._state_store.operation_lock(): return self._stop_locked(self._load_for_operation(identifier), timeout=timeout) def _stop_locked(self, vm: VmRecord, *, timeout: float | None = None) -> VmRecord: deleting = vm.operation == "delete" or vm.status == "terminating" vm.status, vm.operation = "stopping", "delete" if deleting else "stop" self._save(vm) try: self._cleanup_runtime(vm, timeout=timeout) vm.status = "stopped" vm.operation = "delete" if deleting else None vm.last_error = None self._save(vm) return vm except BaseException as error: self._record_failure(vm, f"failed to stop {vm.id}: {error}") if isinstance(error, (KeyboardInterrupt, SystemExit)): raise raise UvmError(vm.last_error) from error def restart(self, identifier: str, *, timeout: float | None = None) -> VmRecord: require_root() with self._state_store.operation_lock(): vm = self._load_for_operation(identifier) self._not_deleting(vm) self._stop_locked(vm, timeout=timeout) return self._start_locked(vm) def destroy(self, identifier: str, *, timeout: float | None = None) -> VmRecord: require_root() with self._state_store.operation_lock(): return self._delete_locked(self._load_for_operation(identifier), timeout=timeout) def _delete_locked(self, vm: VmRecord, *, timeout: float | None = None) -> VmRecord: vm.status, vm.operation = "terminating", "delete" self._save(vm) try: self._cleanup_runtime(vm, timeout=timeout) self._remove_runtime(vm) self._remove_record(vm.id) return vm except BaseException as error: self._record_failure(vm, f"failed to delete {vm.id}: {error}") if isinstance(error, (KeyboardInterrupt, SystemExit)): raise raise UvmError(vm.last_error) from error def _cleanup_runtime(self, vm: VmRecord, *, timeout: float | None = None) -> None: containment_errors = [] try: if self._network.has_tap(vm): try: self._network.firewall.apply(vm, quarantine=True) finally: self._network.deactivate_tap(vm) except UvmError as error: containment_errors.append(str(error)) try: self._process.terminate(vm, timeout=timeout) except UvmError as error: detail = f"; containment: {'; '.join(containment_errors)}" if containment_errors else "" raise UvmError(f"process exit unconfirmed: {error}{detail}") from error vm.pid = vm.process_start_time = None vm.spawn_pending = False if vm.operation == "spawning": vm.operation = "boot" self._network.delete_vm_tap(vm) self._network.firewall.cleanup_connections(vm) self._network.firewall.remove(vm) def set_ports(self, identifier: str, ports: tuple[PortRule, ...], *, mode: str = "set") -> VmRecord: """Mutate a frozen registry ID; the CLI resolves names before confirmation.""" require_root() ports = self._validated_ports(ports) if mode not in {"open", "close", "set"}: raise ValidationError(f"invalid port operation: {mode}") with self._state_store.operation_lock(): vm = self._load_for_operation(identifier, exact_id=True) self._not_deleting(vm) observation = self._process.observe(vm) if observation == "unknown": raise ConflictError("port changes require a known process identity") if vm.ports is None and (mode != "set" or observation != "gone" or vm.status != "stopped"): raise ConflictError("legacy VM policy must be explicitly set while stopped") if vm.operation not in {None, "policy"}: raise ConflictError(f"finish the pending {vm.operation} operation before editing ports") current = {port.port: port for port in vm.ports or ()} if mode == "set": current = {port.port: port for port in ports} elif mode == "open": for port in ports: current.setdefault(port.port, port) else: for port in ports: current.pop(port.port, None) vm.ports = tuple(current[number] for number in sorted(current)) vm.operation = "policy" self._save(vm) return self._sync_ports_locked(vm) def sync_ports(self, identifier: str) -> VmRecord: require_root() with self._state_store.operation_lock(): return self._sync_ports_locked(self._load_for_operation(identifier)) def _sync_ports_locked(self, vm: VmRecord) -> VmRecord: self._not_deleting(vm) if vm.ports is None: raise ConflictError("legacy VM has no managed policy; stop it and use ports set") if vm.operation not in {None, "policy"}: raise ConflictError(f"finish the pending {vm.operation} operation before syncing ports") observation = self._process.observe(vm) if observation == "unknown": raise ConflictError("cannot enforce policy for a VM with unknown process identity") vm.operation = "policy" self._save(vm) try: if observation == "alive": if not self._network.has_tap(vm): raise UvmError("running VM has no owned network attachment") self._network.firewall.apply(vm) self._network.activate_tap(vm) vm.status = "running" else: if vm.status != "stopped" or vm.tap_owned: self._cleanup_runtime(vm) vm.status = "stopped" vm.operation, vm.last_error = None, None self._save(vm) return vm except BaseException as error: # Never restore an older, more permissive policy after a failed close. containment_errors = [] for contain in ( lambda: self._network.deactivate_tap(vm), lambda: (self._network.firewall.apply(vm, quarantine=True) if self._network.has_tap(vm) else None), ): try: contain() except UvmError as containment_error: containment_errors.append(str(containment_error)) detail = f"; containment errors: {'; '.join(containment_errors)}" if containment_errors else "" self._record_failure(vm, f"policy pending for {vm.id}: {error}{detail}") if isinstance(error, (KeyboardInterrupt, SystemExit)): raise raise UvmError(vm.last_error) from error def resolve_targets(self, identifiers: Sequence[str], *, all_vms: bool = False) -> list[VmRecord]: if all_vms and identifiers: raise ValidationError("--all cannot be combined with explicit VM targets") state = self._state_store.load() if all_vms: return list(state.vms.values()) if not identifiers: raise ValidationError("specify a VM or --all") resolved = [self._state_store.resolve(state, identifier) for identifier in identifiers] return list({vm.id: vm for vm in resolved}.values()) def run_many(self, action: str, vm_ids: Sequence[str], *, timeout: float | None = None) -> list[OperationResult]: require_root() if action not in {"start", "stop", "restart", "delete", "ports-sync"}: raise ValidationError(f"invalid lifecycle action: {action}") results = [] with self._state_store.operation_lock(): for index, vm_id in enumerate(vm_ids): try: vm = self._load_for_operation(vm_id, exact_id=True) if action == "start": self._start_locked(vm) elif action == "stop": self._stop_locked(vm, timeout=timeout) elif action == "delete": self._delete_locked(vm, timeout=timeout) elif action == "ports-sync": self._sync_ports_locked(vm) else: self._not_deleting(vm) self._stop_locked(vm, timeout=timeout) self._start_locked(vm) results.append(OperationResult(vm_id, action, "succeeded")) except NotFoundError: results.append(OperationResult(vm_id, action, "skipped", "already absent")) except KeyboardInterrupt: results.append(OperationResult(vm_id, action, "interrupted", "interrupted")) results.extend(OperationResult(remaining, action, "skipped", "not attempted after interruption") for remaining in vm_ids[index + 1:]) break except (UvmError, OSError) as error: results.append(OperationResult(vm_id, action, "failed", str(error))) return results def list_vms(self) -> list[ListedVm]: return [self._listed(vm) for vm in self._state_store.load().vms.values()] def inspect(self, identifier: str) -> ListedVm: return self._listed(self._state_store.find_by_id_or_ip(identifier)) def _listed(self, vm: VmRecord) -> ListedVm: observation = self._process.observe(vm) status = vm.status if observation == "unknown": status = "unknown" elif observation == "alive" and status == "stopped": status = "running" elif observation == "gone" and status == "running": status = "dead" policy = ("inactive" if observation == "gone" and vm.status == "stopped" and vm.operation is None else self._network.firewall.status(vm)) if vm.ports is None and policy == "inactive": policy = "unmanaged" return ListedVm(vm, status, policy) def find_for_ssh(self, identifier: str) -> VmRecord: return self._state_store.find_by_id_or_ip(identifier) def log_path(self, identifier: str) -> Path: vm = self._state_store.find_by_id_or_ip(identifier) self._validate_paths(vm) return Path(vm.log) def diagnostics(self) -> list[dict]: checks = [("root", require_root), ("kvm", check_kvm), ("Firecracker", self._verify_firecracker), ("guest assets", self._images.installed_assets), ("private firewall", self._network.firewall.check_available)] results = [] for name, check in checks: try: check() results.append({"name": name, "ok": True, "message": "available"}) except (UvmError, OSError) as error: results.append({"name": name, "ok": False, "message": str(error)}) for command in ("ip", "iptables", "nft", "conntrack", "debugfs", "openssl", "ssh-keygen", "ssh"): path = shutil.which(command) results.append({"name": command, "ok": path is not None, "message": path or "not installed"}) return results def _load_for_operation(self, identifier: str, *, exact_id: bool = False) -> VmRecord: vm = (self._state_store.get(identifier) if exact_id else self._state_store.find_by_id_or_ip(identifier)) self._validate_paths(vm) if self._process.adopt_legacy_identity(vm): self._save(vm) return vm def _validate_paths(self, vm: VmRecord) -> None: if re.fullmatch(r"[A-Za-z0-9][A-Za-z0-9_.-]{0,80}", vm.id) is None: raise ConflictError("invalid persisted VM ID; refusing resource access") directory = self._settings.vm_dir(vm.id).absolute() if directory.is_symlink(): raise ConflictError("VM directory must not be a symbolic link") for field, filename in (("socket", "firecracker.sock"), ("config", "config.json"), ("log", "firecracker.log"), ("disk", "rootfs.ext4")): path = getattr(vm, field) if field == "disk" and not path: continue # Legacy records may have used a shared template, never delete it. if Path(path).absolute() != directory / filename or Path(path).is_symlink(): raise ConflictError(f"unsafe persisted VM {field} path") @staticmethod def _not_deleting(vm: VmRecord) -> None: if vm.operation == "delete" or vm.status == "terminating": raise ConflictError(f"deletion is pending for {vm.id}; retry delete instead") @staticmethod def _validated_ports(ports: tuple[PortRule, ...]) -> tuple[PortRule, ...]: if not isinstance(ports, tuple) or any(not isinstance(port, PortRule) for port in ports): raise ValidationError("ports must be a tuple of private TCP port rules") return parse_ports([f"{port.port}:{port.protocol}" for port in ports]) def _reserve_vm(self, spec: VmSpec) -> VmRecord: with self._state_store.transaction() as state: for existing in state.vms.values(): if (existing.guest_ip == str(self._settings.gateway) or existing.gateway != str(self._settings.gateway) or existing.network not in {None, str(self._settings.network)} or existing.bridge not in {None, self._settings.bridge}): raise ConflictError("configured topology conflicts with retained VMs; keep their network settings") if spec.name is not None and any(spec.name in {vm.name, vm.id} for vm in state.vms.values()): raise ConflictError(f"VM name already exists: {spec.name}") vm_id = new_vm_id() used_taps = {vm.tap for vm in state.vms.values()} while vm_id in state.vms or any(vm.name == vm_id for vm in state.vms.values()) or self._network.tap_name(vm_id) in used_taps: vm_id = new_vm_id() guest_ip = self._network.allocate_ip(state.vms.values(), spec.guest_ip) mac = self._network.mac_for(state.next_mac_index) state.next_mac_index += 1 directory = self._settings.vm_dir(vm_id) vm = VmRecord( id=vm_id, name=spec.name, cpu=spec.cpu, ram_mib=spec.ram_mib, guest_ip=str(guest_ip), gateway=str(self._settings.gateway), network=str(self._settings.network), bridge=self._settings.bridge, tap=self._network.tap_name(vm_id), mac=mac, socket=str(directory / "firecracker.sock"), config=str(directory / "config.json"), log=str(directory / "firecracker.log"), disk=str(directory / "rootfs.ext4"), username=spec.username, password=spec.password, ports=spec.ports, status="starting", operation="preparing", created_at=int(self._clock()), updated_at=int(self._clock()), ) state.vms[vm.id] = vm return vm def _remove_runtime(self, vm: VmRecord) -> None: self._validate_paths(vm) directory = self._settings.vm_dir(vm.id) if directory.exists(): if not vm.runtime_owned: raise ConflictError(f"refusing to remove unowned VM directory {directory}") if self._process.observe(vm) != "gone": raise ConflictError("refusing to remove disk without confirmed process exit") shutil.rmtree(directory) vm.runtime_owned = False def _save(self, vm: VmRecord) -> None: vm.updated_at = int(self._clock()) with self._state_store.transaction() as state: if vm.id not in state.vms: raise NotFoundError(f"VM not found: {vm.id}") state.vms[vm.id] = vm def _record_failure(self, vm: VmRecord, message: str) -> None: vm.status, vm.last_error = "failed", message self._save(vm) def _remove_record(self, vm_id: str) -> None: with self._state_store.transaction() as state: state.vms.pop(vm_id, None)