Refactor VM lifecycle management and enhance validation
- Introduced a new method `installed_kernel` in `ImageStore` to handle kernel verification and checksum validation. - Updated `Installer` to include additional packages: `nftables` and `conntrack`. - Refined `LifecycleService` to improve error handling and state management during VM creation, starting, stopping, and deletion. - Enhanced port management with new methods for setting and syncing ports, including validation for port rules. - Improved error handling in `StateStore` to raise specific `NotFoundError` and `ConflictError` exceptions. - Updated network management to ensure proper TAP device handling and firewall integration. - Enhanced API responses in `vms.py` to include policy status for better client-side handling. - Added validation for VM names and ports in `validation.py` to enforce stricter rules and prevent conflicts.
This commit is contained in:
+426
-203
@@ -1,43 +1,49 @@
|
||||
"""Ordered VM lifecycle operations and compensation for partial failures."""
|
||||
"""Serialized lifecycle and private-port policy changes with durable intent."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import re
|
||||
import shutil
|
||||
import time
|
||||
from collections.abc import Callable
|
||||
from dataclasses import dataclass
|
||||
from collections.abc import Callable, Sequence
|
||||
from dataclasses import dataclass, replace
|
||||
from pathlib import Path
|
||||
|
||||
from .config import Settings
|
||||
from .domain import VmRecord, VmSpec, new_vm_id
|
||||
from .errors import TapCreationError, UvmError
|
||||
from .domain import PortRule, VmRecord, VmSpec, new_vm_id
|
||||
from .errors import ConflictError, NotFoundError, UvmError, ValidationError
|
||||
from .firecracker.api import FirecrackerClient
|
||||
from .firecracker.config import build_config, write_config
|
||||
from .firecracker.process import FirecrackerProcessManager
|
||||
from .firecracker.process import FirecrackerProcessManager, ProcessInfo
|
||||
from .images import ImageStore
|
||||
from .integrity import load_manifest, verify_file
|
||||
from .network import NetworkManager
|
||||
from .state import StateStore
|
||||
from .system import check_kvm, ensure_data_directories, require_root
|
||||
from .validation import parse_cpu, parse_name, parse_password, parse_ports, parse_username
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class ListedVm:
|
||||
vm: VmRecord
|
||||
observed_status: str
|
||||
policy_status: str = "unknown"
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class OperationResult:
|
||||
id: str
|
||||
action: str
|
||||
outcome: str
|
||||
error: str | None = None
|
||||
|
||||
|
||||
class LifecycleService:
|
||||
"""The only module allowed to coordinate multiple VM host resources."""
|
||||
"""All mutating entry points use operation lock -> short state transaction."""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
settings: Settings,
|
||||
state_store: StateStore,
|
||||
images: ImageStore,
|
||||
network: NetworkManager,
|
||||
process: FirecrackerProcessManager,
|
||||
*,
|
||||
self, settings: Settings, state_store: StateStore, images: ImageStore,
|
||||
network: NetworkManager, process: FirecrackerProcessManager, *,
|
||||
client_factory: Callable[[Path, float], FirecrackerClient] = FirecrackerClient,
|
||||
clock: Callable[[], float] = time.time,
|
||||
) -> None:
|
||||
@@ -50,235 +56,452 @@ class LifecycleService:
|
||||
self._clock = clock
|
||||
|
||||
def create(self, spec: VmSpec) -> VmRecord:
|
||||
"""Reserve identity first, then create all VM resources in dependency order."""
|
||||
|
||||
require_root()
|
||||
parse_cpu(str(spec.cpu))
|
||||
if type(spec.ram_mib) is not int or spec.ram_mib < 128:
|
||||
raise ValidationError("RAM must be an integer of at least 128 MiB")
|
||||
parse_username(spec.username)
|
||||
parse_password(spec.password)
|
||||
if spec.name is not None:
|
||||
parse_name(spec.name)
|
||||
ports = self._validated_ports(spec.ports)
|
||||
ensure_data_directories(self._settings)
|
||||
check_kvm()
|
||||
if not self._settings.firecracker_binary.exists():
|
||||
raise UvmError("Firecracker is not installed. Run: sudo uvm install")
|
||||
with self._state_store.operation_lock():
|
||||
manifest = load_manifest(self._settings.integrity_manifest_path)
|
||||
firecracker_checksum = self._settings.firecracker_binary_sha256
|
||||
if manifest.verified or self._settings.allow_unverified_downloads:
|
||||
firecracker_checksum = firecracker_checksum or manifest.checksums.get("firecracker")
|
||||
verify_file(
|
||||
self._settings.firecracker_binary,
|
||||
firecracker_checksum,
|
||||
"Firecracker binary",
|
||||
"UVM_FIRECRACKER_BINARY_SHA256",
|
||||
allow_unverified=self._settings.allow_unverified_downloads,
|
||||
)
|
||||
return self._create_locked(spec)
|
||||
|
||||
def _create_locked(self, spec: VmSpec) -> VmRecord:
|
||||
assets = self._images.installed_assets()
|
||||
vm = self._reserve_vm(spec)
|
||||
|
||||
tap_created = False
|
||||
try:
|
||||
runtime_dir = self._settings.vm_dir(vm.id)
|
||||
runtime_dir.mkdir(parents=True, exist_ok=False)
|
||||
runtime_dir.chmod(0o700)
|
||||
disk = self._images.create_vm_disk(assets.rootfs, Path(vm.disk))
|
||||
self._images.provision_credentials(disk, vm.username, vm.password)
|
||||
self._network.ensure_bridge()
|
||||
self._check_boot()
|
||||
assets = self._images.installed_assets()
|
||||
vm = self._reserve_vm(replace(spec, ports=ports))
|
||||
try:
|
||||
self._network.create_tap(vm.tap)
|
||||
except TapCreationError as error:
|
||||
tap_created = error.tap_created
|
||||
raise
|
||||
else:
|
||||
tap_created = True
|
||||
runtime = self._settings.vm_dir(vm.id)
|
||||
runtime.mkdir(mode=0o700, exist_ok=False)
|
||||
vm.runtime_owned = True
|
||||
self._save(vm)
|
||||
disk = self._images.create_vm_disk(assets.rootfs, Path(vm.disk))
|
||||
self._images.provision_credentials(disk, vm.username, vm.password)
|
||||
vm.disk_ready = True
|
||||
self._save(vm)
|
||||
self._boot(vm, assets.kernel)
|
||||
return vm
|
||||
except BaseException as error:
|
||||
cleanup_error = None
|
||||
try:
|
||||
if vm.operation != "preparing":
|
||||
self._cleanup_runtime(vm)
|
||||
self._remove_runtime(vm)
|
||||
self._remove_record(vm.id)
|
||||
except BaseException as cleanup:
|
||||
cleanup_error = cleanup
|
||||
message = f"failed to create {vm.id}: {error}"
|
||||
if cleanup_error is not None:
|
||||
message += f"; cleanup incomplete: {cleanup_error}"
|
||||
self._record_failure(vm, message)
|
||||
if isinstance(error, (KeyboardInterrupt, SystemExit)):
|
||||
raise
|
||||
raise UvmError(message) from error
|
||||
|
||||
config = build_config(self._settings, vm, assets.kernel, disk)
|
||||
write_config(Path(vm.config), config)
|
||||
def _check_boot(self) -> None:
|
||||
check_kvm()
|
||||
manifest = load_manifest(self._settings.integrity_manifest_path)
|
||||
checksum = self._settings.firecracker_binary_sha256
|
||||
if manifest.verified or self._settings.allow_unverified_downloads:
|
||||
checksum = checksum or manifest.checksums.get("firecracker")
|
||||
verify_file(self._settings.firecracker_binary, checksum, "Firecracker binary",
|
||||
"UVM_FIRECRACKER_BINARY_SHA256",
|
||||
allow_unverified=self._settings.allow_unverified_downloads)
|
||||
self._network.firewall.check_available()
|
||||
|
||||
process_info = self._process.start(vm)
|
||||
vm.pid = process_info.pid
|
||||
vm.process_start_time = process_info.start_time
|
||||
vm.updated_at = self._now()
|
||||
self._replace_vm(vm)
|
||||
def _boot(self, vm: VmRecord, kernel: Path) -> None:
|
||||
vm.status, vm.operation = "starting", "network"
|
||||
vm.pid = vm.process_start_time = None
|
||||
vm.spawn_pending = False
|
||||
self._save(vm)
|
||||
self._network.ensure_bridge()
|
||||
self._network.prepare_tap(vm)
|
||||
self._save(vm)
|
||||
self._network.firewall.apply(vm)
|
||||
self._network.activate_tap(vm)
|
||||
config = build_config(self._settings, vm, kernel, Path(vm.disk))
|
||||
write_config(Path(vm.config), config)
|
||||
|
||||
client = self._client_factory(Path(vm.socket), self._settings.api_timeout_s)
|
||||
client.configure_and_start(config)
|
||||
# Persist the crash window before spawning, but give start() the already
|
||||
# verified process-free record. A restored pending spawn is never guessed gone.
|
||||
vm.operation = "boot"
|
||||
vm.boot_id = self._process.host_boot_id()
|
||||
self._save(replace(vm, operation="spawning", spawn_pending=True))
|
||||
|
||||
def spawned(info: ProcessInfo) -> None:
|
||||
vm.pid, vm.process_start_time, vm.boot_id = info.pid, info.start_time, info.boot_id
|
||||
vm.spawn_pending = False
|
||||
vm.operation = "configuring"
|
||||
self._save(vm)
|
||||
|
||||
self._process.start(vm, on_spawn=spawned)
|
||||
client = self._client_factory(Path(vm.socket), self._settings.api_timeout_s)
|
||||
client.configure_and_start(config)
|
||||
if self._network.firewall.status(vm) != "enforced":
|
||||
raise UvmError("private-port enforcement could not be confirmed after boot")
|
||||
vm.status, vm.operation, vm.last_error = "running", None, None
|
||||
self._save(vm)
|
||||
|
||||
def start(self, identifier: str) -> VmRecord:
|
||||
require_root()
|
||||
with self._state_store.operation_lock():
|
||||
return self._start_locked(self._load_for_operation(identifier))
|
||||
|
||||
def _start_locked(self, vm: VmRecord) -> VmRecord:
|
||||
self._not_deleting(vm)
|
||||
if vm.ports is None:
|
||||
raise ConflictError("legacy VM has no port policy; stop it, use ports set, then start")
|
||||
observation = self._process.observe(vm)
|
||||
if observation == "unknown":
|
||||
raise ConflictError(f"cannot start {vm.id}: process identity is unknown; inspect before retrying")
|
||||
if observation == "alive":
|
||||
if vm.status != "running" or vm.operation not in {None, "policy"}:
|
||||
raise ConflictError(f"{vm.id} has an interrupted boot/stop; stop it before starting")
|
||||
return self._sync_ports_locked(vm)
|
||||
if not vm.disk_ready:
|
||||
raise ConflictError(f"{vm.id} has no completely provisioned private disk")
|
||||
disk = Path(vm.disk)
|
||||
if not disk.is_file() or disk.is_symlink() or disk.stat().st_nlink != 1:
|
||||
raise UvmError(f"private VM disk is missing or unsafe: {disk}")
|
||||
if (vm.gateway != str(self._settings.gateway)
|
||||
or vm.network not in {None, str(self._settings.network)}
|
||||
or vm.bridge not in {None, self._settings.bridge}):
|
||||
raise ConflictError("configured network differs from the VM's retained network")
|
||||
self._check_boot()
|
||||
kernel = self._images.installed_kernel()
|
||||
try:
|
||||
self._cleanup_runtime(vm)
|
||||
vm.network, vm.bridge = str(self._settings.network), self._settings.bridge
|
||||
self._boot(vm, kernel)
|
||||
return vm
|
||||
except BaseException as error:
|
||||
cleanup_errors = self._rollback_create(vm, tap_created)
|
||||
message = f"failed to create {vm.id}: {error}"
|
||||
if cleanup_errors:
|
||||
message = f"{message}; cleanup failed: {'; '.join(cleanup_errors)}"
|
||||
vm.status = "failed"
|
||||
vm.last_error = message
|
||||
vm.updated_at = self._now()
|
||||
self._replace_vm_if_present(vm)
|
||||
else:
|
||||
self._remove_vm_if_present(vm.id)
|
||||
message = f"failed to start {vm.id}: {error}"
|
||||
try:
|
||||
self._cleanup_runtime(vm)
|
||||
except BaseException as cleanup:
|
||||
message += f"; cleanup incomplete: {cleanup}"
|
||||
self._record_failure(vm, message)
|
||||
if isinstance(error, (KeyboardInterrupt, SystemExit)):
|
||||
raise
|
||||
if isinstance(error, UvmError):
|
||||
raise error
|
||||
raise UvmError(message) from error
|
||||
|
||||
vm.status = "running"
|
||||
vm.last_error = None
|
||||
vm.updated_at = self._now()
|
||||
self._replace_vm(vm)
|
||||
return vm
|
||||
def stop(self, identifier: str, *, timeout: float | None = None) -> VmRecord:
|
||||
require_root()
|
||||
with self._state_store.operation_lock():
|
||||
return self._stop_locked(self._load_for_operation(identifier), timeout=timeout)
|
||||
|
||||
def _stop_locked(self, vm: VmRecord, *, timeout: float | None = None) -> VmRecord:
|
||||
deleting = vm.operation == "delete" or vm.status == "terminating"
|
||||
vm.status, vm.operation = "stopping", "delete" if deleting else "stop"
|
||||
self._save(vm)
|
||||
try:
|
||||
self._cleanup_runtime(vm, timeout=timeout)
|
||||
vm.status = "stopped"
|
||||
vm.operation = "delete" if deleting else None
|
||||
vm.last_error = None
|
||||
self._save(vm)
|
||||
return vm
|
||||
except BaseException as error:
|
||||
self._record_failure(vm, f"failed to stop {vm.id}: {error}")
|
||||
if isinstance(error, (KeyboardInterrupt, SystemExit)):
|
||||
raise
|
||||
raise UvmError(vm.last_error) from error
|
||||
|
||||
def restart(self, identifier: str, *, timeout: float | None = None) -> VmRecord:
|
||||
require_root()
|
||||
with self._state_store.operation_lock():
|
||||
vm = self._load_for_operation(identifier)
|
||||
self._not_deleting(vm)
|
||||
self._stop_locked(vm, timeout=timeout)
|
||||
return self._start_locked(vm)
|
||||
|
||||
def destroy(self, identifier: str, *, timeout: float | None = None) -> VmRecord:
|
||||
require_root()
|
||||
with self._state_store.operation_lock():
|
||||
return self._delete_locked(self._load_for_operation(identifier), timeout=timeout)
|
||||
|
||||
def _delete_locked(self, vm: VmRecord, *, timeout: float | None = None) -> VmRecord:
|
||||
vm.status, vm.operation = "terminating", "delete"
|
||||
self._save(vm)
|
||||
try:
|
||||
self._cleanup_runtime(vm, timeout=timeout)
|
||||
self._remove_runtime(vm)
|
||||
self._remove_record(vm.id)
|
||||
return vm
|
||||
except BaseException as error:
|
||||
self._record_failure(vm, f"failed to delete {vm.id}: {error}")
|
||||
if isinstance(error, (KeyboardInterrupt, SystemExit)):
|
||||
raise
|
||||
raise UvmError(vm.last_error) from error
|
||||
|
||||
def _cleanup_runtime(self, vm: VmRecord, *, timeout: float | None = None) -> None:
|
||||
containment_errors = []
|
||||
try:
|
||||
if self._network.has_tap(vm):
|
||||
try:
|
||||
self._network.firewall.apply(vm, quarantine=True)
|
||||
finally:
|
||||
self._network.deactivate_tap(vm)
|
||||
except UvmError as error:
|
||||
containment_errors.append(str(error))
|
||||
try:
|
||||
self._process.terminate(vm, timeout=timeout)
|
||||
except UvmError as error:
|
||||
detail = f"; containment: {'; '.join(containment_errors)}" if containment_errors else ""
|
||||
raise UvmError(f"process exit unconfirmed: {error}{detail}") from error
|
||||
vm.pid = vm.process_start_time = None
|
||||
vm.spawn_pending = False
|
||||
if vm.operation == "spawning":
|
||||
vm.operation = "boot"
|
||||
self._network.delete_vm_tap(vm)
|
||||
self._network.firewall.cleanup_connections(vm)
|
||||
self._network.firewall.remove(vm)
|
||||
|
||||
def set_ports(self, identifier: str, ports: tuple[PortRule, ...], *, mode: str = "set") -> VmRecord:
|
||||
require_root()
|
||||
ports = self._validated_ports(ports)
|
||||
if mode not in {"open", "close", "set"}:
|
||||
raise ValidationError(f"invalid port operation: {mode}")
|
||||
with self._state_store.operation_lock():
|
||||
vm = self._load_for_operation(identifier)
|
||||
self._not_deleting(vm)
|
||||
observation = self._process.observe(vm)
|
||||
if observation == "unknown":
|
||||
raise ConflictError("port changes require a known process identity")
|
||||
if vm.ports is None and (mode != "set" or observation != "gone" or vm.status != "stopped"):
|
||||
raise ConflictError("legacy VM policy must be explicitly set while stopped")
|
||||
if vm.operation not in {None, "policy"}:
|
||||
raise ConflictError(f"finish the pending {vm.operation} operation before editing ports")
|
||||
current = {port.port: port for port in vm.ports or ()}
|
||||
if mode == "set":
|
||||
current = {port.port: port for port in ports}
|
||||
elif mode == "open":
|
||||
for port in ports:
|
||||
current.setdefault(port.port, port)
|
||||
else:
|
||||
for port in ports:
|
||||
current.pop(port.port, None)
|
||||
vm.ports = tuple(current[number] for number in sorted(current))
|
||||
vm.operation = "policy"
|
||||
self._save(vm)
|
||||
return self._sync_ports_locked(vm)
|
||||
|
||||
def sync_ports(self, identifier: str) -> VmRecord:
|
||||
require_root()
|
||||
with self._state_store.operation_lock():
|
||||
return self._sync_ports_locked(self._load_for_operation(identifier))
|
||||
|
||||
def _sync_ports_locked(self, vm: VmRecord) -> VmRecord:
|
||||
self._not_deleting(vm)
|
||||
if vm.ports is None:
|
||||
raise ConflictError("legacy VM has no managed policy; stop it and use ports set")
|
||||
if vm.operation not in {None, "policy"}:
|
||||
raise ConflictError(f"finish the pending {vm.operation} operation before syncing ports")
|
||||
observation = self._process.observe(vm)
|
||||
if observation == "unknown":
|
||||
raise ConflictError("cannot enforce policy for a VM with unknown process identity")
|
||||
vm.operation = "policy"
|
||||
self._save(vm)
|
||||
try:
|
||||
if observation == "alive":
|
||||
if not self._network.has_tap(vm):
|
||||
raise UvmError("running VM has no owned network attachment")
|
||||
self._network.firewall.apply(vm)
|
||||
self._network.activate_tap(vm)
|
||||
vm.status = "running"
|
||||
else:
|
||||
if vm.status != "stopped" or vm.tap_owned:
|
||||
self._cleanup_runtime(vm)
|
||||
vm.status = "stopped"
|
||||
vm.operation, vm.last_error = None, None
|
||||
self._save(vm)
|
||||
return vm
|
||||
except BaseException as error:
|
||||
# Never restore an older, more permissive policy after a failed close.
|
||||
containment_errors = []
|
||||
for contain in (
|
||||
lambda: self._network.deactivate_tap(vm),
|
||||
lambda: self._network.firewall.apply(vm, quarantine=True),
|
||||
):
|
||||
try:
|
||||
contain()
|
||||
except UvmError as containment_error:
|
||||
containment_errors.append(str(containment_error))
|
||||
detail = f"; containment errors: {'; '.join(containment_errors)}" if containment_errors else ""
|
||||
self._record_failure(vm, f"policy pending for {vm.id}: {error}{detail}")
|
||||
if isinstance(error, (KeyboardInterrupt, SystemExit)):
|
||||
raise
|
||||
raise UvmError(vm.last_error) from error
|
||||
|
||||
def resolve_targets(self, identifiers: Sequence[str], *, all_vms: bool = False) -> list[VmRecord]:
|
||||
if all_vms and identifiers:
|
||||
raise ValidationError("--all cannot be combined with explicit VM targets")
|
||||
state = self._state_store.load()
|
||||
if all_vms:
|
||||
return list(state.vms.values())
|
||||
if not identifiers:
|
||||
raise ValidationError("specify a VM or --all")
|
||||
resolved = [self._state_store.resolve(state, identifier) for identifier in identifiers]
|
||||
return list({vm.id: vm for vm in resolved}.values())
|
||||
|
||||
def run_many(self, action: str, vm_ids: Sequence[str], *, timeout: float | None = None) -> list[OperationResult]:
|
||||
require_root()
|
||||
if action not in {"start", "stop", "restart", "delete", "ports-sync"}:
|
||||
raise ValidationError(f"invalid lifecycle action: {action}")
|
||||
results = []
|
||||
with self._state_store.operation_lock():
|
||||
for index, vm_id in enumerate(vm_ids):
|
||||
try:
|
||||
vm = self._load_for_operation(vm_id, exact_id=True)
|
||||
if action == "start":
|
||||
self._start_locked(vm)
|
||||
elif action == "stop":
|
||||
self._stop_locked(vm, timeout=timeout)
|
||||
elif action == "delete":
|
||||
self._delete_locked(vm, timeout=timeout)
|
||||
elif action == "ports-sync":
|
||||
self._sync_ports_locked(vm)
|
||||
else:
|
||||
self._not_deleting(vm)
|
||||
self._stop_locked(vm, timeout=timeout)
|
||||
self._start_locked(vm)
|
||||
results.append(OperationResult(vm_id, action, "succeeded"))
|
||||
except NotFoundError:
|
||||
results.append(OperationResult(vm_id, action, "skipped", "already absent"))
|
||||
except KeyboardInterrupt:
|
||||
results.append(OperationResult(vm_id, action, "interrupted", "interrupted"))
|
||||
results.extend(OperationResult(remaining, action, "skipped", "not attempted after interruption")
|
||||
for remaining in vm_ids[index + 1:])
|
||||
break
|
||||
except (UvmError, OSError) as error:
|
||||
results.append(OperationResult(vm_id, action, "failed", str(error)))
|
||||
return results
|
||||
|
||||
def list_vms(self) -> list[ListedVm]:
|
||||
state = self._state_store.load()
|
||||
listed: list[ListedVm] = []
|
||||
for vm in state.vms.values():
|
||||
observed_status = vm.status
|
||||
if vm.status in {"starting", "running", "stopping", "terminating"}:
|
||||
if not self._process.is_alive(vm):
|
||||
observed_status = "dead"
|
||||
listed.append(ListedVm(vm=vm, observed_status=observed_status))
|
||||
return listed
|
||||
return [self._listed(vm) for vm in self._state_store.load().vms.values()]
|
||||
|
||||
def inspect(self, identifier: str) -> ListedVm:
|
||||
return self._listed(self._state_store.find_by_id_or_ip(identifier))
|
||||
|
||||
def _listed(self, vm: VmRecord) -> ListedVm:
|
||||
observation = self._process.observe(vm)
|
||||
status = vm.status
|
||||
if observation == "unknown":
|
||||
status = "unknown"
|
||||
elif observation == "alive" and status == "stopped":
|
||||
status = "running"
|
||||
elif observation == "gone" and status == "running":
|
||||
status = "dead"
|
||||
policy = ("inactive" if observation == "gone" and vm.status == "stopped" and vm.operation is None
|
||||
else self._network.firewall.status(vm))
|
||||
if vm.ports is None and policy == "inactive":
|
||||
policy = "unmanaged"
|
||||
return ListedVm(vm, status, policy)
|
||||
|
||||
def find_for_ssh(self, identifier: str) -> VmRecord:
|
||||
return self._state_store.find_by_id_or_ip(identifier)
|
||||
|
||||
def stop(self, vm_id: str) -> VmRecord:
|
||||
"""Stop one VM while retaining its private writable disk for future use."""
|
||||
def log_path(self, identifier: str) -> Path:
|
||||
vm = self._state_store.find_by_id_or_ip(identifier)
|
||||
self._validate_paths(vm)
|
||||
return Path(vm.log)
|
||||
|
||||
require_root()
|
||||
with self._state_store.operation_lock():
|
||||
return self._stop_locked(vm_id)
|
||||
|
||||
def _stop_locked(self, vm_id: str) -> VmRecord:
|
||||
vm = self._begin_transition(vm_id, "stopping")
|
||||
try:
|
||||
self._process.terminate(vm)
|
||||
self._network.delete_tap(vm.tap)
|
||||
except BaseException as error:
|
||||
vm.status = "failed"
|
||||
vm.last_error = f"failed to stop VM: {error}"
|
||||
vm.updated_at = self._now()
|
||||
self._replace_vm_if_present(vm)
|
||||
if isinstance(error, (KeyboardInterrupt, SystemExit)):
|
||||
raise
|
||||
if isinstance(error, UvmError):
|
||||
raise error
|
||||
raise UvmError(vm.last_error) from error
|
||||
|
||||
vm.status = "stopped"
|
||||
vm.pid = None
|
||||
vm.process_start_time = None
|
||||
vm.last_error = None
|
||||
vm.updated_at = self._now()
|
||||
self._replace_vm(vm)
|
||||
return vm
|
||||
|
||||
def destroy(self, vm_id: str) -> VmRecord:
|
||||
"""Stop a VM, remove its private resources, then release its allocation."""
|
||||
|
||||
require_root()
|
||||
with self._state_store.operation_lock():
|
||||
return self._destroy_locked(vm_id)
|
||||
|
||||
def _destroy_locked(self, vm_id: str) -> VmRecord:
|
||||
vm = self._begin_transition(vm_id, "terminating")
|
||||
try:
|
||||
self._process.terminate(vm)
|
||||
self._network.delete_tap(vm.tap)
|
||||
def diagnostics(self) -> list[dict]:
|
||||
checks = [("root", require_root), ("kvm", check_kvm),
|
||||
("guest assets", self._images.installed_assets),
|
||||
("private firewall", self._network.firewall.check_available)]
|
||||
results = []
|
||||
for name, check in checks:
|
||||
try:
|
||||
shutil.rmtree(self._settings.vm_dir(vm.id))
|
||||
except FileNotFoundError:
|
||||
pass
|
||||
except BaseException as error:
|
||||
vm.status = "failed"
|
||||
vm.last_error = f"failed to destroy VM: {error}"
|
||||
vm.updated_at = self._now()
|
||||
self._replace_vm_if_present(vm)
|
||||
if isinstance(error, (KeyboardInterrupt, SystemExit)):
|
||||
raise
|
||||
if isinstance(error, UvmError):
|
||||
raise error
|
||||
raise UvmError(vm.last_error) from error
|
||||
check()
|
||||
results.append({"name": name, "ok": True, "message": "available"})
|
||||
except (UvmError, OSError) as error:
|
||||
results.append({"name": name, "ok": False, "message": str(error)})
|
||||
for command in ("ip", "iptables", "nft", "conntrack", "debugfs", "openssl", "ssh-keygen", "ssh"):
|
||||
path = shutil.which(command)
|
||||
results.append({"name": command, "ok": path is not None, "message": path or "not installed"})
|
||||
return results
|
||||
|
||||
self._remove_vm_if_present(vm.id)
|
||||
def _load_for_operation(self, identifier: str, *, exact_id: bool = False) -> VmRecord:
|
||||
vm = (self._state_store.get(identifier) if exact_id
|
||||
else self._state_store.find_by_id_or_ip(identifier))
|
||||
self._validate_paths(vm)
|
||||
if self._process.adopt_legacy_identity(vm):
|
||||
self._save(vm)
|
||||
return vm
|
||||
|
||||
def _validate_paths(self, vm: VmRecord) -> None:
|
||||
if re.fullmatch(r"[A-Za-z0-9][A-Za-z0-9_.-]{0,80}", vm.id) is None:
|
||||
raise ConflictError("invalid persisted VM ID; refusing resource access")
|
||||
directory = self._settings.vm_dir(vm.id).absolute()
|
||||
if directory.is_symlink():
|
||||
raise ConflictError("VM directory must not be a symbolic link")
|
||||
for field, filename in (("socket", "firecracker.sock"), ("config", "config.json"),
|
||||
("log", "firecracker.log"), ("disk", "rootfs.ext4")):
|
||||
path = getattr(vm, field)
|
||||
if field == "disk" and not path:
|
||||
continue # Legacy records may have used a shared template, never delete it.
|
||||
if Path(path).absolute() != directory / filename or Path(path).is_symlink():
|
||||
raise ConflictError(f"unsafe persisted VM {field} path")
|
||||
|
||||
@staticmethod
|
||||
def _not_deleting(vm: VmRecord) -> None:
|
||||
if vm.operation == "delete" or vm.status == "terminating":
|
||||
raise ConflictError(f"deletion is pending for {vm.id}; retry delete instead")
|
||||
|
||||
@staticmethod
|
||||
def _validated_ports(ports: tuple[PortRule, ...]) -> tuple[PortRule, ...]:
|
||||
if not isinstance(ports, tuple) or any(not isinstance(port, PortRule) for port in ports):
|
||||
raise ValidationError("ports must be a tuple of private TCP port rules")
|
||||
return parse_ports([f"{port.port}:{port.protocol}" for port in ports])
|
||||
|
||||
def _reserve_vm(self, spec: VmSpec) -> VmRecord:
|
||||
with self._state_store.transaction() as state:
|
||||
if spec.name is not None and any(spec.name in {vm.name, vm.id} for vm in state.vms.values()):
|
||||
raise ConflictError(f"VM name already exists: {spec.name}")
|
||||
vm_id = new_vm_id()
|
||||
used_taps = {existing_vm.tap for existing_vm in state.vms.values()}
|
||||
while vm_id in state.vms or self._network.tap_name(vm_id) in used_taps:
|
||||
used_taps = {vm.tap for vm in state.vms.values()}
|
||||
while vm_id in state.vms or any(vm.name == vm_id for vm in state.vms.values()) or self._network.tap_name(vm_id) in used_taps:
|
||||
vm_id = new_vm_id()
|
||||
guest_ip = self._network.allocate_ip(state.vms.values(), spec.guest_ip)
|
||||
mac = self._network.mac_for(state.next_mac_index)
|
||||
state.next_mac_index += 1
|
||||
now = self._now()
|
||||
runtime_dir = self._settings.vm_dir(vm_id)
|
||||
directory = self._settings.vm_dir(vm_id)
|
||||
vm = VmRecord(
|
||||
id=vm_id,
|
||||
cpu=spec.cpu,
|
||||
ram_mib=spec.ram_mib,
|
||||
guest_ip=str(guest_ip),
|
||||
gateway=str(self._settings.gateway),
|
||||
tap=self._network.tap_name(vm_id),
|
||||
mac=mac,
|
||||
socket=str(runtime_dir / "firecracker.sock"),
|
||||
config=str(runtime_dir / "config.json"),
|
||||
log=str(runtime_dir / "firecracker.log"),
|
||||
disk=str(runtime_dir / "rootfs.ext4"),
|
||||
username=spec.username,
|
||||
password=spec.password,
|
||||
status="starting",
|
||||
created_at=now,
|
||||
updated_at=now,
|
||||
id=vm_id, name=spec.name, cpu=spec.cpu, ram_mib=spec.ram_mib,
|
||||
guest_ip=str(guest_ip), gateway=str(self._settings.gateway),
|
||||
network=str(self._settings.network), bridge=self._settings.bridge,
|
||||
tap=self._network.tap_name(vm_id), mac=mac,
|
||||
socket=str(directory / "firecracker.sock"), config=str(directory / "config.json"),
|
||||
log=str(directory / "firecracker.log"), disk=str(directory / "rootfs.ext4"),
|
||||
username=spec.username, password=spec.password, ports=spec.ports,
|
||||
status="starting", operation="preparing", created_at=int(self._clock()), updated_at=int(self._clock()),
|
||||
)
|
||||
state.vms[vm.id] = vm
|
||||
return vm
|
||||
|
||||
def _begin_transition(self, vm_id: str, target_status: str) -> VmRecord:
|
||||
with self._state_store.transaction() as state:
|
||||
try:
|
||||
vm = state.vms[vm_id]
|
||||
except KeyError as error:
|
||||
raise UvmError(f"VM not found: {vm_id}") from error
|
||||
if vm.status in {"stopping", "terminating"}:
|
||||
raise UvmError(f"VM operation is already in progress: {vm_id}")
|
||||
vm.status = target_status
|
||||
vm.updated_at = self._now()
|
||||
state.vms[vm_id] = vm
|
||||
return vm
|
||||
def _remove_runtime(self, vm: VmRecord) -> None:
|
||||
self._validate_paths(vm)
|
||||
directory = self._settings.vm_dir(vm.id)
|
||||
if directory.exists():
|
||||
if not vm.runtime_owned:
|
||||
raise ConflictError(f"refusing to remove unowned VM directory {directory}")
|
||||
if self._process.observe(vm) != "gone":
|
||||
raise ConflictError("refusing to remove disk without confirmed process exit")
|
||||
shutil.rmtree(directory)
|
||||
vm.runtime_owned = False
|
||||
|
||||
def _replace_vm(self, vm: VmRecord) -> None:
|
||||
def _save(self, vm: VmRecord) -> None:
|
||||
vm.updated_at = int(self._clock())
|
||||
with self._state_store.transaction() as state:
|
||||
if vm.id not in state.vms:
|
||||
raise UvmError(f"VM not found: {vm.id}")
|
||||
raise NotFoundError(f"VM not found: {vm.id}")
|
||||
state.vms[vm.id] = vm
|
||||
|
||||
def _replace_vm_if_present(self, vm: VmRecord) -> None:
|
||||
with self._state_store.transaction() as state:
|
||||
if vm.id in state.vms:
|
||||
state.vms[vm.id] = vm
|
||||
def _record_failure(self, vm: VmRecord, message: str) -> None:
|
||||
vm.status, vm.last_error = "failed", message
|
||||
self._save(vm)
|
||||
|
||||
def _remove_vm_if_present(self, vm_id: str) -> None:
|
||||
def _remove_record(self, vm_id: str) -> None:
|
||||
with self._state_store.transaction() as state:
|
||||
state.vms.pop(vm_id, None)
|
||||
|
||||
def _rollback_create(self, vm: VmRecord, tap_created: bool) -> list[str]:
|
||||
errors: list[str] = []
|
||||
if vm.pid is not None:
|
||||
try:
|
||||
self._process.terminate(vm)
|
||||
except Exception as error:
|
||||
errors.append(f"process: {error}")
|
||||
if tap_created:
|
||||
try:
|
||||
self._network.delete_tap(vm.tap)
|
||||
except Exception as error:
|
||||
errors.append(f"network: {error}")
|
||||
try:
|
||||
shutil.rmtree(self._settings.vm_dir(vm.id))
|
||||
except FileNotFoundError:
|
||||
pass
|
||||
except OSError as error:
|
||||
errors.append(f"files: {error}")
|
||||
return errors
|
||||
|
||||
def _now(self) -> int:
|
||||
return int(self._clock())
|
||||
|
||||
Reference in New Issue
Block a user