kstyagi
d8a25209b9
- Introduced a new method `installed_kernel` in `ImageStore` to handle kernel verification and checksum validation. - Updated `Installer` to include additional packages: `nftables` and `conntrack`. - Refined `LifecycleService` to improve error handling and state management during VM creation, starting, stopping, and deletion. - Enhanced port management with new methods for setting and syncing ports, including validation for port rules. - Improved error handling in `StateStore` to raise specific `NotFoundError` and `ConflictError` exceptions. - Updated network management to ensure proper TAP device handling and firewall integration. - Enhanced API responses in `vms.py` to include policy status for better client-side handling. - Added validation for VM names and ports in `validation.py` to enforce stricter rules and prevent conflicts.
508 lines
24 KiB
Python
508 lines
24 KiB
Python
"""Serialized lifecycle and private-port policy changes with durable intent."""
|
|
|
|
from __future__ import annotations
|
|
|
|
import re
|
|
import shutil
|
|
import time
|
|
from collections.abc import Callable, Sequence
|
|
from dataclasses import dataclass, replace
|
|
from pathlib import Path
|
|
|
|
from .config import Settings
|
|
from .domain import PortRule, VmRecord, VmSpec, new_vm_id
|
|
from .errors import ConflictError, NotFoundError, UvmError, ValidationError
|
|
from .firecracker.api import FirecrackerClient
|
|
from .firecracker.config import build_config, write_config
|
|
from .firecracker.process import FirecrackerProcessManager, ProcessInfo
|
|
from .images import ImageStore
|
|
from .integrity import load_manifest, verify_file
|
|
from .network import NetworkManager
|
|
from .state import StateStore
|
|
from .system import check_kvm, ensure_data_directories, require_root
|
|
from .validation import parse_cpu, parse_name, parse_password, parse_ports, parse_username
|
|
|
|
|
|
@dataclass(frozen=True, slots=True)
|
|
class ListedVm:
|
|
vm: VmRecord
|
|
observed_status: str
|
|
policy_status: str = "unknown"
|
|
|
|
|
|
@dataclass(frozen=True, slots=True)
|
|
class OperationResult:
|
|
id: str
|
|
action: str
|
|
outcome: str
|
|
error: str | None = None
|
|
|
|
|
|
class LifecycleService:
|
|
"""All mutating entry points use operation lock -> short state transaction."""
|
|
|
|
def __init__(
|
|
self, settings: Settings, state_store: StateStore, images: ImageStore,
|
|
network: NetworkManager, process: FirecrackerProcessManager, *,
|
|
client_factory: Callable[[Path, float], FirecrackerClient] = FirecrackerClient,
|
|
clock: Callable[[], float] = time.time,
|
|
) -> None:
|
|
self._settings = settings
|
|
self._state_store = state_store
|
|
self._images = images
|
|
self._network = network
|
|
self._process = process
|
|
self._client_factory = client_factory
|
|
self._clock = clock
|
|
|
|
def create(self, spec: VmSpec) -> VmRecord:
|
|
require_root()
|
|
parse_cpu(str(spec.cpu))
|
|
if type(spec.ram_mib) is not int or spec.ram_mib < 128:
|
|
raise ValidationError("RAM must be an integer of at least 128 MiB")
|
|
parse_username(spec.username)
|
|
parse_password(spec.password)
|
|
if spec.name is not None:
|
|
parse_name(spec.name)
|
|
ports = self._validated_ports(spec.ports)
|
|
ensure_data_directories(self._settings)
|
|
with self._state_store.operation_lock():
|
|
self._check_boot()
|
|
assets = self._images.installed_assets()
|
|
vm = self._reserve_vm(replace(spec, ports=ports))
|
|
try:
|
|
runtime = self._settings.vm_dir(vm.id)
|
|
runtime.mkdir(mode=0o700, exist_ok=False)
|
|
vm.runtime_owned = True
|
|
self._save(vm)
|
|
disk = self._images.create_vm_disk(assets.rootfs, Path(vm.disk))
|
|
self._images.provision_credentials(disk, vm.username, vm.password)
|
|
vm.disk_ready = True
|
|
self._save(vm)
|
|
self._boot(vm, assets.kernel)
|
|
return vm
|
|
except BaseException as error:
|
|
cleanup_error = None
|
|
try:
|
|
if vm.operation != "preparing":
|
|
self._cleanup_runtime(vm)
|
|
self._remove_runtime(vm)
|
|
self._remove_record(vm.id)
|
|
except BaseException as cleanup:
|
|
cleanup_error = cleanup
|
|
message = f"failed to create {vm.id}: {error}"
|
|
if cleanup_error is not None:
|
|
message += f"; cleanup incomplete: {cleanup_error}"
|
|
self._record_failure(vm, message)
|
|
if isinstance(error, (KeyboardInterrupt, SystemExit)):
|
|
raise
|
|
raise UvmError(message) from error
|
|
|
|
def _check_boot(self) -> None:
|
|
check_kvm()
|
|
manifest = load_manifest(self._settings.integrity_manifest_path)
|
|
checksum = self._settings.firecracker_binary_sha256
|
|
if manifest.verified or self._settings.allow_unverified_downloads:
|
|
checksum = checksum or manifest.checksums.get("firecracker")
|
|
verify_file(self._settings.firecracker_binary, checksum, "Firecracker binary",
|
|
"UVM_FIRECRACKER_BINARY_SHA256",
|
|
allow_unverified=self._settings.allow_unverified_downloads)
|
|
self._network.firewall.check_available()
|
|
|
|
def _boot(self, vm: VmRecord, kernel: Path) -> None:
|
|
vm.status, vm.operation = "starting", "network"
|
|
vm.pid = vm.process_start_time = None
|
|
vm.spawn_pending = False
|
|
self._save(vm)
|
|
self._network.ensure_bridge()
|
|
self._network.prepare_tap(vm)
|
|
self._save(vm)
|
|
self._network.firewall.apply(vm)
|
|
self._network.activate_tap(vm)
|
|
config = build_config(self._settings, vm, kernel, Path(vm.disk))
|
|
write_config(Path(vm.config), config)
|
|
|
|
# Persist the crash window before spawning, but give start() the already
|
|
# verified process-free record. A restored pending spawn is never guessed gone.
|
|
vm.operation = "boot"
|
|
vm.boot_id = self._process.host_boot_id()
|
|
self._save(replace(vm, operation="spawning", spawn_pending=True))
|
|
|
|
def spawned(info: ProcessInfo) -> None:
|
|
vm.pid, vm.process_start_time, vm.boot_id = info.pid, info.start_time, info.boot_id
|
|
vm.spawn_pending = False
|
|
vm.operation = "configuring"
|
|
self._save(vm)
|
|
|
|
self._process.start(vm, on_spawn=spawned)
|
|
client = self._client_factory(Path(vm.socket), self._settings.api_timeout_s)
|
|
client.configure_and_start(config)
|
|
if self._network.firewall.status(vm) != "enforced":
|
|
raise UvmError("private-port enforcement could not be confirmed after boot")
|
|
vm.status, vm.operation, vm.last_error = "running", None, None
|
|
self._save(vm)
|
|
|
|
def start(self, identifier: str) -> VmRecord:
|
|
require_root()
|
|
with self._state_store.operation_lock():
|
|
return self._start_locked(self._load_for_operation(identifier))
|
|
|
|
def _start_locked(self, vm: VmRecord) -> VmRecord:
|
|
self._not_deleting(vm)
|
|
if vm.ports is None:
|
|
raise ConflictError("legacy VM has no port policy; stop it, use ports set, then start")
|
|
observation = self._process.observe(vm)
|
|
if observation == "unknown":
|
|
raise ConflictError(f"cannot start {vm.id}: process identity is unknown; inspect before retrying")
|
|
if observation == "alive":
|
|
if vm.status != "running" or vm.operation not in {None, "policy"}:
|
|
raise ConflictError(f"{vm.id} has an interrupted boot/stop; stop it before starting")
|
|
return self._sync_ports_locked(vm)
|
|
if not vm.disk_ready:
|
|
raise ConflictError(f"{vm.id} has no completely provisioned private disk")
|
|
disk = Path(vm.disk)
|
|
if not disk.is_file() or disk.is_symlink() or disk.stat().st_nlink != 1:
|
|
raise UvmError(f"private VM disk is missing or unsafe: {disk}")
|
|
if (vm.gateway != str(self._settings.gateway)
|
|
or vm.network not in {None, str(self._settings.network)}
|
|
or vm.bridge not in {None, self._settings.bridge}):
|
|
raise ConflictError("configured network differs from the VM's retained network")
|
|
self._check_boot()
|
|
kernel = self._images.installed_kernel()
|
|
try:
|
|
self._cleanup_runtime(vm)
|
|
vm.network, vm.bridge = str(self._settings.network), self._settings.bridge
|
|
self._boot(vm, kernel)
|
|
return vm
|
|
except BaseException as error:
|
|
message = f"failed to start {vm.id}: {error}"
|
|
try:
|
|
self._cleanup_runtime(vm)
|
|
except BaseException as cleanup:
|
|
message += f"; cleanup incomplete: {cleanup}"
|
|
self._record_failure(vm, message)
|
|
if isinstance(error, (KeyboardInterrupt, SystemExit)):
|
|
raise
|
|
raise UvmError(message) from error
|
|
|
|
def stop(self, identifier: str, *, timeout: float | None = None) -> VmRecord:
|
|
require_root()
|
|
with self._state_store.operation_lock():
|
|
return self._stop_locked(self._load_for_operation(identifier), timeout=timeout)
|
|
|
|
def _stop_locked(self, vm: VmRecord, *, timeout: float | None = None) -> VmRecord:
|
|
deleting = vm.operation == "delete" or vm.status == "terminating"
|
|
vm.status, vm.operation = "stopping", "delete" if deleting else "stop"
|
|
self._save(vm)
|
|
try:
|
|
self._cleanup_runtime(vm, timeout=timeout)
|
|
vm.status = "stopped"
|
|
vm.operation = "delete" if deleting else None
|
|
vm.last_error = None
|
|
self._save(vm)
|
|
return vm
|
|
except BaseException as error:
|
|
self._record_failure(vm, f"failed to stop {vm.id}: {error}")
|
|
if isinstance(error, (KeyboardInterrupt, SystemExit)):
|
|
raise
|
|
raise UvmError(vm.last_error) from error
|
|
|
|
def restart(self, identifier: str, *, timeout: float | None = None) -> VmRecord:
|
|
require_root()
|
|
with self._state_store.operation_lock():
|
|
vm = self._load_for_operation(identifier)
|
|
self._not_deleting(vm)
|
|
self._stop_locked(vm, timeout=timeout)
|
|
return self._start_locked(vm)
|
|
|
|
def destroy(self, identifier: str, *, timeout: float | None = None) -> VmRecord:
|
|
require_root()
|
|
with self._state_store.operation_lock():
|
|
return self._delete_locked(self._load_for_operation(identifier), timeout=timeout)
|
|
|
|
def _delete_locked(self, vm: VmRecord, *, timeout: float | None = None) -> VmRecord:
|
|
vm.status, vm.operation = "terminating", "delete"
|
|
self._save(vm)
|
|
try:
|
|
self._cleanup_runtime(vm, timeout=timeout)
|
|
self._remove_runtime(vm)
|
|
self._remove_record(vm.id)
|
|
return vm
|
|
except BaseException as error:
|
|
self._record_failure(vm, f"failed to delete {vm.id}: {error}")
|
|
if isinstance(error, (KeyboardInterrupt, SystemExit)):
|
|
raise
|
|
raise UvmError(vm.last_error) from error
|
|
|
|
def _cleanup_runtime(self, vm: VmRecord, *, timeout: float | None = None) -> None:
|
|
containment_errors = []
|
|
try:
|
|
if self._network.has_tap(vm):
|
|
try:
|
|
self._network.firewall.apply(vm, quarantine=True)
|
|
finally:
|
|
self._network.deactivate_tap(vm)
|
|
except UvmError as error:
|
|
containment_errors.append(str(error))
|
|
try:
|
|
self._process.terminate(vm, timeout=timeout)
|
|
except UvmError as error:
|
|
detail = f"; containment: {'; '.join(containment_errors)}" if containment_errors else ""
|
|
raise UvmError(f"process exit unconfirmed: {error}{detail}") from error
|
|
vm.pid = vm.process_start_time = None
|
|
vm.spawn_pending = False
|
|
if vm.operation == "spawning":
|
|
vm.operation = "boot"
|
|
self._network.delete_vm_tap(vm)
|
|
self._network.firewall.cleanup_connections(vm)
|
|
self._network.firewall.remove(vm)
|
|
|
|
def set_ports(self, identifier: str, ports: tuple[PortRule, ...], *, mode: str = "set") -> VmRecord:
|
|
require_root()
|
|
ports = self._validated_ports(ports)
|
|
if mode not in {"open", "close", "set"}:
|
|
raise ValidationError(f"invalid port operation: {mode}")
|
|
with self._state_store.operation_lock():
|
|
vm = self._load_for_operation(identifier)
|
|
self._not_deleting(vm)
|
|
observation = self._process.observe(vm)
|
|
if observation == "unknown":
|
|
raise ConflictError("port changes require a known process identity")
|
|
if vm.ports is None and (mode != "set" or observation != "gone" or vm.status != "stopped"):
|
|
raise ConflictError("legacy VM policy must be explicitly set while stopped")
|
|
if vm.operation not in {None, "policy"}:
|
|
raise ConflictError(f"finish the pending {vm.operation} operation before editing ports")
|
|
current = {port.port: port for port in vm.ports or ()}
|
|
if mode == "set":
|
|
current = {port.port: port for port in ports}
|
|
elif mode == "open":
|
|
for port in ports:
|
|
current.setdefault(port.port, port)
|
|
else:
|
|
for port in ports:
|
|
current.pop(port.port, None)
|
|
vm.ports = tuple(current[number] for number in sorted(current))
|
|
vm.operation = "policy"
|
|
self._save(vm)
|
|
return self._sync_ports_locked(vm)
|
|
|
|
def sync_ports(self, identifier: str) -> VmRecord:
|
|
require_root()
|
|
with self._state_store.operation_lock():
|
|
return self._sync_ports_locked(self._load_for_operation(identifier))
|
|
|
|
def _sync_ports_locked(self, vm: VmRecord) -> VmRecord:
|
|
self._not_deleting(vm)
|
|
if vm.ports is None:
|
|
raise ConflictError("legacy VM has no managed policy; stop it and use ports set")
|
|
if vm.operation not in {None, "policy"}:
|
|
raise ConflictError(f"finish the pending {vm.operation} operation before syncing ports")
|
|
observation = self._process.observe(vm)
|
|
if observation == "unknown":
|
|
raise ConflictError("cannot enforce policy for a VM with unknown process identity")
|
|
vm.operation = "policy"
|
|
self._save(vm)
|
|
try:
|
|
if observation == "alive":
|
|
if not self._network.has_tap(vm):
|
|
raise UvmError("running VM has no owned network attachment")
|
|
self._network.firewall.apply(vm)
|
|
self._network.activate_tap(vm)
|
|
vm.status = "running"
|
|
else:
|
|
if vm.status != "stopped" or vm.tap_owned:
|
|
self._cleanup_runtime(vm)
|
|
vm.status = "stopped"
|
|
vm.operation, vm.last_error = None, None
|
|
self._save(vm)
|
|
return vm
|
|
except BaseException as error:
|
|
# Never restore an older, more permissive policy after a failed close.
|
|
containment_errors = []
|
|
for contain in (
|
|
lambda: self._network.deactivate_tap(vm),
|
|
lambda: self._network.firewall.apply(vm, quarantine=True),
|
|
):
|
|
try:
|
|
contain()
|
|
except UvmError as containment_error:
|
|
containment_errors.append(str(containment_error))
|
|
detail = f"; containment errors: {'; '.join(containment_errors)}" if containment_errors else ""
|
|
self._record_failure(vm, f"policy pending for {vm.id}: {error}{detail}")
|
|
if isinstance(error, (KeyboardInterrupt, SystemExit)):
|
|
raise
|
|
raise UvmError(vm.last_error) from error
|
|
|
|
def resolve_targets(self, identifiers: Sequence[str], *, all_vms: bool = False) -> list[VmRecord]:
|
|
if all_vms and identifiers:
|
|
raise ValidationError("--all cannot be combined with explicit VM targets")
|
|
state = self._state_store.load()
|
|
if all_vms:
|
|
return list(state.vms.values())
|
|
if not identifiers:
|
|
raise ValidationError("specify a VM or --all")
|
|
resolved = [self._state_store.resolve(state, identifier) for identifier in identifiers]
|
|
return list({vm.id: vm for vm in resolved}.values())
|
|
|
|
def run_many(self, action: str, vm_ids: Sequence[str], *, timeout: float | None = None) -> list[OperationResult]:
|
|
require_root()
|
|
if action not in {"start", "stop", "restart", "delete", "ports-sync"}:
|
|
raise ValidationError(f"invalid lifecycle action: {action}")
|
|
results = []
|
|
with self._state_store.operation_lock():
|
|
for index, vm_id in enumerate(vm_ids):
|
|
try:
|
|
vm = self._load_for_operation(vm_id, exact_id=True)
|
|
if action == "start":
|
|
self._start_locked(vm)
|
|
elif action == "stop":
|
|
self._stop_locked(vm, timeout=timeout)
|
|
elif action == "delete":
|
|
self._delete_locked(vm, timeout=timeout)
|
|
elif action == "ports-sync":
|
|
self._sync_ports_locked(vm)
|
|
else:
|
|
self._not_deleting(vm)
|
|
self._stop_locked(vm, timeout=timeout)
|
|
self._start_locked(vm)
|
|
results.append(OperationResult(vm_id, action, "succeeded"))
|
|
except NotFoundError:
|
|
results.append(OperationResult(vm_id, action, "skipped", "already absent"))
|
|
except KeyboardInterrupt:
|
|
results.append(OperationResult(vm_id, action, "interrupted", "interrupted"))
|
|
results.extend(OperationResult(remaining, action, "skipped", "not attempted after interruption")
|
|
for remaining in vm_ids[index + 1:])
|
|
break
|
|
except (UvmError, OSError) as error:
|
|
results.append(OperationResult(vm_id, action, "failed", str(error)))
|
|
return results
|
|
|
|
def list_vms(self) -> list[ListedVm]:
|
|
return [self._listed(vm) for vm in self._state_store.load().vms.values()]
|
|
|
|
def inspect(self, identifier: str) -> ListedVm:
|
|
return self._listed(self._state_store.find_by_id_or_ip(identifier))
|
|
|
|
def _listed(self, vm: VmRecord) -> ListedVm:
|
|
observation = self._process.observe(vm)
|
|
status = vm.status
|
|
if observation == "unknown":
|
|
status = "unknown"
|
|
elif observation == "alive" and status == "stopped":
|
|
status = "running"
|
|
elif observation == "gone" and status == "running":
|
|
status = "dead"
|
|
policy = ("inactive" if observation == "gone" and vm.status == "stopped" and vm.operation is None
|
|
else self._network.firewall.status(vm))
|
|
if vm.ports is None and policy == "inactive":
|
|
policy = "unmanaged"
|
|
return ListedVm(vm, status, policy)
|
|
|
|
def find_for_ssh(self, identifier: str) -> VmRecord:
|
|
return self._state_store.find_by_id_or_ip(identifier)
|
|
|
|
def log_path(self, identifier: str) -> Path:
|
|
vm = self._state_store.find_by_id_or_ip(identifier)
|
|
self._validate_paths(vm)
|
|
return Path(vm.log)
|
|
|
|
def diagnostics(self) -> list[dict]:
|
|
checks = [("root", require_root), ("kvm", check_kvm),
|
|
("guest assets", self._images.installed_assets),
|
|
("private firewall", self._network.firewall.check_available)]
|
|
results = []
|
|
for name, check in checks:
|
|
try:
|
|
check()
|
|
results.append({"name": name, "ok": True, "message": "available"})
|
|
except (UvmError, OSError) as error:
|
|
results.append({"name": name, "ok": False, "message": str(error)})
|
|
for command in ("ip", "iptables", "nft", "conntrack", "debugfs", "openssl", "ssh-keygen", "ssh"):
|
|
path = shutil.which(command)
|
|
results.append({"name": command, "ok": path is not None, "message": path or "not installed"})
|
|
return results
|
|
|
|
def _load_for_operation(self, identifier: str, *, exact_id: bool = False) -> VmRecord:
|
|
vm = (self._state_store.get(identifier) if exact_id
|
|
else self._state_store.find_by_id_or_ip(identifier))
|
|
self._validate_paths(vm)
|
|
if self._process.adopt_legacy_identity(vm):
|
|
self._save(vm)
|
|
return vm
|
|
|
|
def _validate_paths(self, vm: VmRecord) -> None:
|
|
if re.fullmatch(r"[A-Za-z0-9][A-Za-z0-9_.-]{0,80}", vm.id) is None:
|
|
raise ConflictError("invalid persisted VM ID; refusing resource access")
|
|
directory = self._settings.vm_dir(vm.id).absolute()
|
|
if directory.is_symlink():
|
|
raise ConflictError("VM directory must not be a symbolic link")
|
|
for field, filename in (("socket", "firecracker.sock"), ("config", "config.json"),
|
|
("log", "firecracker.log"), ("disk", "rootfs.ext4")):
|
|
path = getattr(vm, field)
|
|
if field == "disk" and not path:
|
|
continue # Legacy records may have used a shared template, never delete it.
|
|
if Path(path).absolute() != directory / filename or Path(path).is_symlink():
|
|
raise ConflictError(f"unsafe persisted VM {field} path")
|
|
|
|
@staticmethod
|
|
def _not_deleting(vm: VmRecord) -> None:
|
|
if vm.operation == "delete" or vm.status == "terminating":
|
|
raise ConflictError(f"deletion is pending for {vm.id}; retry delete instead")
|
|
|
|
@staticmethod
|
|
def _validated_ports(ports: tuple[PortRule, ...]) -> tuple[PortRule, ...]:
|
|
if not isinstance(ports, tuple) or any(not isinstance(port, PortRule) for port in ports):
|
|
raise ValidationError("ports must be a tuple of private TCP port rules")
|
|
return parse_ports([f"{port.port}:{port.protocol}" for port in ports])
|
|
|
|
def _reserve_vm(self, spec: VmSpec) -> VmRecord:
|
|
with self._state_store.transaction() as state:
|
|
if spec.name is not None and any(spec.name in {vm.name, vm.id} for vm in state.vms.values()):
|
|
raise ConflictError(f"VM name already exists: {spec.name}")
|
|
vm_id = new_vm_id()
|
|
used_taps = {vm.tap for vm in state.vms.values()}
|
|
while vm_id in state.vms or any(vm.name == vm_id for vm in state.vms.values()) or self._network.tap_name(vm_id) in used_taps:
|
|
vm_id = new_vm_id()
|
|
guest_ip = self._network.allocate_ip(state.vms.values(), spec.guest_ip)
|
|
mac = self._network.mac_for(state.next_mac_index)
|
|
state.next_mac_index += 1
|
|
directory = self._settings.vm_dir(vm_id)
|
|
vm = VmRecord(
|
|
id=vm_id, name=spec.name, cpu=spec.cpu, ram_mib=spec.ram_mib,
|
|
guest_ip=str(guest_ip), gateway=str(self._settings.gateway),
|
|
network=str(self._settings.network), bridge=self._settings.bridge,
|
|
tap=self._network.tap_name(vm_id), mac=mac,
|
|
socket=str(directory / "firecracker.sock"), config=str(directory / "config.json"),
|
|
log=str(directory / "firecracker.log"), disk=str(directory / "rootfs.ext4"),
|
|
username=spec.username, password=spec.password, ports=spec.ports,
|
|
status="starting", operation="preparing", created_at=int(self._clock()), updated_at=int(self._clock()),
|
|
)
|
|
state.vms[vm.id] = vm
|
|
return vm
|
|
|
|
def _remove_runtime(self, vm: VmRecord) -> None:
|
|
self._validate_paths(vm)
|
|
directory = self._settings.vm_dir(vm.id)
|
|
if directory.exists():
|
|
if not vm.runtime_owned:
|
|
raise ConflictError(f"refusing to remove unowned VM directory {directory}")
|
|
if self._process.observe(vm) != "gone":
|
|
raise ConflictError("refusing to remove disk without confirmed process exit")
|
|
shutil.rmtree(directory)
|
|
vm.runtime_owned = False
|
|
|
|
def _save(self, vm: VmRecord) -> None:
|
|
vm.updated_at = int(self._clock())
|
|
with self._state_store.transaction() as state:
|
|
if vm.id not in state.vms:
|
|
raise NotFoundError(f"VM not found: {vm.id}")
|
|
state.vms[vm.id] = vm
|
|
|
|
def _record_failure(self, vm: VmRecord, message: str) -> None:
|
|
vm.status, vm.last_error = "failed", message
|
|
self._save(vm)
|
|
|
|
def _remove_record(self, vm_id: str) -> None:
|
|
with self._state_store.transaction() as state:
|
|
state.vms.pop(vm_id, None)
|