Refactor VM lifecycle management and enhance validation
- Introduced a new method `installed_kernel` in `ImageStore` to handle kernel verification and checksum validation. - Updated `Installer` to include additional packages: `nftables` and `conntrack`. - Refined `LifecycleService` to improve error handling and state management during VM creation, starting, stopping, and deletion. - Enhanced port management with new methods for setting and syncing ports, including validation for port rules. - Improved error handling in `StateStore` to raise specific `NotFoundError` and `ConflictError` exceptions. - Updated network management to ensure proper TAP device handling and firewall integration. - Enhanced API responses in `vms.py` to include policy status for better client-side handling. - Added validation for VM names and ports in `validation.py` to enforce stricter rules and prevent conflicts.
This commit is contained in:
+368
-96
@@ -2,8 +2,13 @@
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import ctypes
|
||||
import errno
|
||||
import math
|
||||
import os
|
||||
import select
|
||||
import signal
|
||||
import stat
|
||||
import subprocess
|
||||
import time
|
||||
from collections.abc import Callable
|
||||
@@ -26,6 +31,50 @@ class _PopenLike(Protocol):
|
||||
class ProcessInfo:
|
||||
pid: int
|
||||
start_time: str
|
||||
boot_id: str | None = None
|
||||
|
||||
|
||||
def _libc_pidfd(name: str, *arguments: int | None) -> int:
|
||||
# Some Linux Python builds omit pidfd bindings even when libc supports them.
|
||||
try:
|
||||
function = getattr(ctypes.CDLL(None, use_errno=True), name)
|
||||
except (AttributeError, OSError) as error:
|
||||
raise OSError(errno.ENOSYS, f"{name} is unavailable") from error
|
||||
function.argtypes = (
|
||||
[ctypes.c_int, ctypes.c_uint]
|
||||
if name == "pidfd_open"
|
||||
else [ctypes.c_int, ctypes.c_int, ctypes.c_void_p, ctypes.c_uint]
|
||||
)
|
||||
function.restype = ctypes.c_int
|
||||
result = function(*arguments)
|
||||
if result == -1:
|
||||
error_number = ctypes.get_errno()
|
||||
raise OSError(error_number, os.strerror(error_number))
|
||||
return result
|
||||
|
||||
|
||||
def _open_pidfd(pid: int) -> int:
|
||||
native = getattr(os, "pidfd_open", None)
|
||||
if native is not None:
|
||||
return native(pid, 0)
|
||||
return _libc_pidfd("pidfd_open", pid, 0)
|
||||
|
||||
|
||||
def _signal_pidfd(pidfd: int, signal_number: int) -> None:
|
||||
native = getattr(signal, "pidfd_send_signal", None)
|
||||
if native is not None:
|
||||
native(pidfd, signal_number)
|
||||
else:
|
||||
_libc_pidfd("pidfd_send_signal", pidfd, signal_number, None, 0)
|
||||
|
||||
|
||||
def _pidfd_exited(pidfd: int) -> bool:
|
||||
poller = select.poll()
|
||||
poller.register(pidfd, select.POLLIN)
|
||||
events = poller.poll(0)
|
||||
if any(event & (select.POLLERR | select.POLLNVAL) for _, event in events):
|
||||
raise OSError(errno.EIO, "could not observe the Firecracker pidfd")
|
||||
return any(event & (select.POLLIN | select.POLLHUP) for _, event in events)
|
||||
|
||||
|
||||
class FirecrackerProcessManager:
|
||||
@@ -43,103 +92,334 @@ class FirecrackerProcessManager:
|
||||
self._popen = popen
|
||||
self._sleep = sleep
|
||||
self._monotonic = monotonic
|
||||
self._children: dict[int, _PopenLike] = {}
|
||||
|
||||
def start(self, vm: VmRecord) -> ProcessInfo:
|
||||
socket_path = Path(vm.socket)
|
||||
if socket_path.exists():
|
||||
def start(
|
||||
self,
|
||||
vm: VmRecord,
|
||||
*,
|
||||
on_spawn: Callable[[ProcessInfo], None] | None = None,
|
||||
) -> ProcessInfo:
|
||||
"""Publish identity before readiness; never replace an ambiguous process."""
|
||||
|
||||
observation = self.observe(vm)
|
||||
if observation != "gone":
|
||||
raise FirecrackerError(
|
||||
f"refusing to start Firecracker: existing process is {observation}"
|
||||
)
|
||||
boot_id = self._boot_id()
|
||||
if boot_id is None:
|
||||
raise FirecrackerError("could not establish the current host boot identity")
|
||||
|
||||
# Check support before creating a child that we might be unable to stop safely.
|
||||
try:
|
||||
pidfd = _open_pidfd(os.getpid())
|
||||
try:
|
||||
socket_path.unlink()
|
||||
except OSError as error:
|
||||
raise FirecrackerError(
|
||||
f"could not remove stale Firecracker socket {socket_path}: {error}"
|
||||
) from error
|
||||
_signal_pidfd(pidfd, 0)
|
||||
finally:
|
||||
os.close(pidfd)
|
||||
except (OSError, NotImplementedError) as error:
|
||||
raise FirecrackerError(
|
||||
f"safe pidfd support is required to start Firecracker: {error}"
|
||||
) from error
|
||||
|
||||
socket_path = Path(vm.socket)
|
||||
self._remove_socket(socket_path)
|
||||
|
||||
log_path = Path(vm.log)
|
||||
log_path.parent.mkdir(parents=True, exist_ok=True)
|
||||
process: _PopenLike | None = None
|
||||
try:
|
||||
log_path.parent.mkdir(parents=True, exist_ok=True)
|
||||
with log_path.open("ab", buffering=0) as log_file:
|
||||
process = self._popen(
|
||||
[str(self._settings.firecracker_binary), "--api-sock", str(socket_path)],
|
||||
stdout=log_file,
|
||||
stderr=log_file,
|
||||
start_new_session=True,
|
||||
previous_identity = (
|
||||
vm.pid, vm.process_start_time, vm.boot_id, vm.operation, vm.spawn_pending,
|
||||
)
|
||||
vm.pid = None
|
||||
vm.process_start_time = None
|
||||
vm.boot_id = boot_id
|
||||
vm.operation = "spawning"
|
||||
vm.spawn_pending = True
|
||||
try:
|
||||
process = self._popen(
|
||||
[str(self._settings.firecracker_binary), "--api-sock", str(socket_path)],
|
||||
stdout=log_file,
|
||||
stderr=log_file,
|
||||
start_new_session=True,
|
||||
)
|
||||
except OSError as error:
|
||||
# Popen cleans up an exec/fork failure, but an interrupt is uncertain.
|
||||
if not isinstance(error, InterruptedError):
|
||||
(
|
||||
vm.pid, vm.process_start_time, vm.boot_id, vm.operation, vm.spawn_pending,
|
||||
) = previous_identity
|
||||
raise
|
||||
vm.pid = process.pid
|
||||
self._children[process.pid] = process
|
||||
vm.process_start_time = self.process_start_time(process.pid)
|
||||
if vm.process_start_time is None:
|
||||
raise FirecrackerError(
|
||||
f"could not establish a safe process identity for Firecracker process {process.pid}"
|
||||
)
|
||||
info = ProcessInfo(process.pid, vm.process_start_time, boot_id)
|
||||
if on_spawn is not None:
|
||||
on_spawn(info)
|
||||
else:
|
||||
vm.spawn_pending = False
|
||||
self._wait_for_socket(process, socket_path, log_path)
|
||||
start_time = self.process_start_time(process.pid)
|
||||
if start_time is None:
|
||||
raise FirecrackerError(
|
||||
f"could not establish a safe process identity for Firecracker process {process.pid}"
|
||||
)
|
||||
return ProcessInfo(pid=process.pid, start_time=start_time)
|
||||
except OSError as error:
|
||||
return info
|
||||
except BaseException as error:
|
||||
if process is not None:
|
||||
self._cleanup_started_process(process, socket_path)
|
||||
raise FirecrackerError(f"could not start Firecracker: {error}") from error
|
||||
except BaseException:
|
||||
if process is not None:
|
||||
self._cleanup_started_process(process, socket_path)
|
||||
try:
|
||||
self.terminate(vm)
|
||||
except BaseException as cleanup_error:
|
||||
raise FirecrackerError(
|
||||
f"Firecracker startup failed ({type(error).__name__}: {error}); "
|
||||
f"cleanup incomplete for PID {vm.pid}: {cleanup_error}"
|
||||
) from error
|
||||
if isinstance(error, OSError):
|
||||
raise FirecrackerError(f"could not start Firecracker: {error}") from error
|
||||
raise
|
||||
|
||||
def is_alive(self, vm: VmRecord) -> bool:
|
||||
if vm.pid is None or vm.pid <= 0 or vm.process_start_time is None:
|
||||
def adopt_legacy_identity(self, vm: VmRecord) -> bool:
|
||||
"""Validate a v1 process; the caller must lock and persist the adopted boot ID."""
|
||||
|
||||
pid, start_time = vm.pid, vm.process_start_time
|
||||
if (
|
||||
vm.boot_id is not None
|
||||
or type(pid) is not int
|
||||
or not 0 < pid < 2**31
|
||||
or not isinstance(start_time, str)
|
||||
or not start_time.isascii()
|
||||
or not start_time.isdigit()
|
||||
):
|
||||
return False
|
||||
|
||||
try:
|
||||
os.kill(vm.pid, 0)
|
||||
except ProcessLookupError:
|
||||
pidfd = _open_pidfd(pid)
|
||||
try:
|
||||
process_stat = self._process_stat(pid)
|
||||
if (
|
||||
process_stat is None
|
||||
or process_stat[1] != start_time
|
||||
or process_stat[0] in {"Z", "X", "x"}
|
||||
):
|
||||
return False
|
||||
|
||||
binary_path = self._settings.firecracker_binary
|
||||
socket_path = self._settings.vm_dir(vm.id) / "firecracker.sock"
|
||||
if (
|
||||
not binary_path.is_absolute()
|
||||
or ".." in binary_path.parts
|
||||
or not vm.id
|
||||
or vm.id in {".", ".."}
|
||||
or Path(vm.id).name != vm.id
|
||||
or vm.socket != str(socket_path)
|
||||
):
|
||||
return False
|
||||
|
||||
# Sticky ancestors are safe only because every existing entry is root-owned.
|
||||
for directory in set(binary_path.parents) | set(socket_path.parents):
|
||||
metadata = directory.lstat()
|
||||
if (
|
||||
not stat.S_ISDIR(metadata.st_mode)
|
||||
or metadata.st_uid != 0
|
||||
or (directory == socket_path.parent and metadata.st_mode & 0o077)
|
||||
or (metadata.st_mode & 0o022 and not metadata.st_mode & stat.S_ISVTX)
|
||||
):
|
||||
return False
|
||||
binary_stat = binary_path.lstat()
|
||||
socket_stat = socket_path.lstat()
|
||||
if (
|
||||
not stat.S_ISREG(binary_stat.st_mode)
|
||||
or binary_stat.st_uid != 0
|
||||
or binary_stat.st_mode & 0o022
|
||||
or not stat.S_ISSOCK(socket_stat.st_mode)
|
||||
or socket_stat.st_uid != 0
|
||||
):
|
||||
return False
|
||||
|
||||
proc_path = Path(f"/proc/{pid}")
|
||||
executable = proc_path / "exe"
|
||||
if executable.readlink() != binary_path or not executable.samefile(binary_path):
|
||||
return False
|
||||
expected_cmdline = b"\0".join(
|
||||
(os.fsencode(binary_path), b"--api-sock", os.fsencode(socket_path), b"")
|
||||
)
|
||||
if (proc_path / "cmdline").read_bytes() != expected_cmdline:
|
||||
return False
|
||||
boot_id = self._boot_id()
|
||||
if boot_id is None or _pidfd_exited(pidfd):
|
||||
return False
|
||||
finally:
|
||||
os.close(pidfd)
|
||||
except (OSError, ValueError, NotImplementedError):
|
||||
return False
|
||||
except PermissionError:
|
||||
return True
|
||||
|
||||
return self.process_start_time(vm.pid) == vm.process_start_time
|
||||
if (vm.pid, vm.process_start_time, vm.boot_id) != (pid, start_time, None):
|
||||
return False
|
||||
vm.boot_id = boot_id
|
||||
return True
|
||||
|
||||
def terminate(self, vm: VmRecord) -> None:
|
||||
def observe(self, vm: VmRecord) -> str:
|
||||
"""Distinguish proven exit from missing or unreadable process identity."""
|
||||
|
||||
if vm.pid is not None and vm.pid <= 0:
|
||||
return "unknown"
|
||||
try:
|
||||
self._reap_child(vm.pid)
|
||||
except OSError:
|
||||
return "unknown"
|
||||
boot_id = self._boot_id()
|
||||
if vm.boot_id and boot_id and vm.boot_id != boot_id:
|
||||
return "gone"
|
||||
if vm.pid is None:
|
||||
self._remove_socket(Path(vm.socket))
|
||||
return
|
||||
if vm.pid <= 0:
|
||||
if vm.spawn_pending or vm.operation == "spawning" or vm.process_start_time is not None:
|
||||
return "unknown"
|
||||
try:
|
||||
Path(vm.socket).lstat()
|
||||
except FileNotFoundError:
|
||||
return "gone"
|
||||
except OSError:
|
||||
return "unknown"
|
||||
# An unowned socket may belong to a child whose identity was never saved.
|
||||
return "unknown"
|
||||
|
||||
stat = self._process_stat(vm.pid)
|
||||
if stat is None:
|
||||
return "unknown" if self._pid_exists(vm.pid) else "gone"
|
||||
state, start_time = stat
|
||||
has_start_time = (
|
||||
isinstance(vm.process_start_time, str)
|
||||
and vm.process_start_time.isascii()
|
||||
and vm.process_start_time.isdigit()
|
||||
)
|
||||
if has_start_time and vm.process_start_time != start_time:
|
||||
return "gone"
|
||||
if state in {"Z", "X", "x"}:
|
||||
# A zombie thread-group leader can still have live worker threads.
|
||||
try:
|
||||
pidfd = _open_pidfd(vm.pid)
|
||||
try:
|
||||
return "gone" if _pidfd_exited(pidfd) else "unknown"
|
||||
finally:
|
||||
os.close(pidfd)
|
||||
except ProcessLookupError:
|
||||
return "gone"
|
||||
except (OSError, NotImplementedError):
|
||||
return "unknown"
|
||||
if not has_start_time or not vm.boot_id or not boot_id:
|
||||
return "unknown"
|
||||
return "alive"
|
||||
|
||||
def is_alive(self, vm: VmRecord) -> bool:
|
||||
"""Conservative compatibility query: unknown processes may still be alive."""
|
||||
|
||||
return self.observe(vm) != "gone"
|
||||
|
||||
def terminate(self, vm: VmRecord, *, timeout: float | None = None) -> None:
|
||||
"""Wait for exit before unlinking; timeout controls the SIGTERM grace period."""
|
||||
|
||||
if vm.pid is not None and vm.pid <= 0:
|
||||
raise FirecrackerError(f"invalid persisted Firecracker PID: {vm.pid}")
|
||||
if vm.process_start_time is None:
|
||||
if not self._pid_exists(vm.pid):
|
||||
self._remove_socket(Path(vm.socket))
|
||||
return
|
||||
grace_timeout = self._settings.terminate_timeout_s if timeout is None else timeout
|
||||
if not math.isfinite(grace_timeout) or grace_timeout < 0:
|
||||
raise FirecrackerError("Firecracker termination timeout must be finite and non-negative")
|
||||
observation = self.observe(vm)
|
||||
if observation == "unknown":
|
||||
raise FirecrackerError(
|
||||
f"refusing to signal Firecracker PID {vm.pid} without a process identity token"
|
||||
f"refusing to stop Firecracker PID {vm.pid}: process identity is ambiguous"
|
||||
)
|
||||
if not self.is_alive(vm):
|
||||
if observation == "gone":
|
||||
self._remove_socket(Path(vm.socket))
|
||||
return
|
||||
|
||||
try:
|
||||
os.kill(vm.pid, signal.SIGTERM)
|
||||
pidfd = _open_pidfd(vm.pid)
|
||||
except ProcessLookupError:
|
||||
self._reap_child(vm.pid)
|
||||
self._remove_socket(Path(vm.socket))
|
||||
return
|
||||
except OSError as error:
|
||||
raise FirecrackerError(f"could not stop Firecracker process {vm.pid}: {error}") from error
|
||||
except (OSError, NotImplementedError) as error:
|
||||
raise FirecrackerError(
|
||||
f"could not safely open a pidfd for Firecracker process {vm.pid}: {error}"
|
||||
) from error
|
||||
|
||||
deadline = self._monotonic() + self._settings.terminate_timeout_s
|
||||
while self.is_alive(vm) and self._monotonic() < deadline:
|
||||
self._sleep(0.05)
|
||||
if self.is_alive(vm):
|
||||
self._terminate_process_id(vm.pid, force=True)
|
||||
self._remove_socket(Path(vm.socket))
|
||||
try:
|
||||
if not _pidfd_exited(pidfd):
|
||||
# Open the handle BEFORE rechecking identity; never signal by bare PID.
|
||||
observation = self.observe(vm)
|
||||
if observation == "unknown":
|
||||
raise FirecrackerError(
|
||||
f"refusing to stop Firecracker PID {vm.pid}: process identity became ambiguous"
|
||||
)
|
||||
if observation == "alive":
|
||||
try:
|
||||
_signal_pidfd(pidfd, signal.SIGTERM)
|
||||
except ProcessLookupError:
|
||||
pass
|
||||
if not self._wait_for_exit(pidfd, grace_timeout):
|
||||
try:
|
||||
_signal_pidfd(pidfd, signal.SIGKILL)
|
||||
except ProcessLookupError:
|
||||
pass
|
||||
# Even an immediate forced stop needs time to confirm kernel exit.
|
||||
if not self._wait_for_exit(pidfd, max(1.0, grace_timeout)):
|
||||
raise FirecrackerError(
|
||||
f"could not confirm exit of Firecracker process {vm.pid} after SIGKILL; "
|
||||
"process identity and socket retained"
|
||||
)
|
||||
self._reap_child(vm.pid)
|
||||
self._remove_socket(Path(vm.socket))
|
||||
except (OSError, NotImplementedError) as error:
|
||||
raise FirecrackerError(
|
||||
f"could not safely stop Firecracker process {vm.pid}: {error}"
|
||||
) from error
|
||||
finally:
|
||||
os.close(pidfd)
|
||||
|
||||
def process_start_time(self, pid: int) -> str | None:
|
||||
"""Read Linux proc start time so a reused PID is not mistaken for a VM."""
|
||||
|
||||
stat = self._process_stat(pid)
|
||||
return stat[1] if stat is not None else None
|
||||
|
||||
def host_boot_id(self) -> str | None:
|
||||
"""Return the current host boot identity, or None when it is unreadable."""
|
||||
|
||||
return self._boot_id()
|
||||
|
||||
@staticmethod
|
||||
def _boot_id() -> str | None:
|
||||
try:
|
||||
stat = Path(f"/proc/{pid}/stat").read_text(encoding="utf-8")
|
||||
except OSError:
|
||||
return Path("/proc/sys/kernel/random/boot_id").read_text(encoding="ascii").strip() or None
|
||||
except (OSError, UnicodeError):
|
||||
return None
|
||||
|
||||
@staticmethod
|
||||
def _process_stat(pid: int) -> tuple[str, str] | None:
|
||||
if pid <= 0:
|
||||
return None
|
||||
try:
|
||||
stat = Path(f"/proc/{pid}/stat").read_text(encoding="utf-8", errors="replace")
|
||||
except (OSError, UnicodeError):
|
||||
return None
|
||||
opening_parenthesis = stat.find("(")
|
||||
closing_parenthesis = stat.rfind(")")
|
||||
if closing_parenthesis < 0:
|
||||
if (
|
||||
opening_parenthesis < 0
|
||||
or closing_parenthesis < opening_parenthesis
|
||||
or stat[:opening_parenthesis].strip() != str(pid)
|
||||
):
|
||||
return None
|
||||
fields = stat[closing_parenthesis + 2 :].split()
|
||||
try:
|
||||
return fields[19]
|
||||
except IndexError:
|
||||
fields = stat[closing_parenthesis + 1 :].split()
|
||||
if (
|
||||
len(fields) < 20
|
||||
or fields[0] not in {"R", "S", "D", "Z", "T", "t", "X", "x", "K", "W", "P", "I"}
|
||||
or not fields[19].isascii()
|
||||
or not fields[19].isdigit()
|
||||
):
|
||||
return None
|
||||
return fields[0], fields[19]
|
||||
|
||||
def _wait_for_socket(
|
||||
self,
|
||||
@@ -148,13 +428,32 @@ class FirecrackerProcessManager:
|
||||
log_path: Path,
|
||||
) -> None:
|
||||
deadline = self._monotonic() + self._settings.api_socket_timeout_s
|
||||
while self._monotonic() < deadline:
|
||||
if socket_path.exists():
|
||||
return
|
||||
while True:
|
||||
if process.poll() is not None:
|
||||
raise FirecrackerError(f"Firecracker exited early; see {log_path}")
|
||||
self._sleep(0.05)
|
||||
raise FirecrackerError("timed out waiting for Firecracker API socket")
|
||||
if socket_path.is_socket():
|
||||
return
|
||||
remaining = deadline - self._monotonic()
|
||||
if remaining <= 0:
|
||||
raise FirecrackerError("timed out waiting for Firecracker API socket")
|
||||
self._sleep(min(0.05, remaining))
|
||||
|
||||
def _wait_for_exit(self, pidfd: int, timeout: float) -> bool:
|
||||
deadline = self._monotonic() + timeout
|
||||
while True:
|
||||
if _pidfd_exited(pidfd):
|
||||
return True
|
||||
remaining = deadline - self._monotonic()
|
||||
if remaining <= 0:
|
||||
return False
|
||||
self._sleep(min(0.05, remaining))
|
||||
|
||||
def _reap_child(self, pid: int | None) -> None:
|
||||
if pid is None:
|
||||
return
|
||||
process = self._children.get(pid)
|
||||
if process is not None and process.poll() is not None:
|
||||
self._children.pop(pid, None)
|
||||
|
||||
@staticmethod
|
||||
def _remove_socket(socket_path: Path) -> None:
|
||||
@@ -162,9 +461,10 @@ class FirecrackerProcessManager:
|
||||
socket_path.unlink()
|
||||
except FileNotFoundError:
|
||||
pass
|
||||
except OSError:
|
||||
# A stale socket is cleaned on the next launch; never hide a successful stop.
|
||||
pass
|
||||
except OSError as error:
|
||||
raise FirecrackerError(
|
||||
f"could not remove Firecracker socket {socket_path}: {error}"
|
||||
) from error
|
||||
|
||||
@staticmethod
|
||||
def _pid_exists(pid: int) -> bool:
|
||||
@@ -172,34 +472,6 @@ class FirecrackerProcessManager:
|
||||
os.kill(pid, 0)
|
||||
except ProcessLookupError:
|
||||
return False
|
||||
except PermissionError:
|
||||
except (OSError, OverflowError):
|
||||
return True
|
||||
return True
|
||||
|
||||
def _cleanup_started_process(self, process: _PopenLike, socket_path: Path) -> None:
|
||||
try:
|
||||
self._terminate_process_id(process.pid)
|
||||
except FirecrackerError:
|
||||
pass
|
||||
deadline = self._monotonic() + self._settings.terminate_timeout_s
|
||||
while process.poll() is None and self._monotonic() < deadline:
|
||||
self._sleep(0.05)
|
||||
if process.poll() is None:
|
||||
try:
|
||||
self._terminate_process_id(process.pid, force=True)
|
||||
except FirecrackerError:
|
||||
pass
|
||||
self._remove_socket(socket_path)
|
||||
|
||||
@staticmethod
|
||||
def _terminate_process_id(pid: int, *, force: bool = False) -> None:
|
||||
if pid <= 0:
|
||||
raise FirecrackerError(f"invalid Firecracker PID: {pid}")
|
||||
signal_to_send = signal.SIGKILL if force else signal.SIGTERM
|
||||
try:
|
||||
os.kill(pid, signal_to_send)
|
||||
except ProcessLookupError:
|
||||
pass
|
||||
except OSError as error:
|
||||
action = "force-stop" if force else "stop"
|
||||
raise FirecrackerError(f"could not {action} Firecracker process {pid}: {error}") from error
|
||||
|
||||
Reference in New Issue
Block a user