feat: Add guest execution agent and vsock support
- Introduced a standalone guest execution agent to facilitate command execution within the guest VM over Firecracker's vsock. - Enhanced FirecrackerClient to support vsock configuration, validating guest CID and UDS path. - Updated configuration to include vsock settings by default. - Refactored FirecrackerProcessManager to manage runtime sockets more effectively. - Implemented guest agent installation and provisioning in the image store. - Added API endpoints for executing commands in the VM and installing the guest agent. - Improved error handling for guest execution failures and agent availability.
This commit is contained in:
+38
-1
@@ -5,9 +5,12 @@ from __future__ import annotations
|
||||
from ipaddress import IPv4Address
|
||||
from typing import Literal
|
||||
|
||||
from pydantic import BaseModel, ConfigDict, Field, SecretStr
|
||||
from pydantic import BaseModel, ConfigDict, Field, SecretStr, model_validator
|
||||
|
||||
from .domain import VmRecord
|
||||
from .errors import ValidationError
|
||||
from .guest_agent import DEFAULT_TIMEOUT_SECONDS, MAX_COMMAND_BYTES, MAX_TIMEOUT_SECONDS
|
||||
from .guest_exec import validate_exec
|
||||
|
||||
|
||||
class HealthResponse(BaseModel):
|
||||
@@ -112,6 +115,40 @@ class VmLogResponse(BaseModel):
|
||||
truncated: bool
|
||||
|
||||
|
||||
class VmExecRequest(BaseModel):
|
||||
model_config = ConfigDict(extra="forbid")
|
||||
|
||||
command: str = Field(strict=True, min_length=1, max_length=MAX_COMMAND_BYTES, repr=False,
|
||||
description="Command or multiline script interpreted by /bin/sh inside the guest; at most 64 KiB UTF-8")
|
||||
sudo: bool = Field(default=False, strict=True, description="Run as guest root instead of the VM's configured login user")
|
||||
timeout_seconds: float = Field(default=DEFAULT_TIMEOUT_SECONDS, strict=True, gt=0,
|
||||
le=MAX_TIMEOUT_SECONDS, allow_inf_nan=False)
|
||||
|
||||
@model_validator(mode="before")
|
||||
@classmethod
|
||||
def validate_execution(cls, value: object) -> object:
|
||||
# Use the domain error envelope: echoing NaN or unpaired surrogates in
|
||||
# FastAPI's validation-error input can itself fail JSON serialization.
|
||||
if not isinstance(value, dict) or set(value) - {"command", "sudo", "timeout_seconds"}:
|
||||
raise ValidationError("execution requires an object with command, sudo, and timeout_seconds fields only")
|
||||
validate_exec(value.get("command"), value.get("sudo", False), value.get("timeout_seconds", DEFAULT_TIMEOUT_SECONDS))
|
||||
return value
|
||||
|
||||
|
||||
class VmExecResponse(BaseModel):
|
||||
exit_code: int | None
|
||||
stdout: str
|
||||
stderr: str
|
||||
timed_out: bool
|
||||
truncated: bool
|
||||
|
||||
|
||||
class VmAgentInstallResponse(BaseModel):
|
||||
id: str
|
||||
status: Literal["installed"] = "installed"
|
||||
restart_required: bool = True
|
||||
|
||||
|
||||
class DiagnosticResponse(BaseModel):
|
||||
name: str
|
||||
ok: bool
|
||||
|
||||
@@ -122,6 +122,9 @@ class Settings:
|
||||
def vm_dir(self, vm_id: str) -> Path:
|
||||
return self.vms_dir / vm_id
|
||||
|
||||
def guest_socket_path(self, vm_id: str) -> Path:
|
||||
return self.vm_dir(vm_id) / "vsock.sock"
|
||||
|
||||
@classmethod
|
||||
def from_environment(cls) -> "Settings":
|
||||
"""Load optional deployment overrides while retaining script defaults."""
|
||||
|
||||
@@ -39,6 +39,18 @@ class FirecrackerError(UvmError):
|
||||
"""Raised when Firecracker cannot be started or configured."""
|
||||
|
||||
|
||||
class GuestAgentUnavailableError(UvmError):
|
||||
"""The VM's execution agent could not be reached before submission."""
|
||||
|
||||
|
||||
class GuestExecutionError(UvmError):
|
||||
"""Guest execution failed; a lost response does not prove the command failed."""
|
||||
|
||||
def __init__(self, message: str, *, outcome_unknown: bool = False) -> None:
|
||||
super().__init__(message)
|
||||
self.outcome_unknown = outcome_unknown
|
||||
|
||||
|
||||
class TapCreationError(UvmError):
|
||||
"""Raised when TAP setup fails after creating a host interface."""
|
||||
|
||||
|
||||
@@ -36,6 +36,7 @@ class FirecrackerClient:
|
||||
drives = config["drives"]
|
||||
interfaces = config["network-interfaces"]
|
||||
entropy = config.get("entropy")
|
||||
vsock = config.get("vsock")
|
||||
if not isinstance(machine, Mapping) or not isinstance(boot, Mapping):
|
||||
raise FirecrackerError("invalid Firecracker configuration")
|
||||
if not isinstance(drives, list) or not drives:
|
||||
@@ -44,6 +45,10 @@ class FirecrackerClient:
|
||||
raise FirecrackerError("Firecracker configuration has no network interface")
|
||||
if not isinstance(entropy, Mapping):
|
||||
raise FirecrackerError("Firecracker configuration has no valid entropy device")
|
||||
if (not isinstance(vsock, Mapping) or type(vsock.get("guest_cid")) is not int
|
||||
or not 3 <= vsock["guest_cid"] < 2**32 - 1
|
||||
or not isinstance(vsock.get("uds_path"), str) or not vsock["uds_path"]):
|
||||
raise FirecrackerError("Firecracker configuration has no valid vsock device")
|
||||
|
||||
self._request(
|
||||
"PUT",
|
||||
@@ -57,6 +62,7 @@ class FirecrackerClient:
|
||||
self._request("PUT", "/boot-source", boot)
|
||||
self._request("PUT", "/drives/rootfs", drives[0])
|
||||
self._request("PUT", "/network-interfaces/eth0", interfaces[0])
|
||||
self._request("PUT", "/vsock", vsock)
|
||||
self._request("PUT", "/entropy", entropy)
|
||||
self._request("PUT", "/actions", {"action_type": "InstanceStart"})
|
||||
|
||||
|
||||
@@ -72,6 +72,9 @@ def build_config(
|
||||
],
|
||||
# Supply host-backed randomness before guest services such as sshd start.
|
||||
"entropy": {},
|
||||
# Each Firecracker instance has its own Unix-socket-backed vsock device;
|
||||
# CIDs need not be allocated across these independent devices.
|
||||
"vsock": {"guest_cid": 3, "uds_path": str(settings.guest_socket_path(vm.id))},
|
||||
}
|
||||
|
||||
|
||||
|
||||
+17
-11
@@ -124,7 +124,7 @@ class FirecrackerProcessManager:
|
||||
) from error
|
||||
|
||||
socket_path = Path(vm.socket)
|
||||
self._remove_socket(socket_path)
|
||||
self._remove_runtime_sockets(vm)
|
||||
|
||||
log_path = Path(vm.log)
|
||||
process: _PopenLike | None = None
|
||||
@@ -275,14 +275,16 @@ class FirecrackerProcessManager:
|
||||
if vm.pid is None:
|
||||
if vm.spawn_pending or vm.operation == "spawning" or vm.process_start_time is not None:
|
||||
return "unknown"
|
||||
try:
|
||||
Path(vm.socket).lstat()
|
||||
except FileNotFoundError:
|
||||
return "gone"
|
||||
except OSError:
|
||||
for socket_path in (Path(vm.socket), self._settings.guest_socket_path(vm.id)):
|
||||
try:
|
||||
socket_path.lstat()
|
||||
except FileNotFoundError:
|
||||
continue
|
||||
except OSError:
|
||||
return "unknown"
|
||||
# An unowned socket may belong to a child whose identity was never saved.
|
||||
return "unknown"
|
||||
# An unowned socket may belong to a child whose identity was never saved.
|
||||
return "unknown"
|
||||
return "gone"
|
||||
|
||||
stat = self._process_stat(vm.pid)
|
||||
if stat is None:
|
||||
@@ -330,14 +332,14 @@ class FirecrackerProcessManager:
|
||||
f"refusing to stop Firecracker PID {vm.pid}: process identity is ambiguous"
|
||||
)
|
||||
if observation == "gone":
|
||||
self._remove_socket(Path(vm.socket))
|
||||
self._remove_runtime_sockets(vm)
|
||||
return
|
||||
|
||||
try:
|
||||
pidfd = _open_pidfd(vm.pid)
|
||||
except ProcessLookupError:
|
||||
self._reap_child(vm.pid)
|
||||
self._remove_socket(Path(vm.socket))
|
||||
self._remove_runtime_sockets(vm)
|
||||
return
|
||||
except (OSError, NotImplementedError) as error:
|
||||
raise FirecrackerError(
|
||||
@@ -369,7 +371,7 @@ class FirecrackerProcessManager:
|
||||
"process identity and socket retained"
|
||||
)
|
||||
self._reap_child(vm.pid)
|
||||
self._remove_socket(Path(vm.socket))
|
||||
self._remove_runtime_sockets(vm)
|
||||
except (OSError, NotImplementedError) as error:
|
||||
raise FirecrackerError(
|
||||
f"could not safely stop Firecracker process {vm.pid}: {error}"
|
||||
@@ -455,6 +457,10 @@ class FirecrackerProcessManager:
|
||||
if process is not None and process.poll() is not None:
|
||||
self._children.pop(pid, None)
|
||||
|
||||
def _remove_runtime_sockets(self, vm: VmRecord) -> None:
|
||||
self._remove_socket(self._settings.guest_socket_path(vm.id))
|
||||
self._remove_socket(Path(vm.socket))
|
||||
|
||||
@staticmethod
|
||||
def _remove_socket(socket_path: Path) -> None:
|
||||
try:
|
||||
|
||||
@@ -0,0 +1,342 @@
|
||||
"""Standalone, stdlib-only guest execution agent (Python 3.10+).
|
||||
|
||||
This file is copied into guest images and must not import the host application.
|
||||
The host also imports its wire-format helpers; importing it never starts a server.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import math
|
||||
import os
|
||||
import pwd
|
||||
import re
|
||||
import selectors
|
||||
import signal
|
||||
import socket
|
||||
import stat
|
||||
import struct
|
||||
import subprocess
|
||||
import sys
|
||||
import threading
|
||||
import time
|
||||
import uuid
|
||||
from pathlib import Path
|
||||
|
||||
|
||||
PROTOCOL_VERSION = 1
|
||||
AGENT_PORT = 1070
|
||||
DEFAULT_TIMEOUT_SECONDS = 60.0
|
||||
MAX_TIMEOUT_SECONDS = 3600.0
|
||||
MAX_COMMAND_BYTES = 65536
|
||||
MAX_OUTPUT_BYTES = 1024 * 1024 # Combined stdout and stderr, before UTF-8 decoding.
|
||||
MAX_REQUEST_BYTES = 6 * MAX_COMMAND_BYTES + 4096
|
||||
MAX_RESPONSE_BYTES = 6 * MAX_OUTPUT_BYTES + 4096
|
||||
CONNECT_TIMEOUT_SECONDS = 5.0
|
||||
CLEANUP_SECONDS = 5.0
|
||||
MAX_CONCURRENT_COMMANDS = 8
|
||||
CONFIG_PATH = Path("/etc/uvm-agent.json")
|
||||
|
||||
|
||||
class ProtocolError(ValueError):
|
||||
"""An invalid or oversized wire message; never include command text."""
|
||||
|
||||
|
||||
class ExecutionFailure(Exception):
|
||||
def __init__(self, message: str, *, outcome_unknown: bool = False) -> None:
|
||||
super().__init__(message)
|
||||
self.outcome_unknown = outcome_unknown
|
||||
|
||||
|
||||
def validate_command(command: object, sudo: object, timeout_seconds: object) -> None:
|
||||
if not isinstance(command, str) or not command.strip() or "\0" in command:
|
||||
raise ProtocolError("command must be non-empty text without NUL characters")
|
||||
try:
|
||||
size = len(command.encode("utf-8"))
|
||||
except UnicodeError as error:
|
||||
raise ProtocolError("command must be valid UTF-8 text") from error
|
||||
if size > MAX_COMMAND_BYTES:
|
||||
raise ProtocolError(f"command exceeds {MAX_COMMAND_BYTES} UTF-8 bytes")
|
||||
if type(sudo) is not bool:
|
||||
raise ProtocolError("sudo must be a boolean")
|
||||
try:
|
||||
valid_timeout = (
|
||||
type(timeout_seconds) in (int, float)
|
||||
and math.isfinite(timeout_seconds)
|
||||
and 0 < timeout_seconds <= MAX_TIMEOUT_SECONDS
|
||||
)
|
||||
except OverflowError:
|
||||
valid_timeout = False
|
||||
if not valid_timeout:
|
||||
raise ProtocolError(f"timeout_seconds must be finite and greater than 0, up to {MAX_TIMEOUT_SECONDS:g}")
|
||||
|
||||
|
||||
def _remaining(deadline: float) -> float:
|
||||
remaining = deadline - time.monotonic()
|
||||
if remaining <= 0:
|
||||
raise TimeoutError("guest protocol deadline exceeded")
|
||||
return remaining
|
||||
|
||||
|
||||
def receive_exact(connection: socket.socket, size: int, deadline: float) -> bytes:
|
||||
parts = bytearray()
|
||||
while len(parts) < size:
|
||||
connection.settimeout(_remaining(deadline))
|
||||
chunk = connection.recv(size - len(parts))
|
||||
if not chunk:
|
||||
raise EOFError("guest connection closed")
|
||||
parts.extend(chunk)
|
||||
return bytes(parts)
|
||||
|
||||
|
||||
def receive_frame(connection: socket.socket, limit: int, deadline: float) -> dict:
|
||||
size, = struct.unpack("!I", receive_exact(connection, 4, deadline))
|
||||
if not 0 < size <= limit:
|
||||
raise ProtocolError("invalid guest message length")
|
||||
try:
|
||||
message = json.loads(receive_exact(connection, size, deadline))
|
||||
except (ValueError, UnicodeError, RecursionError) as error:
|
||||
raise ProtocolError("invalid guest JSON message") from error
|
||||
if not isinstance(message, dict):
|
||||
raise ProtocolError("guest message must be an object")
|
||||
return message
|
||||
|
||||
|
||||
def send_frame(connection: socket.socket, message: dict, limit: int, deadline: float) -> None:
|
||||
payload = json.dumps(message, ensure_ascii=True, allow_nan=False, separators=(",", ":")).encode("ascii")
|
||||
if len(payload) > limit:
|
||||
raise ProtocolError("guest message exceeds its size limit")
|
||||
connection.settimeout(_remaining(deadline))
|
||||
connection.sendall(struct.pack("!I", len(payload)) + payload)
|
||||
|
||||
|
||||
class JobScope:
|
||||
"""A delegated guest cgroup, keeping detached descendants in the same job."""
|
||||
|
||||
def __init__(self) -> None:
|
||||
entries = Path("/proc/self/cgroup").read_text(encoding="ascii").splitlines()
|
||||
unified = next((entry[3:] for entry in entries if entry.startswith("0::")), None)
|
||||
if unified is None or ".." in Path(unified).parts:
|
||||
raise ExecutionFailure("guest execution requires a delegated cgroup v2 hierarchy")
|
||||
parent = Path("/sys/fs/cgroup") / unified.lstrip("/")
|
||||
self.path = parent / f"uvm-exec-{uuid.uuid4().hex}"
|
||||
self.path.mkdir(mode=0o700)
|
||||
if not (self.path / "cgroup.kill").is_file():
|
||||
self.path.rmdir()
|
||||
raise ExecutionFailure("guest execution requires cgroup.kill (Linux 5.14 or newer)")
|
||||
|
||||
def kill(self) -> None:
|
||||
(self.path / "cgroup.kill").write_text("1", encoding="ascii")
|
||||
|
||||
def close(self) -> None:
|
||||
# cgroup.events reports descendants as well as processes in this cgroup.
|
||||
deadline = time.monotonic() + 1.0
|
||||
while "populated 1" in (self.path / "cgroup.events").read_text(encoding="ascii"):
|
||||
if time.monotonic() >= deadline:
|
||||
raise ExecutionFailure("guest job cleanup could not be confirmed", outcome_unknown=True)
|
||||
time.sleep(0.01)
|
||||
for child in sorted(self.path.rglob("*"), key=lambda path: len(path.parts), reverse=True):
|
||||
if child.is_dir():
|
||||
child.rmdir()
|
||||
self.path.rmdir()
|
||||
|
||||
|
||||
def _signal_group(process: subprocess.Popen, number: int) -> None:
|
||||
# Do not reap the group leader until after signaling: its PID pins the PGID.
|
||||
try:
|
||||
os.killpg(process.pid, number)
|
||||
except ProcessLookupError:
|
||||
pass
|
||||
|
||||
|
||||
def _exited(process: subprocess.Popen) -> bool:
|
||||
return os.waitid(os.P_PID, process.pid, os.WEXITED | os.WNOHANG | os.WNOWAIT) is not None
|
||||
|
||||
|
||||
def _cleanup_job(process: subprocess.Popen | None, scope: JobScope) -> None:
|
||||
failed = False
|
||||
try:
|
||||
scope.kill()
|
||||
except (OSError, ExecutionFailure):
|
||||
failed = True
|
||||
if process is not None:
|
||||
try:
|
||||
_signal_group(process, signal.SIGKILL)
|
||||
process.wait(timeout=1.0)
|
||||
except (OSError, subprocess.TimeoutExpired):
|
||||
failed = True
|
||||
finally:
|
||||
for name in ("stdout", "stderr"):
|
||||
getattr(process, name).close()
|
||||
try:
|
||||
scope.close()
|
||||
except (OSError, ExecutionFailure):
|
||||
failed = True
|
||||
if failed:
|
||||
raise ExecutionFailure("guest job cleanup could not be confirmed", outcome_unknown=process is not None)
|
||||
|
||||
|
||||
def run_command(command: str, username: str, timeout_seconds: float, *, scope_factory=JobScope) -> dict:
|
||||
"""Run one non-interactive guest job, drain both pipes, and bound its lifetime."""
|
||||
scope = scope_factory()
|
||||
process = None
|
||||
buffers = {"stdout": bytearray(), "stderr": bytearray()}
|
||||
captured = 0
|
||||
truncated = False
|
||||
timed_out = False
|
||||
try:
|
||||
# The fresh child joins its scope BEFORE dropping credentials or executing
|
||||
# user text. Avoid preexec_fn in the threaded agent.
|
||||
process = subprocess.Popen(
|
||||
[sys.executable, "-I", str(Path(__file__).resolve()), "--child", str(scope.path), username, command],
|
||||
stdin=subprocess.DEVNULL, stdout=subprocess.PIPE, stderr=subprocess.PIPE,
|
||||
start_new_session=True, env={"PATH": "/usr/sbin:/usr/bin:/sbin:/bin", "LANG": "C.UTF-8"},
|
||||
)
|
||||
deadline = time.monotonic() + timeout_seconds
|
||||
terminating = None
|
||||
killed = False
|
||||
with selectors.DefaultSelector() as selector:
|
||||
for name in buffers:
|
||||
stream = getattr(process, name)
|
||||
os.set_blocking(stream.fileno(), False)
|
||||
selector.register(stream, selectors.EVENT_READ, name)
|
||||
while True:
|
||||
now = time.monotonic()
|
||||
exited = _exited(process)
|
||||
if terminating is None and (exited or now >= deadline):
|
||||
timed_out = not exited
|
||||
terminating = now
|
||||
_signal_group(process, signal.SIGTERM if timed_out else signal.SIGKILL)
|
||||
if terminating is not None:
|
||||
if not killed and (exited or now >= terminating + 0.5):
|
||||
scope.kill()
|
||||
_signal_group(process, signal.SIGKILL)
|
||||
killed = True
|
||||
if killed and exited and not selector.get_map():
|
||||
break
|
||||
if now >= terminating + 3.0:
|
||||
raise ExecutionFailure("guest job cleanup could not be confirmed", outcome_unknown=True)
|
||||
for key, _events in selector.select(0.05):
|
||||
chunk = os.read(key.fileobj.fileno(), 65536)
|
||||
if not chunk:
|
||||
selector.unregister(key.fileobj)
|
||||
continue
|
||||
keep = min(len(chunk), MAX_OUTPUT_BYTES - captured)
|
||||
buffers[key.data].extend(chunk[:keep])
|
||||
captured += keep
|
||||
truncated |= keep < len(chunk)
|
||||
# Finally below kills the scope and reaps the leader before returning.
|
||||
except OSError as error:
|
||||
raise ExecutionFailure("guest command execution failed", outcome_unknown=process is not None) from error
|
||||
finally:
|
||||
_cleanup_job(process, scope)
|
||||
return {
|
||||
"exit_code": None if timed_out else process.returncode,
|
||||
"stdout": buffers["stdout"].decode("utf-8", errors="replace"),
|
||||
"stderr": buffers["stderr"].decode("utf-8", errors="replace"),
|
||||
"timed_out": timed_out,
|
||||
"truncated": truncated,
|
||||
}
|
||||
|
||||
|
||||
def child_main(scope: Path, username: str, command: str) -> None:
|
||||
"""Enter the job cgroup and account in a fresh interpreter, then replace it."""
|
||||
(scope / "cgroup.procs").write_text(str(os.getpid()), encoding="ascii")
|
||||
account = pwd.getpwnam(username)
|
||||
if os.geteuid() == 0:
|
||||
os.initgroups(account.pw_name, account.pw_gid)
|
||||
os.setgid(account.pw_gid)
|
||||
os.setuid(account.pw_uid)
|
||||
elif (account.pw_uid, account.pw_gid) != (os.geteuid(), os.getegid()):
|
||||
raise PermissionError("guest agent cannot assume the requested account")
|
||||
os.chdir(account.pw_dir)
|
||||
os.umask(0o022)
|
||||
environment = {
|
||||
"HOME": account.pw_dir, "USER": account.pw_name, "LOGNAME": account.pw_name,
|
||||
"SHELL": "/bin/sh", "PATH": "/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin",
|
||||
"LANG": "C.UTF-8",
|
||||
}
|
||||
os.execve("/bin/sh", ["sh", "-c", command], environment)
|
||||
|
||||
|
||||
def serve_connection(connection: socket.socket, config: dict, *, runner=run_command) -> None:
|
||||
with connection:
|
||||
try:
|
||||
deadline = time.monotonic() + CONNECT_TIMEOUT_SECONDS
|
||||
send_frame(connection, {"version": PROTOCOL_VERSION, **config}, 4096, deadline)
|
||||
request = receive_frame(connection, MAX_REQUEST_BYTES, deadline)
|
||||
if set(request) != {"version", "command", "sudo", "timeout_seconds"}:
|
||||
raise ProtocolError("invalid execution request fields")
|
||||
if type(request["version"]) is not int or request["version"] != PROTOCOL_VERSION:
|
||||
raise ProtocolError("unsupported guest protocol version")
|
||||
validate_command(request["command"], request["sudo"], request["timeout_seconds"])
|
||||
username = "root" if request["sudo"] else config["username"]
|
||||
result = runner(request["command"], username, request["timeout_seconds"])
|
||||
send_frame(connection, result, MAX_RESPONSE_BYTES, time.monotonic() + CONNECT_TIMEOUT_SECONDS)
|
||||
except (ProtocolError, ExecutionFailure, OSError, KeyError) as error:
|
||||
# Never log request text, guest output, or credentials to the console.
|
||||
try:
|
||||
send_frame(connection, {
|
||||
"error": "guest execution failed: " + str(error),
|
||||
"outcome_unknown": isinstance(error, ExecutionFailure) and error.outcome_unknown,
|
||||
}, 4096, time.monotonic() + CONNECT_TIMEOUT_SECONDS)
|
||||
except (OSError, ProtocolError):
|
||||
pass
|
||||
except EOFError:
|
||||
pass
|
||||
|
||||
|
||||
def load_config(path: Path = CONFIG_PATH) -> dict:
|
||||
descriptor = os.open(path, os.O_RDONLY | os.O_NOFOLLOW)
|
||||
with os.fdopen(descriptor, encoding="utf-8") as source:
|
||||
metadata = os.fstat(source.fileno())
|
||||
if (not stat.S_ISREG(metadata.st_mode) or metadata.st_uid != 0
|
||||
or metadata.st_mode & 0o077 or metadata.st_size > 4096):
|
||||
raise ValueError("guest agent configuration must be a root-owned private regular file")
|
||||
config = json.load(source)
|
||||
if (not isinstance(config, dict) or set(config) != {"vm_id", "username"}
|
||||
or not isinstance(config["vm_id"], str)
|
||||
or re.fullmatch(r"[A-Za-z0-9][A-Za-z0-9_.-]{0,80}", config["vm_id"]) is None
|
||||
or not isinstance(config["username"], str)
|
||||
or re.fullmatch(r"[a-z_][a-z0-9_-]{0,31}\$?", config["username"]) is None):
|
||||
raise ValueError("invalid guest agent configuration")
|
||||
pwd.getpwnam(config["username"])
|
||||
return config
|
||||
|
||||
|
||||
def main() -> None:
|
||||
if len(sys.argv) == 5 and sys.argv[1] == "--child":
|
||||
try:
|
||||
child_main(Path(sys.argv[2]), sys.argv[3], sys.argv[4])
|
||||
except (OSError, KeyError) as error:
|
||||
print(f"uvm-agent: could not launch guest command: {error}", file=sys.stderr)
|
||||
sys.exit(126)
|
||||
return
|
||||
if len(sys.argv) != 1 or os.geteuid() != 0:
|
||||
raise SystemExit("uvm-agent must run as a root systemd service")
|
||||
config = load_config()
|
||||
slots = threading.BoundedSemaphore(MAX_CONCURRENT_COMMANDS)
|
||||
|
||||
def handle(connection: socket.socket) -> None:
|
||||
try:
|
||||
serve_connection(connection, config)
|
||||
finally:
|
||||
slots.release()
|
||||
|
||||
with socket.socket(socket.AF_VSOCK, socket.SOCK_STREAM) as listener:
|
||||
listener.bind((socket.VMADDR_CID_ANY, AGENT_PORT))
|
||||
listener.listen(MAX_CONCURRENT_COMMANDS)
|
||||
while True:
|
||||
connection, peer = listener.accept()
|
||||
# Guest-local callers must not use the root agent to elevate themselves.
|
||||
if peer[0] != socket.VMADDR_CID_HOST:
|
||||
connection.close()
|
||||
continue
|
||||
if not slots.acquire(blocking=False):
|
||||
connection.close()
|
||||
continue
|
||||
threading.Thread(target=handle, args=(connection,), daemon=True).start()
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,114 @@
|
||||
"""Host-side, one-shot guest execution over Firecracker's vsock Unix socket."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import socket
|
||||
import stat
|
||||
import struct
|
||||
import time
|
||||
from dataclasses import dataclass
|
||||
from pathlib import Path
|
||||
|
||||
from .errors import GuestAgentUnavailableError, GuestExecutionError, ValidationError
|
||||
from .guest_agent import (
|
||||
AGENT_PORT, CLEANUP_SECONDS, CONNECT_TIMEOUT_SECONDS, MAX_OUTPUT_BYTES,
|
||||
MAX_REQUEST_BYTES, MAX_RESPONSE_BYTES, PROTOCOL_VERSION, ProtocolError,
|
||||
receive_exact, receive_frame, send_frame, validate_command,
|
||||
)
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class ExecResult:
|
||||
exit_code: int | None
|
||||
stdout: str
|
||||
stderr: str
|
||||
timed_out: bool
|
||||
truncated: bool
|
||||
|
||||
|
||||
def validate_exec(command: object, sudo: object, timeout_seconds: object) -> None:
|
||||
try:
|
||||
validate_command(command, sudo, timeout_seconds)
|
||||
except ProtocolError as error:
|
||||
raise ValidationError(str(error)) from error
|
||||
|
||||
|
||||
class GuestSession:
|
||||
"""A connected session pins the selected VMM; never reconnect after submission."""
|
||||
|
||||
def __init__(self, connection: socket.socket) -> None:
|
||||
self._connection = connection
|
||||
|
||||
@classmethod
|
||||
def connect(cls, path: Path, vm_id: str, username: str, pid: int) -> GuestSession:
|
||||
connection = None
|
||||
try:
|
||||
connection = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM)
|
||||
metadata = path.lstat()
|
||||
if not stat.S_ISSOCK(metadata.st_mode):
|
||||
raise ProtocolError("guest socket is not a Unix socket")
|
||||
deadline = time.monotonic() + CONNECT_TIMEOUT_SECONDS
|
||||
connection.settimeout(CONNECT_TIMEOUT_SECONDS)
|
||||
connection.connect(str(path))
|
||||
peer_pid, _uid, _gid = struct.unpack("3i", connection.getsockopt(socket.SOL_SOCKET, socket.SO_PEERCRED, 12))
|
||||
if peer_pid != pid:
|
||||
raise ProtocolError("guest socket does not belong to the selected VM process")
|
||||
connection.sendall(f"CONNECT {AGENT_PORT}\n".encode("ascii"))
|
||||
acknowledgement = bytearray()
|
||||
while not acknowledgement.endswith(b"\n") and len(acknowledgement) < 32:
|
||||
acknowledgement.extend(receive_exact(connection, 1, deadline))
|
||||
parts = bytes(acknowledgement).split()
|
||||
if (not acknowledgement.endswith(b"\n") or len(parts) != 2
|
||||
or parts[0] != b"OK" or not parts[1].isdigit()):
|
||||
raise ProtocolError("invalid Firecracker vsock acknowledgement")
|
||||
hello = receive_frame(connection, 4096, deadline)
|
||||
if (set(hello) != {"version", "vm_id", "username"}
|
||||
or type(hello["version"]) is not int or hello["version"] != PROTOCOL_VERSION
|
||||
or hello["vm_id"] != vm_id or hello["username"] != username):
|
||||
raise ProtocolError("guest agent identity or protocol version does not match this VM")
|
||||
return cls(connection)
|
||||
except (OSError, EOFError, ProtocolError) as error:
|
||||
if connection is not None:
|
||||
connection.close()
|
||||
raise GuestAgentUnavailableError(
|
||||
"guest agent is unavailable or not ready; install it with POST /vms/{vm_id}/agent/install "
|
||||
"while stopped, then start the VM (vsock-capable kernel required)"
|
||||
) from error
|
||||
|
||||
def execute(self, command: str, *, sudo: bool, timeout_seconds: float) -> ExecResult:
|
||||
validate_exec(command, sudo, timeout_seconds)
|
||||
try:
|
||||
send_frame(self._connection, {
|
||||
"version": PROTOCOL_VERSION, "command": command,
|
||||
"sudo": sudo, "timeout_seconds": timeout_seconds,
|
||||
}, MAX_REQUEST_BYTES, time.monotonic() + CONNECT_TIMEOUT_SECONDS)
|
||||
result = receive_frame(
|
||||
self._connection, MAX_RESPONSE_BYTES,
|
||||
time.monotonic() + timeout_seconds + CLEANUP_SECONDS + CONNECT_TIMEOUT_SECONDS,
|
||||
)
|
||||
if "error" in result:
|
||||
if (set(result) != {"error", "outcome_unknown"} or not isinstance(result["error"], str)
|
||||
or len(result["error"]) > 4096 or type(result["outcome_unknown"]) is not bool):
|
||||
raise ProtocolError("invalid guest failure response")
|
||||
result["error"].encode("utf-8") # Reject invalid Unicode before rendering an HTTP error.
|
||||
raise GuestExecutionError(result["error"], outcome_unknown=result["outcome_unknown"])
|
||||
if set(result) != {"exit_code", "stdout", "stderr", "timed_out", "truncated"}:
|
||||
raise ProtocolError("invalid guest result fields")
|
||||
if (type(result["timed_out"]) is not bool or type(result["truncated"]) is not bool
|
||||
or not isinstance(result["stdout"], str) or not isinstance(result["stderr"], str)
|
||||
or (result["timed_out"] and result["exit_code"] is not None)
|
||||
or (not result["timed_out"] and (
|
||||
type(result["exit_code"]) is not int or not -64 <= result["exit_code"] <= 255))):
|
||||
raise ProtocolError("invalid guest result types")
|
||||
if sum(len(result[name].encode("utf-8")) for name in ("stdout", "stderr")) > 3 * MAX_OUTPUT_BYTES:
|
||||
raise ProtocolError("guest output exceeds its limit")
|
||||
return ExecResult(**result)
|
||||
except (OSError, EOFError, ProtocolError, UnicodeError) as error:
|
||||
raise GuestExecutionError(
|
||||
"guest execution connection was lost or returned an invalid result; "
|
||||
"the command may have run and will not be retried",
|
||||
outcome_unknown=True,
|
||||
) from error
|
||||
|
||||
def close(self) -> None:
|
||||
self._connection.close()
|
||||
+165
@@ -2,6 +2,8 @@
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import hashlib
|
||||
import json
|
||||
import os
|
||||
import re
|
||||
import stat
|
||||
@@ -11,6 +13,7 @@ from dataclasses import dataclass
|
||||
from ipaddress import IPv4Address
|
||||
from pathlib import Path
|
||||
|
||||
from . import guest_agent
|
||||
from .config import MAX_STORAGE_MIB, MIN_STORAGE_MIB, Settings
|
||||
from .errors import UvmError, ValidationError
|
||||
from .integrity import load_manifest, verify_file
|
||||
@@ -46,6 +49,54 @@ _LEGACY_GUEST_NETWORK_TEMPLATE = GUEST_NETWORK_TEMPLATE.replace(
|
||||
"KeepConfiguration=yes", "KeepConfiguration=static",
|
||||
)
|
||||
|
||||
GUEST_AGENT_SERVICE = """\
|
||||
[Unit]
|
||||
Description=UVM guest command execution agent
|
||||
After=local-fs.target network.target
|
||||
ConditionPathExists=/etc/uvm-agent.json
|
||||
StartLimitIntervalSec=0
|
||||
|
||||
[Service]
|
||||
Type=simple
|
||||
ExecStartPre=/bin/sh /usr/lib/uvm/prepare-agent.sh
|
||||
ExecStart=/usr/bin/python3 -I /usr/lib/uvm/agent.py
|
||||
Restart=on-failure
|
||||
RestartSec=5
|
||||
TimeoutStartSec=600
|
||||
TimeoutStopSec=5
|
||||
KillMode=control-group
|
||||
Delegate=yes
|
||||
UMask=0077
|
||||
|
||||
[Install]
|
||||
WantedBy=multi-user.target
|
||||
"""
|
||||
|
||||
# Runs only INSIDE the VM. Old Jammy guests can acquire the runtime without SSH
|
||||
# or executing a retained guest's binaries in a privileged host chroot.
|
||||
GUEST_AGENT_PREPARE = """\
|
||||
set -eu
|
||||
check_python() {
|
||||
/usr/bin/python3 -I -c 'import sys, json, math, pwd, selectors, socket, struct, subprocess, threading, uuid; assert sys.version_info >= (3, 10)' 2>/dev/null
|
||||
}
|
||||
if check_python; then
|
||||
exit 0
|
||||
fi
|
||||
export DEBIAN_FRONTEND=noninteractive LC_ALL=C
|
||||
/usr/bin/apt-get -o DPkg::Lock::Timeout=300 -o APT::Update::Error-Mode=any update
|
||||
/usr/bin/apt-get -o DPkg::Lock::Timeout=300 -y --no-install-recommends install python3
|
||||
check_python
|
||||
"""
|
||||
|
||||
|
||||
def guest_agent_files() -> dict[str, bytes]:
|
||||
"""The same standalone payload is used by the builder and offline upgrades."""
|
||||
return {
|
||||
"/usr/lib/uvm/agent.py": Path(guest_agent.__file__).read_bytes(),
|
||||
"/usr/lib/uvm/prepare-agent.sh": GUEST_AGENT_PREPARE.encode("utf-8"),
|
||||
"/etc/systemd/system/uvm-agent.service": GUEST_AGENT_SERVICE.encode("utf-8"),
|
||||
}
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class GuestAssets:
|
||||
@@ -232,6 +283,7 @@ class ImageStore:
|
||||
)
|
||||
if marker is None or marker.path.read_text(encoding="utf-8") not in {
|
||||
"uvm-ubuntu-jammy-v1\n", "uvm-ubuntu-jammy-v2\n", "uvm-ubuntu-jammy-v3\n",
|
||||
"uvm-ubuntu-jammy-v4\n",
|
||||
}:
|
||||
return None
|
||||
profile = self._read_guest_file(
|
||||
@@ -246,6 +298,119 @@ class ImageStore:
|
||||
return None
|
||||
return profile
|
||||
|
||||
def provision_guest_agent(self, disk: Path, vm_id: str, username: str, *, install: bool = False) -> bool:
|
||||
"""Configure new templates, or explicitly install on a proven-offline disk.
|
||||
|
||||
The lifecycle lock and confirmed VMM exit are required. A checked sparse
|
||||
copy preserves the original disk on failure and makes retries atomic.
|
||||
"""
|
||||
parse_username(username)
|
||||
if re.fullmatch(r"[A-Za-z0-9][A-Za-z0-9_.-]{0,80}", vm_id) is None:
|
||||
raise ValidationError("invalid VM ID for guest agent configuration")
|
||||
try:
|
||||
metadata = disk.lstat()
|
||||
if (not stat.S_ISREG(metadata.st_mode) or metadata.st_nlink != 1
|
||||
or disk.resolve() == self._settings.rootfs_image.resolve()):
|
||||
raise UvmError("private VM disk is unsafe for guest agent installation")
|
||||
with tempfile.TemporaryDirectory(prefix="uvm-guest-agent-") as name:
|
||||
files = Path(name)
|
||||
if not install:
|
||||
marker = self._read_guest_file(disk, "/etc/uvm-image", files / "marker",
|
||||
required=False, regular_only=True)
|
||||
if marker is None or marker.path.read_text(encoding="utf-8") != "uvm-ubuntu-jammy-v4\n":
|
||||
return False
|
||||
descriptor, candidate = tempfile.mkstemp(prefix=".rootfs-agent-", suffix=".ext4", dir=disk.parent)
|
||||
os.close(descriptor)
|
||||
temporary = Path(candidate)
|
||||
try:
|
||||
self._copy_sparse(disk, temporary)
|
||||
temporary.chmod(0o600)
|
||||
checked = self._runner.run(("e2fsck", "-f", "-p", temporary), check=False, capture=True)
|
||||
if checked.returncode not in (0, 1):
|
||||
raise UvmError("filesystem check failed before guest agent installation")
|
||||
# All ownership/content checks happen AFTER journal replay.
|
||||
self._install_agent_files(temporary, files, vm_id, username)
|
||||
self._runner.run(("e2fsck", "-f", "-n", temporary), capture=True)
|
||||
with temporary.open("rb") as image:
|
||||
os.fsync(image.fileno())
|
||||
os.replace(temporary, disk)
|
||||
directory = os.open(disk.parent, os.O_RDONLY)
|
||||
try:
|
||||
os.fsync(directory)
|
||||
finally:
|
||||
os.close(directory)
|
||||
finally:
|
||||
temporary.unlink(missing_ok=True)
|
||||
return True
|
||||
except (OSError, UnicodeError) as error:
|
||||
raise UvmError(f"could not provision guest agent: {error}") from error
|
||||
|
||||
def _install_agent_files(self, disk: Path, files: Path, vm_id: str, username: str) -> None:
|
||||
for binary in ("/usr/lib/systemd/systemd", "/usr/bin/apt-get"):
|
||||
result = self._runner.run(("debugfs", "-R", f"stat {binary}", disk), capture=True)
|
||||
if re.search(r"Type:\s+regular\b", result.stdout) is None:
|
||||
raise UvmError("guest agent installation requires a systemd/apt image (Ubuntu 22.04 or newer)")
|
||||
for directory in ("/etc", "/usr", "/usr/lib", "/usr/lib/uvm", "/etc/systemd",
|
||||
"/etc/systemd/system", "/etc/systemd/system/multi-user.target.wants"):
|
||||
self._agent_directory(disk, directory)
|
||||
payloads = guest_agent_files()
|
||||
payloads["/etc/uvm-agent.json"] = (json.dumps({"vm_id": vm_id, "username": username}) + "\n").encode("utf-8")
|
||||
manifest_path = "/etc/uvm-agent-files.json"
|
||||
previous = self._agent_file(disk, manifest_path, files / "manifest")
|
||||
old_hashes = {}
|
||||
if previous is not None:
|
||||
try:
|
||||
old_hashes = json.loads(previous.path.read_bytes())
|
||||
except (ValueError, UnicodeError) as error:
|
||||
raise UvmError("invalid guest agent ownership manifest") from error
|
||||
if (not isinstance(old_hashes, dict) or set(old_hashes) - set(payloads)
|
||||
or any(not isinstance(value, str) or re.fullmatch(r"[0-9a-f]{64}", value) is None
|
||||
for value in old_hashes.values())):
|
||||
raise UvmError("invalid guest agent ownership manifest")
|
||||
hashes = {path: hashlib.sha256(content).hexdigest() for path, content in payloads.items()}
|
||||
for index, (path, content) in enumerate(payloads.items()):
|
||||
existing = self._agent_file(disk, path, files / f"existing-{index}")
|
||||
if existing is not None:
|
||||
digest = hashlib.sha256(existing.path.read_bytes()).hexdigest()
|
||||
if digest not in {hashes[path], old_hashes.get(path)}:
|
||||
raise UvmError(f"refusing to overwrite a customized guest agent file: {path}")
|
||||
source = files / f"payload-{index}"
|
||||
source.write_bytes(content)
|
||||
self._write_guest_file(disk, path, _GuestFile(source, 0o600 if path.endswith(".json") else 0o644, 0, 0))
|
||||
source = files / "new-manifest"
|
||||
source.write_text(json.dumps(hashes, sort_keys=True) + "\n", encoding="utf-8")
|
||||
self._write_guest_file(disk, manifest_path, _GuestFile(source, 0o600, 0, 0))
|
||||
link = "/etc/systemd/system/multi-user.target.wants/uvm-agent.service"
|
||||
target = "/etc/systemd/system/uvm-agent.service"
|
||||
result = self._runner.run(("debugfs", "-R", f"stat {link}", disk), capture=True)
|
||||
if "file not found" in (result.stdout + result.stderr).lower():
|
||||
self._runner.run(("debugfs", "-w", "-R", f"symlink {link} {target}", disk), capture=True)
|
||||
result = self._runner.run(("debugfs", "-R", f"stat {link}", disk), capture=True)
|
||||
if (re.search(r"Type:\s+symlink\b", result.stdout) is None
|
||||
or f'Fast link dest: "{target}"' not in result.stdout):
|
||||
raise UvmError("could not enable guest agent without replacing a customized service link")
|
||||
|
||||
def _agent_directory(self, disk: Path, path: str) -> None:
|
||||
result = self._runner.run(("debugfs", "-R", f"stat {path}", disk), capture=True)
|
||||
if "file not found" in (result.stdout + result.stderr).lower():
|
||||
self._create_guest_directory(disk, path, 0o755, 0, 0)
|
||||
return
|
||||
mode, owner = _MODE_PATTERN.search(result.stdout), _OWNER_PATTERN.search(result.stdout)
|
||||
if (re.search(r"Type:\s+directory\b", result.stdout) is None or mode is None or owner is None
|
||||
or int(owner.group(1)) != 0 or int(mode.group(1), 8) & 0o022):
|
||||
raise UvmError(f"unsafe guest agent directory: {path}")
|
||||
|
||||
def _agent_file(self, disk: Path, path: str, destination: Path) -> _GuestFile | None:
|
||||
result = self._runner.run(("debugfs", "-R", f"stat {path}", disk), capture=True)
|
||||
if "file not found" in (result.stdout + result.stderr).lower():
|
||||
return None
|
||||
if re.search(r"Type:\s+regular\b", result.stdout) is None:
|
||||
raise UvmError(f"unsafe guest agent file: {path}")
|
||||
source = self._read_guest_file(disk, path, destination)
|
||||
if (source.uid != 0 or source.mode & 0o022 or source.links != 1 or source.has_extended_attributes):
|
||||
raise UvmError(f"unsafe guest agent file: {path}")
|
||||
return source
|
||||
|
||||
@staticmethod
|
||||
def _validate_storage(storage_mib: int) -> None:
|
||||
if type(storage_mib) is not int or not MIN_STORAGE_MIB <= storage_mib <= MAX_STORAGE_MIB:
|
||||
|
||||
+13
-5
@@ -14,17 +14,17 @@ from pathlib import Path
|
||||
|
||||
from .config import Settings
|
||||
from .errors import UvmError
|
||||
from .images import GUEST_NETWORK_TEMPLATE, GuestAssets
|
||||
from .images import GUEST_NETWORK_TEMPLATE, GuestAssets, guest_agent_files
|
||||
from .integrity import load_manifest, require_checksum, sha256_file, verify_file, write_manifest
|
||||
from .state import StateStore
|
||||
from .system import CommandRunner, check_kvm, ensure_data_directories, require_root
|
||||
|
||||
|
||||
_UBUNTU_RELEASE = "jammy"
|
||||
_ROOTFS_RECIPE = "uvm-ubuntu-jammy-v3\n"
|
||||
_ROOTFS_RECIPE = "uvm-ubuntu-jammy-v4\n"
|
||||
_GUEST_PACKAGES = (
|
||||
"ca-certificates", "openssh-server", "sudo", "systemd-sysv", "udev",
|
||||
"dbus", "libpam-systemd", "iproute2", "iputils-ping", "curl",
|
||||
"dbus", "libpam-systemd", "iproute2", "iputils-ping", "curl", "python3",
|
||||
)
|
||||
# Keep the image mountable by the older Firecracker quickstart guest kernels.
|
||||
_EXT4_FEATURES = (
|
||||
@@ -397,7 +397,14 @@ class Installer:
|
||||
machine_id.symlink_to("../../../etc/machine-id")
|
||||
for key in (root / "etc/ssh").glob("ssh_host_*_key*"):
|
||||
key.unlink()
|
||||
self._runner.run(("systemctl", "--root", root, "enable", "ssh.service", "systemd-networkd.service"), capture=True)
|
||||
for guest_path, contents in guest_agent_files().items():
|
||||
path = root / guest_path.lstrip("/")
|
||||
path.parent.mkdir(parents=True, exist_ok=True)
|
||||
path.parent.chmod(0o755)
|
||||
path.write_bytes(contents)
|
||||
path.chmod(0o644)
|
||||
self._runner.run(("systemctl", "--root", root, "enable", "ssh.service", "systemd-networkd.service",
|
||||
"uvm-agent.service"), capture=True)
|
||||
self._runner.run(("systemctl", "--root", root, "set-default", "multi-user.target"), capture=True)
|
||||
|
||||
@staticmethod
|
||||
@@ -406,6 +413,7 @@ class Installer:
|
||||
"etc/passwd", "etc/shadow", "etc/group", "etc/sudoers", "etc/ssh/sshd_config",
|
||||
"usr/bin/sudo", "usr/bin/apt-get", "usr/sbin/sshd", "usr/lib/systemd/systemd",
|
||||
"var/lib/dpkg/status", "var/lib/dpkg/info/apt.list", "var/lib/dpkg/info/sudo.list",
|
||||
"usr/bin/python3", "usr/lib/uvm/agent.py", "etc/systemd/system/uvm-agent.service",
|
||||
):
|
||||
if not (root / relative).is_file():
|
||||
raise UvmError(f"built Ubuntu rootfs is missing /{relative}")
|
||||
@@ -414,7 +422,7 @@ class Installer:
|
||||
fields = dict(line.split(": ", 1) for line in paragraph.splitlines() if ": " in line)
|
||||
if fields.get("Status") == "install ok installed":
|
||||
installed.add(fields.get("Package"))
|
||||
if not {"apt", "sudo", "openssh-server", "systemd-sysv"}.issubset(installed):
|
||||
if not {"apt", "sudo", "openssh-server", "systemd-sysv", "python3"}.issubset(installed):
|
||||
raise UvmError("built Ubuntu rootfs has incomplete apt/dpkg package metadata")
|
||||
if not (root / "usr/bin/sudo").stat().st_mode & 0o4000:
|
||||
raise UvmError("built Ubuntu rootfs sudo is missing its setuid permission")
|
||||
|
||||
@@ -17,6 +17,8 @@ from .errors import ConflictError, NotFoundError, UvmError, ValidationError
|
||||
from .firecracker.api import FirecrackerClient
|
||||
from .firecracker.config import build_config, write_config
|
||||
from .firecracker.process import FirecrackerProcessManager, ProcessInfo
|
||||
from .guest_agent import DEFAULT_TIMEOUT_SECONDS
|
||||
from .guest_exec import ExecResult, GuestSession, validate_exec
|
||||
from .images import ImageStore
|
||||
from .integrity import load_manifest, verify_file
|
||||
from .known_hosts import KnownHostsManager
|
||||
@@ -58,6 +60,7 @@ class LifecycleService:
|
||||
network: NetworkManager, process: FirecrackerProcessManager, *,
|
||||
known_hosts: KnownHostsManager,
|
||||
client_factory: Callable[[Path, float], FirecrackerClient] = FirecrackerClient,
|
||||
guest_connector: Callable[[Path, str, str, int], GuestSession] = GuestSession.connect,
|
||||
clock: Callable[[], float] = time.time,
|
||||
) -> None:
|
||||
self._settings = settings
|
||||
@@ -67,6 +70,7 @@ class LifecycleService:
|
||||
self._process = process
|
||||
self._known_hosts = known_hosts
|
||||
self._client_factory = client_factory
|
||||
self._guest_connector = guest_connector
|
||||
self._clock = clock
|
||||
|
||||
def create(self, spec: VmSpec) -> VmRecord:
|
||||
@@ -90,6 +94,7 @@ class LifecycleService:
|
||||
disk = self._images.create_vm_disk(assets.rootfs, Path(vm.disk), storage_mib=spec.storage_mib)
|
||||
self._images.provision_credentials(disk, vm.username, vm.password)
|
||||
self._images.configure_guest_network(disk, vm.guest_ip)
|
||||
self._images.provision_guest_agent(disk, vm.id, vm.username)
|
||||
vm.disk_ready = True
|
||||
self._save(vm)
|
||||
self._boot(vm, assets.kernel)
|
||||
@@ -494,6 +499,41 @@ class LifecycleService:
|
||||
def find_for_ssh(self, identifier: str) -> VmRecord:
|
||||
return self._state_store.find_by_id_or_ip(identifier)
|
||||
|
||||
def execute(
|
||||
self, vm_id: str, command: str, *, sudo: bool = False,
|
||||
timeout_seconds: float = DEFAULT_TIMEOUT_SECONDS,
|
||||
) -> ExecResult:
|
||||
"""Pin a running VM's guest connection under the lock, then wait unlocked."""
|
||||
require_root()
|
||||
validate_exec(command, sudo, timeout_seconds)
|
||||
with self._state_store.operation_lock():
|
||||
vm = self._load_for_operation(vm_id, exact_id=True)
|
||||
self._not_deleting(vm)
|
||||
if (vm.status != "running" or vm.operation is not None or vm.pending_resize is not None
|
||||
or self._process.observe(vm) != "alive"):
|
||||
raise ConflictError("guest execution requires a running VM with no pending lifecycle operation")
|
||||
session = self._guest_connector(self._settings.guest_socket_path(vm.id), vm.id, vm.username, vm.pid)
|
||||
try:
|
||||
# This session stays attached to that VMM even if the VM is stopped,
|
||||
# deleted, or restarted concurrently. Never reconnect/retry a command.
|
||||
return session.execute(command, sudo=sudo, timeout_seconds=timeout_seconds)
|
||||
finally:
|
||||
session.close()
|
||||
|
||||
def install_guest_agent(self, vm_id: str) -> VmRecord:
|
||||
"""Explicitly install/upgrade the agent on a cleanly stopped private disk."""
|
||||
require_root()
|
||||
with self._state_store.operation_lock():
|
||||
vm = self._load_for_operation(vm_id, exact_id=True)
|
||||
self._not_deleting(vm)
|
||||
if (vm.status != "stopped" or vm.operation is not None or vm.pending_resize is not None
|
||||
or self._process.observe(vm) != "gone"):
|
||||
raise ConflictError("stop the VM and finish pending operations before installing the guest agent")
|
||||
if not vm.disk_ready or not vm.disk:
|
||||
raise ConflictError("guest agent installation requires a completely provisioned private disk")
|
||||
self._images.provision_guest_agent(Path(vm.disk), vm.id, vm.username, install=True)
|
||||
return vm
|
||||
|
||||
def log_path(self, identifier: str) -> Path:
|
||||
vm = self._state_store.find_by_id_or_ip(identifier)
|
||||
self._validate_paths(vm)
|
||||
@@ -566,6 +606,8 @@ class LifecycleService:
|
||||
directory = self._settings.vm_dir(vm.id).absolute()
|
||||
if directory.is_symlink():
|
||||
raise ConflictError("VM directory must not be a symbolic link")
|
||||
if self._settings.guest_socket_path(vm.id).is_symlink():
|
||||
raise ConflictError("unsafe guest vsock path")
|
||||
for field, filename in (("socket", "firecracker.sock"), ("config", "config.json"),
|
||||
("log", "firecracker.log"), ("disk", "rootfs.ext4")):
|
||||
path = getattr(vm, field)
|
||||
|
||||
@@ -9,8 +9,11 @@ from ..api_models import (
|
||||
PortSyncResponse,
|
||||
PortUpdateRequest,
|
||||
VmActionResultResponse,
|
||||
VmAgentInstallResponse,
|
||||
VmBulkActionRequest,
|
||||
VmCreateRequest,
|
||||
VmExecRequest,
|
||||
VmExecResponse,
|
||||
VmLogResponse,
|
||||
VmResponse,
|
||||
VmResizeRequest,
|
||||
@@ -202,6 +205,32 @@ def sync_ports(
|
||||
)
|
||||
|
||||
|
||||
@router.post("/{vm_id}/exec", response_model=VmExecResponse)
|
||||
def execute_in_vm(
|
||||
vm_id: str,
|
||||
request: VmExecRequest,
|
||||
application: Application = Depends(get_authorized_application),
|
||||
) -> VmExecResponse:
|
||||
"""Execute a bounded, non-interactive guest command over vsock (exact VM ID)."""
|
||||
result = application.lifecycle.execute(
|
||||
vm_id, request.command, sudo=request.sudo, timeout_seconds=request.timeout_seconds,
|
||||
)
|
||||
return VmExecResponse(
|
||||
exit_code=result.exit_code, stdout=result.stdout, stderr=result.stderr,
|
||||
timed_out=result.timed_out, truncated=result.truncated,
|
||||
)
|
||||
|
||||
|
||||
@router.post("/{vm_id}/agent/install", response_model=VmAgentInstallResponse)
|
||||
def install_guest_agent(
|
||||
vm_id: str,
|
||||
application: Application = Depends(get_authorized_application),
|
||||
) -> VmAgentInstallResponse:
|
||||
"""Install/upgrade the execution agent on a stopped VM's retained private disk."""
|
||||
vm = application.lifecycle.install_guest_agent(vm_id)
|
||||
return VmAgentInstallResponse(id=vm.id)
|
||||
|
||||
|
||||
@router.get("/{identifier}/logs", response_model=VmLogResponse)
|
||||
def get_vm_logs(
|
||||
identifier: str,
|
||||
|
||||
+12
-2
@@ -9,7 +9,10 @@ import ssl
|
||||
from typing import Any
|
||||
|
||||
from .app import Application, build_application
|
||||
from .errors import ConfigurationError, ConflictError, NotFoundError, UvmError, ValidationError
|
||||
from .errors import (
|
||||
ConfigurationError, ConflictError, GuestAgentUnavailableError, GuestExecutionError,
|
||||
NotFoundError, UvmError, ValidationError,
|
||||
)
|
||||
|
||||
|
||||
def create_api(application: Application | None = None, *, host: str | None = None) -> Any:
|
||||
@@ -49,9 +52,12 @@ def create_api(application: Application | None = None, *, host: str | None = Non
|
||||
|
||||
@api.exception_handler(UvmError)
|
||||
async def handle_uvm_error(_request: Any, error: UvmError) -> Any:
|
||||
detail: dict[str, Any] = {"message": str(error)}
|
||||
if isinstance(error, GuestExecutionError):
|
||||
detail["outcome_unknown"] = error.outcome_unknown
|
||||
return JSONResponse(
|
||||
status_code=_http_status_for(error),
|
||||
content={"error": {"message": str(error)}},
|
||||
content={"error": detail},
|
||||
)
|
||||
|
||||
api.include_router(health_router)
|
||||
@@ -130,6 +136,10 @@ def _validate_server_settings(
|
||||
|
||||
|
||||
def _http_status_for(error: UvmError) -> int:
|
||||
if isinstance(error, GuestAgentUnavailableError):
|
||||
return 503
|
||||
if isinstance(error, GuestExecutionError):
|
||||
return 502
|
||||
if isinstance(error, NotFoundError):
|
||||
return 404
|
||||
if isinstance(error, ConflictError):
|
||||
|
||||
Reference in New Issue
Block a user