feat: Add guest execution agent and vsock support

- Introduced a standalone guest execution agent to facilitate command execution within the guest VM over Firecracker's vsock.
- Enhanced FirecrackerClient to support vsock configuration, validating guest CID and UDS path.
- Updated configuration to include vsock settings by default.
- Refactored FirecrackerProcessManager to manage runtime sockets more effectively.
- Implemented guest agent installation and provisioning in the image store.
- Added API endpoints for executing commands in the VM and installing the guest agent.
- Improved error handling for guest execution failures and agent availability.
This commit is contained in:
2026-09-25 17:39:06 +05:30
parent dca7f78dae
commit b0bf99a6ee
25 changed files with 1893 additions and 30 deletions
+38 -1
View File
@@ -5,9 +5,12 @@ from __future__ import annotations
from ipaddress import IPv4Address
from typing import Literal
from pydantic import BaseModel, ConfigDict, Field, SecretStr
from pydantic import BaseModel, ConfigDict, Field, SecretStr, model_validator
from .domain import VmRecord
from .errors import ValidationError
from .guest_agent import DEFAULT_TIMEOUT_SECONDS, MAX_COMMAND_BYTES, MAX_TIMEOUT_SECONDS
from .guest_exec import validate_exec
class HealthResponse(BaseModel):
@@ -112,6 +115,40 @@ class VmLogResponse(BaseModel):
truncated: bool
class VmExecRequest(BaseModel):
model_config = ConfigDict(extra="forbid")
command: str = Field(strict=True, min_length=1, max_length=MAX_COMMAND_BYTES, repr=False,
description="Command or multiline script interpreted by /bin/sh inside the guest; at most 64 KiB UTF-8")
sudo: bool = Field(default=False, strict=True, description="Run as guest root instead of the VM's configured login user")
timeout_seconds: float = Field(default=DEFAULT_TIMEOUT_SECONDS, strict=True, gt=0,
le=MAX_TIMEOUT_SECONDS, allow_inf_nan=False)
@model_validator(mode="before")
@classmethod
def validate_execution(cls, value: object) -> object:
# Use the domain error envelope: echoing NaN or unpaired surrogates in
# FastAPI's validation-error input can itself fail JSON serialization.
if not isinstance(value, dict) or set(value) - {"command", "sudo", "timeout_seconds"}:
raise ValidationError("execution requires an object with command, sudo, and timeout_seconds fields only")
validate_exec(value.get("command"), value.get("sudo", False), value.get("timeout_seconds", DEFAULT_TIMEOUT_SECONDS))
return value
class VmExecResponse(BaseModel):
exit_code: int | None
stdout: str
stderr: str
timed_out: bool
truncated: bool
class VmAgentInstallResponse(BaseModel):
id: str
status: Literal["installed"] = "installed"
restart_required: bool = True
class DiagnosticResponse(BaseModel):
name: str
ok: bool
+3
View File
@@ -122,6 +122,9 @@ class Settings:
def vm_dir(self, vm_id: str) -> Path:
return self.vms_dir / vm_id
def guest_socket_path(self, vm_id: str) -> Path:
return self.vm_dir(vm_id) / "vsock.sock"
@classmethod
def from_environment(cls) -> "Settings":
"""Load optional deployment overrides while retaining script defaults."""
+12
View File
@@ -39,6 +39,18 @@ class FirecrackerError(UvmError):
"""Raised when Firecracker cannot be started or configured."""
class GuestAgentUnavailableError(UvmError):
"""The VM's execution agent could not be reached before submission."""
class GuestExecutionError(UvmError):
"""Guest execution failed; a lost response does not prove the command failed."""
def __init__(self, message: str, *, outcome_unknown: bool = False) -> None:
super().__init__(message)
self.outcome_unknown = outcome_unknown
class TapCreationError(UvmError):
"""Raised when TAP setup fails after creating a host interface."""
+6
View File
@@ -36,6 +36,7 @@ class FirecrackerClient:
drives = config["drives"]
interfaces = config["network-interfaces"]
entropy = config.get("entropy")
vsock = config.get("vsock")
if not isinstance(machine, Mapping) or not isinstance(boot, Mapping):
raise FirecrackerError("invalid Firecracker configuration")
if not isinstance(drives, list) or not drives:
@@ -44,6 +45,10 @@ class FirecrackerClient:
raise FirecrackerError("Firecracker configuration has no network interface")
if not isinstance(entropy, Mapping):
raise FirecrackerError("Firecracker configuration has no valid entropy device")
if (not isinstance(vsock, Mapping) or type(vsock.get("guest_cid")) is not int
or not 3 <= vsock["guest_cid"] < 2**32 - 1
or not isinstance(vsock.get("uds_path"), str) or not vsock["uds_path"]):
raise FirecrackerError("Firecracker configuration has no valid vsock device")
self._request(
"PUT",
@@ -57,6 +62,7 @@ class FirecrackerClient:
self._request("PUT", "/boot-source", boot)
self._request("PUT", "/drives/rootfs", drives[0])
self._request("PUT", "/network-interfaces/eth0", interfaces[0])
self._request("PUT", "/vsock", vsock)
self._request("PUT", "/entropy", entropy)
self._request("PUT", "/actions", {"action_type": "InstanceStart"})
+3
View File
@@ -72,6 +72,9 @@ def build_config(
],
# Supply host-backed randomness before guest services such as sshd start.
"entropy": {},
# Each Firecracker instance has its own Unix-socket-backed vsock device;
# CIDs need not be allocated across these independent devices.
"vsock": {"guest_cid": 3, "uds_path": str(settings.guest_socket_path(vm.id))},
}
+17 -11
View File
@@ -124,7 +124,7 @@ class FirecrackerProcessManager:
) from error
socket_path = Path(vm.socket)
self._remove_socket(socket_path)
self._remove_runtime_sockets(vm)
log_path = Path(vm.log)
process: _PopenLike | None = None
@@ -275,14 +275,16 @@ class FirecrackerProcessManager:
if vm.pid is None:
if vm.spawn_pending or vm.operation == "spawning" or vm.process_start_time is not None:
return "unknown"
try:
Path(vm.socket).lstat()
except FileNotFoundError:
return "gone"
except OSError:
for socket_path in (Path(vm.socket), self._settings.guest_socket_path(vm.id)):
try:
socket_path.lstat()
except FileNotFoundError:
continue
except OSError:
return "unknown"
# An unowned socket may belong to a child whose identity was never saved.
return "unknown"
# An unowned socket may belong to a child whose identity was never saved.
return "unknown"
return "gone"
stat = self._process_stat(vm.pid)
if stat is None:
@@ -330,14 +332,14 @@ class FirecrackerProcessManager:
f"refusing to stop Firecracker PID {vm.pid}: process identity is ambiguous"
)
if observation == "gone":
self._remove_socket(Path(vm.socket))
self._remove_runtime_sockets(vm)
return
try:
pidfd = _open_pidfd(vm.pid)
except ProcessLookupError:
self._reap_child(vm.pid)
self._remove_socket(Path(vm.socket))
self._remove_runtime_sockets(vm)
return
except (OSError, NotImplementedError) as error:
raise FirecrackerError(
@@ -369,7 +371,7 @@ class FirecrackerProcessManager:
"process identity and socket retained"
)
self._reap_child(vm.pid)
self._remove_socket(Path(vm.socket))
self._remove_runtime_sockets(vm)
except (OSError, NotImplementedError) as error:
raise FirecrackerError(
f"could not safely stop Firecracker process {vm.pid}: {error}"
@@ -455,6 +457,10 @@ class FirecrackerProcessManager:
if process is not None and process.poll() is not None:
self._children.pop(pid, None)
def _remove_runtime_sockets(self, vm: VmRecord) -> None:
self._remove_socket(self._settings.guest_socket_path(vm.id))
self._remove_socket(Path(vm.socket))
@staticmethod
def _remove_socket(socket_path: Path) -> None:
try:
+342
View File
@@ -0,0 +1,342 @@
"""Standalone, stdlib-only guest execution agent (Python 3.10+).
This file is copied into guest images and must not import the host application.
The host also imports its wire-format helpers; importing it never starts a server.
"""
from __future__ import annotations
import json
import math
import os
import pwd
import re
import selectors
import signal
import socket
import stat
import struct
import subprocess
import sys
import threading
import time
import uuid
from pathlib import Path
PROTOCOL_VERSION = 1
AGENT_PORT = 1070
DEFAULT_TIMEOUT_SECONDS = 60.0
MAX_TIMEOUT_SECONDS = 3600.0
MAX_COMMAND_BYTES = 65536
MAX_OUTPUT_BYTES = 1024 * 1024 # Combined stdout and stderr, before UTF-8 decoding.
MAX_REQUEST_BYTES = 6 * MAX_COMMAND_BYTES + 4096
MAX_RESPONSE_BYTES = 6 * MAX_OUTPUT_BYTES + 4096
CONNECT_TIMEOUT_SECONDS = 5.0
CLEANUP_SECONDS = 5.0
MAX_CONCURRENT_COMMANDS = 8
CONFIG_PATH = Path("/etc/uvm-agent.json")
class ProtocolError(ValueError):
"""An invalid or oversized wire message; never include command text."""
class ExecutionFailure(Exception):
def __init__(self, message: str, *, outcome_unknown: bool = False) -> None:
super().__init__(message)
self.outcome_unknown = outcome_unknown
def validate_command(command: object, sudo: object, timeout_seconds: object) -> None:
if not isinstance(command, str) or not command.strip() or "\0" in command:
raise ProtocolError("command must be non-empty text without NUL characters")
try:
size = len(command.encode("utf-8"))
except UnicodeError as error:
raise ProtocolError("command must be valid UTF-8 text") from error
if size > MAX_COMMAND_BYTES:
raise ProtocolError(f"command exceeds {MAX_COMMAND_BYTES} UTF-8 bytes")
if type(sudo) is not bool:
raise ProtocolError("sudo must be a boolean")
try:
valid_timeout = (
type(timeout_seconds) in (int, float)
and math.isfinite(timeout_seconds)
and 0 < timeout_seconds <= MAX_TIMEOUT_SECONDS
)
except OverflowError:
valid_timeout = False
if not valid_timeout:
raise ProtocolError(f"timeout_seconds must be finite and greater than 0, up to {MAX_TIMEOUT_SECONDS:g}")
def _remaining(deadline: float) -> float:
remaining = deadline - time.monotonic()
if remaining <= 0:
raise TimeoutError("guest protocol deadline exceeded")
return remaining
def receive_exact(connection: socket.socket, size: int, deadline: float) -> bytes:
parts = bytearray()
while len(parts) < size:
connection.settimeout(_remaining(deadline))
chunk = connection.recv(size - len(parts))
if not chunk:
raise EOFError("guest connection closed")
parts.extend(chunk)
return bytes(parts)
def receive_frame(connection: socket.socket, limit: int, deadline: float) -> dict:
size, = struct.unpack("!I", receive_exact(connection, 4, deadline))
if not 0 < size <= limit:
raise ProtocolError("invalid guest message length")
try:
message = json.loads(receive_exact(connection, size, deadline))
except (ValueError, UnicodeError, RecursionError) as error:
raise ProtocolError("invalid guest JSON message") from error
if not isinstance(message, dict):
raise ProtocolError("guest message must be an object")
return message
def send_frame(connection: socket.socket, message: dict, limit: int, deadline: float) -> None:
payload = json.dumps(message, ensure_ascii=True, allow_nan=False, separators=(",", ":")).encode("ascii")
if len(payload) > limit:
raise ProtocolError("guest message exceeds its size limit")
connection.settimeout(_remaining(deadline))
connection.sendall(struct.pack("!I", len(payload)) + payload)
class JobScope:
"""A delegated guest cgroup, keeping detached descendants in the same job."""
def __init__(self) -> None:
entries = Path("/proc/self/cgroup").read_text(encoding="ascii").splitlines()
unified = next((entry[3:] for entry in entries if entry.startswith("0::")), None)
if unified is None or ".." in Path(unified).parts:
raise ExecutionFailure("guest execution requires a delegated cgroup v2 hierarchy")
parent = Path("/sys/fs/cgroup") / unified.lstrip("/")
self.path = parent / f"uvm-exec-{uuid.uuid4().hex}"
self.path.mkdir(mode=0o700)
if not (self.path / "cgroup.kill").is_file():
self.path.rmdir()
raise ExecutionFailure("guest execution requires cgroup.kill (Linux 5.14 or newer)")
def kill(self) -> None:
(self.path / "cgroup.kill").write_text("1", encoding="ascii")
def close(self) -> None:
# cgroup.events reports descendants as well as processes in this cgroup.
deadline = time.monotonic() + 1.0
while "populated 1" in (self.path / "cgroup.events").read_text(encoding="ascii"):
if time.monotonic() >= deadline:
raise ExecutionFailure("guest job cleanup could not be confirmed", outcome_unknown=True)
time.sleep(0.01)
for child in sorted(self.path.rglob("*"), key=lambda path: len(path.parts), reverse=True):
if child.is_dir():
child.rmdir()
self.path.rmdir()
def _signal_group(process: subprocess.Popen, number: int) -> None:
# Do not reap the group leader until after signaling: its PID pins the PGID.
try:
os.killpg(process.pid, number)
except ProcessLookupError:
pass
def _exited(process: subprocess.Popen) -> bool:
return os.waitid(os.P_PID, process.pid, os.WEXITED | os.WNOHANG | os.WNOWAIT) is not None
def _cleanup_job(process: subprocess.Popen | None, scope: JobScope) -> None:
failed = False
try:
scope.kill()
except (OSError, ExecutionFailure):
failed = True
if process is not None:
try:
_signal_group(process, signal.SIGKILL)
process.wait(timeout=1.0)
except (OSError, subprocess.TimeoutExpired):
failed = True
finally:
for name in ("stdout", "stderr"):
getattr(process, name).close()
try:
scope.close()
except (OSError, ExecutionFailure):
failed = True
if failed:
raise ExecutionFailure("guest job cleanup could not be confirmed", outcome_unknown=process is not None)
def run_command(command: str, username: str, timeout_seconds: float, *, scope_factory=JobScope) -> dict:
"""Run one non-interactive guest job, drain both pipes, and bound its lifetime."""
scope = scope_factory()
process = None
buffers = {"stdout": bytearray(), "stderr": bytearray()}
captured = 0
truncated = False
timed_out = False
try:
# The fresh child joins its scope BEFORE dropping credentials or executing
# user text. Avoid preexec_fn in the threaded agent.
process = subprocess.Popen(
[sys.executable, "-I", str(Path(__file__).resolve()), "--child", str(scope.path), username, command],
stdin=subprocess.DEVNULL, stdout=subprocess.PIPE, stderr=subprocess.PIPE,
start_new_session=True, env={"PATH": "/usr/sbin:/usr/bin:/sbin:/bin", "LANG": "C.UTF-8"},
)
deadline = time.monotonic() + timeout_seconds
terminating = None
killed = False
with selectors.DefaultSelector() as selector:
for name in buffers:
stream = getattr(process, name)
os.set_blocking(stream.fileno(), False)
selector.register(stream, selectors.EVENT_READ, name)
while True:
now = time.monotonic()
exited = _exited(process)
if terminating is None and (exited or now >= deadline):
timed_out = not exited
terminating = now
_signal_group(process, signal.SIGTERM if timed_out else signal.SIGKILL)
if terminating is not None:
if not killed and (exited or now >= terminating + 0.5):
scope.kill()
_signal_group(process, signal.SIGKILL)
killed = True
if killed and exited and not selector.get_map():
break
if now >= terminating + 3.0:
raise ExecutionFailure("guest job cleanup could not be confirmed", outcome_unknown=True)
for key, _events in selector.select(0.05):
chunk = os.read(key.fileobj.fileno(), 65536)
if not chunk:
selector.unregister(key.fileobj)
continue
keep = min(len(chunk), MAX_OUTPUT_BYTES - captured)
buffers[key.data].extend(chunk[:keep])
captured += keep
truncated |= keep < len(chunk)
# Finally below kills the scope and reaps the leader before returning.
except OSError as error:
raise ExecutionFailure("guest command execution failed", outcome_unknown=process is not None) from error
finally:
_cleanup_job(process, scope)
return {
"exit_code": None if timed_out else process.returncode,
"stdout": buffers["stdout"].decode("utf-8", errors="replace"),
"stderr": buffers["stderr"].decode("utf-8", errors="replace"),
"timed_out": timed_out,
"truncated": truncated,
}
def child_main(scope: Path, username: str, command: str) -> None:
"""Enter the job cgroup and account in a fresh interpreter, then replace it."""
(scope / "cgroup.procs").write_text(str(os.getpid()), encoding="ascii")
account = pwd.getpwnam(username)
if os.geteuid() == 0:
os.initgroups(account.pw_name, account.pw_gid)
os.setgid(account.pw_gid)
os.setuid(account.pw_uid)
elif (account.pw_uid, account.pw_gid) != (os.geteuid(), os.getegid()):
raise PermissionError("guest agent cannot assume the requested account")
os.chdir(account.pw_dir)
os.umask(0o022)
environment = {
"HOME": account.pw_dir, "USER": account.pw_name, "LOGNAME": account.pw_name,
"SHELL": "/bin/sh", "PATH": "/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin",
"LANG": "C.UTF-8",
}
os.execve("/bin/sh", ["sh", "-c", command], environment)
def serve_connection(connection: socket.socket, config: dict, *, runner=run_command) -> None:
with connection:
try:
deadline = time.monotonic() + CONNECT_TIMEOUT_SECONDS
send_frame(connection, {"version": PROTOCOL_VERSION, **config}, 4096, deadline)
request = receive_frame(connection, MAX_REQUEST_BYTES, deadline)
if set(request) != {"version", "command", "sudo", "timeout_seconds"}:
raise ProtocolError("invalid execution request fields")
if type(request["version"]) is not int or request["version"] != PROTOCOL_VERSION:
raise ProtocolError("unsupported guest protocol version")
validate_command(request["command"], request["sudo"], request["timeout_seconds"])
username = "root" if request["sudo"] else config["username"]
result = runner(request["command"], username, request["timeout_seconds"])
send_frame(connection, result, MAX_RESPONSE_BYTES, time.monotonic() + CONNECT_TIMEOUT_SECONDS)
except (ProtocolError, ExecutionFailure, OSError, KeyError) as error:
# Never log request text, guest output, or credentials to the console.
try:
send_frame(connection, {
"error": "guest execution failed: " + str(error),
"outcome_unknown": isinstance(error, ExecutionFailure) and error.outcome_unknown,
}, 4096, time.monotonic() + CONNECT_TIMEOUT_SECONDS)
except (OSError, ProtocolError):
pass
except EOFError:
pass
def load_config(path: Path = CONFIG_PATH) -> dict:
descriptor = os.open(path, os.O_RDONLY | os.O_NOFOLLOW)
with os.fdopen(descriptor, encoding="utf-8") as source:
metadata = os.fstat(source.fileno())
if (not stat.S_ISREG(metadata.st_mode) or metadata.st_uid != 0
or metadata.st_mode & 0o077 or metadata.st_size > 4096):
raise ValueError("guest agent configuration must be a root-owned private regular file")
config = json.load(source)
if (not isinstance(config, dict) or set(config) != {"vm_id", "username"}
or not isinstance(config["vm_id"], str)
or re.fullmatch(r"[A-Za-z0-9][A-Za-z0-9_.-]{0,80}", config["vm_id"]) is None
or not isinstance(config["username"], str)
or re.fullmatch(r"[a-z_][a-z0-9_-]{0,31}\$?", config["username"]) is None):
raise ValueError("invalid guest agent configuration")
pwd.getpwnam(config["username"])
return config
def main() -> None:
if len(sys.argv) == 5 and sys.argv[1] == "--child":
try:
child_main(Path(sys.argv[2]), sys.argv[3], sys.argv[4])
except (OSError, KeyError) as error:
print(f"uvm-agent: could not launch guest command: {error}", file=sys.stderr)
sys.exit(126)
return
if len(sys.argv) != 1 or os.geteuid() != 0:
raise SystemExit("uvm-agent must run as a root systemd service")
config = load_config()
slots = threading.BoundedSemaphore(MAX_CONCURRENT_COMMANDS)
def handle(connection: socket.socket) -> None:
try:
serve_connection(connection, config)
finally:
slots.release()
with socket.socket(socket.AF_VSOCK, socket.SOCK_STREAM) as listener:
listener.bind((socket.VMADDR_CID_ANY, AGENT_PORT))
listener.listen(MAX_CONCURRENT_COMMANDS)
while True:
connection, peer = listener.accept()
# Guest-local callers must not use the root agent to elevate themselves.
if peer[0] != socket.VMADDR_CID_HOST:
connection.close()
continue
if not slots.acquire(blocking=False):
connection.close()
continue
threading.Thread(target=handle, args=(connection,), daemon=True).start()
if __name__ == "__main__":
main()
+114
View File
@@ -0,0 +1,114 @@
"""Host-side, one-shot guest execution over Firecracker's vsock Unix socket."""
from __future__ import annotations
import socket
import stat
import struct
import time
from dataclasses import dataclass
from pathlib import Path
from .errors import GuestAgentUnavailableError, GuestExecutionError, ValidationError
from .guest_agent import (
AGENT_PORT, CLEANUP_SECONDS, CONNECT_TIMEOUT_SECONDS, MAX_OUTPUT_BYTES,
MAX_REQUEST_BYTES, MAX_RESPONSE_BYTES, PROTOCOL_VERSION, ProtocolError,
receive_exact, receive_frame, send_frame, validate_command,
)
@dataclass(frozen=True, slots=True)
class ExecResult:
exit_code: int | None
stdout: str
stderr: str
timed_out: bool
truncated: bool
def validate_exec(command: object, sudo: object, timeout_seconds: object) -> None:
try:
validate_command(command, sudo, timeout_seconds)
except ProtocolError as error:
raise ValidationError(str(error)) from error
class GuestSession:
"""A connected session pins the selected VMM; never reconnect after submission."""
def __init__(self, connection: socket.socket) -> None:
self._connection = connection
@classmethod
def connect(cls, path: Path, vm_id: str, username: str, pid: int) -> GuestSession:
connection = None
try:
connection = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM)
metadata = path.lstat()
if not stat.S_ISSOCK(metadata.st_mode):
raise ProtocolError("guest socket is not a Unix socket")
deadline = time.monotonic() + CONNECT_TIMEOUT_SECONDS
connection.settimeout(CONNECT_TIMEOUT_SECONDS)
connection.connect(str(path))
peer_pid, _uid, _gid = struct.unpack("3i", connection.getsockopt(socket.SOL_SOCKET, socket.SO_PEERCRED, 12))
if peer_pid != pid:
raise ProtocolError("guest socket does not belong to the selected VM process")
connection.sendall(f"CONNECT {AGENT_PORT}\n".encode("ascii"))
acknowledgement = bytearray()
while not acknowledgement.endswith(b"\n") and len(acknowledgement) < 32:
acknowledgement.extend(receive_exact(connection, 1, deadline))
parts = bytes(acknowledgement).split()
if (not acknowledgement.endswith(b"\n") or len(parts) != 2
or parts[0] != b"OK" or not parts[1].isdigit()):
raise ProtocolError("invalid Firecracker vsock acknowledgement")
hello = receive_frame(connection, 4096, deadline)
if (set(hello) != {"version", "vm_id", "username"}
or type(hello["version"]) is not int or hello["version"] != PROTOCOL_VERSION
or hello["vm_id"] != vm_id or hello["username"] != username):
raise ProtocolError("guest agent identity or protocol version does not match this VM")
return cls(connection)
except (OSError, EOFError, ProtocolError) as error:
if connection is not None:
connection.close()
raise GuestAgentUnavailableError(
"guest agent is unavailable or not ready; install it with POST /vms/{vm_id}/agent/install "
"while stopped, then start the VM (vsock-capable kernel required)"
) from error
def execute(self, command: str, *, sudo: bool, timeout_seconds: float) -> ExecResult:
validate_exec(command, sudo, timeout_seconds)
try:
send_frame(self._connection, {
"version": PROTOCOL_VERSION, "command": command,
"sudo": sudo, "timeout_seconds": timeout_seconds,
}, MAX_REQUEST_BYTES, time.monotonic() + CONNECT_TIMEOUT_SECONDS)
result = receive_frame(
self._connection, MAX_RESPONSE_BYTES,
time.monotonic() + timeout_seconds + CLEANUP_SECONDS + CONNECT_TIMEOUT_SECONDS,
)
if "error" in result:
if (set(result) != {"error", "outcome_unknown"} or not isinstance(result["error"], str)
or len(result["error"]) > 4096 or type(result["outcome_unknown"]) is not bool):
raise ProtocolError("invalid guest failure response")
result["error"].encode("utf-8") # Reject invalid Unicode before rendering an HTTP error.
raise GuestExecutionError(result["error"], outcome_unknown=result["outcome_unknown"])
if set(result) != {"exit_code", "stdout", "stderr", "timed_out", "truncated"}:
raise ProtocolError("invalid guest result fields")
if (type(result["timed_out"]) is not bool or type(result["truncated"]) is not bool
or not isinstance(result["stdout"], str) or not isinstance(result["stderr"], str)
or (result["timed_out"] and result["exit_code"] is not None)
or (not result["timed_out"] and (
type(result["exit_code"]) is not int or not -64 <= result["exit_code"] <= 255))):
raise ProtocolError("invalid guest result types")
if sum(len(result[name].encode("utf-8")) for name in ("stdout", "stderr")) > 3 * MAX_OUTPUT_BYTES:
raise ProtocolError("guest output exceeds its limit")
return ExecResult(**result)
except (OSError, EOFError, ProtocolError, UnicodeError) as error:
raise GuestExecutionError(
"guest execution connection was lost or returned an invalid result; "
"the command may have run and will not be retried",
outcome_unknown=True,
) from error
def close(self) -> None:
self._connection.close()
+165
View File
@@ -2,6 +2,8 @@
from __future__ import annotations
import hashlib
import json
import os
import re
import stat
@@ -11,6 +13,7 @@ from dataclasses import dataclass
from ipaddress import IPv4Address
from pathlib import Path
from . import guest_agent
from .config import MAX_STORAGE_MIB, MIN_STORAGE_MIB, Settings
from .errors import UvmError, ValidationError
from .integrity import load_manifest, verify_file
@@ -46,6 +49,54 @@ _LEGACY_GUEST_NETWORK_TEMPLATE = GUEST_NETWORK_TEMPLATE.replace(
"KeepConfiguration=yes", "KeepConfiguration=static",
)
GUEST_AGENT_SERVICE = """\
[Unit]
Description=UVM guest command execution agent
After=local-fs.target network.target
ConditionPathExists=/etc/uvm-agent.json
StartLimitIntervalSec=0
[Service]
Type=simple
ExecStartPre=/bin/sh /usr/lib/uvm/prepare-agent.sh
ExecStart=/usr/bin/python3 -I /usr/lib/uvm/agent.py
Restart=on-failure
RestartSec=5
TimeoutStartSec=600
TimeoutStopSec=5
KillMode=control-group
Delegate=yes
UMask=0077
[Install]
WantedBy=multi-user.target
"""
# Runs only INSIDE the VM. Old Jammy guests can acquire the runtime without SSH
# or executing a retained guest's binaries in a privileged host chroot.
GUEST_AGENT_PREPARE = """\
set -eu
check_python() {
/usr/bin/python3 -I -c 'import sys, json, math, pwd, selectors, socket, struct, subprocess, threading, uuid; assert sys.version_info >= (3, 10)' 2>/dev/null
}
if check_python; then
exit 0
fi
export DEBIAN_FRONTEND=noninteractive LC_ALL=C
/usr/bin/apt-get -o DPkg::Lock::Timeout=300 -o APT::Update::Error-Mode=any update
/usr/bin/apt-get -o DPkg::Lock::Timeout=300 -y --no-install-recommends install python3
check_python
"""
def guest_agent_files() -> dict[str, bytes]:
"""The same standalone payload is used by the builder and offline upgrades."""
return {
"/usr/lib/uvm/agent.py": Path(guest_agent.__file__).read_bytes(),
"/usr/lib/uvm/prepare-agent.sh": GUEST_AGENT_PREPARE.encode("utf-8"),
"/etc/systemd/system/uvm-agent.service": GUEST_AGENT_SERVICE.encode("utf-8"),
}
@dataclass(frozen=True, slots=True)
class GuestAssets:
@@ -232,6 +283,7 @@ class ImageStore:
)
if marker is None or marker.path.read_text(encoding="utf-8") not in {
"uvm-ubuntu-jammy-v1\n", "uvm-ubuntu-jammy-v2\n", "uvm-ubuntu-jammy-v3\n",
"uvm-ubuntu-jammy-v4\n",
}:
return None
profile = self._read_guest_file(
@@ -246,6 +298,119 @@ class ImageStore:
return None
return profile
def provision_guest_agent(self, disk: Path, vm_id: str, username: str, *, install: bool = False) -> bool:
"""Configure new templates, or explicitly install on a proven-offline disk.
The lifecycle lock and confirmed VMM exit are required. A checked sparse
copy preserves the original disk on failure and makes retries atomic.
"""
parse_username(username)
if re.fullmatch(r"[A-Za-z0-9][A-Za-z0-9_.-]{0,80}", vm_id) is None:
raise ValidationError("invalid VM ID for guest agent configuration")
try:
metadata = disk.lstat()
if (not stat.S_ISREG(metadata.st_mode) or metadata.st_nlink != 1
or disk.resolve() == self._settings.rootfs_image.resolve()):
raise UvmError("private VM disk is unsafe for guest agent installation")
with tempfile.TemporaryDirectory(prefix="uvm-guest-agent-") as name:
files = Path(name)
if not install:
marker = self._read_guest_file(disk, "/etc/uvm-image", files / "marker",
required=False, regular_only=True)
if marker is None or marker.path.read_text(encoding="utf-8") != "uvm-ubuntu-jammy-v4\n":
return False
descriptor, candidate = tempfile.mkstemp(prefix=".rootfs-agent-", suffix=".ext4", dir=disk.parent)
os.close(descriptor)
temporary = Path(candidate)
try:
self._copy_sparse(disk, temporary)
temporary.chmod(0o600)
checked = self._runner.run(("e2fsck", "-f", "-p", temporary), check=False, capture=True)
if checked.returncode not in (0, 1):
raise UvmError("filesystem check failed before guest agent installation")
# All ownership/content checks happen AFTER journal replay.
self._install_agent_files(temporary, files, vm_id, username)
self._runner.run(("e2fsck", "-f", "-n", temporary), capture=True)
with temporary.open("rb") as image:
os.fsync(image.fileno())
os.replace(temporary, disk)
directory = os.open(disk.parent, os.O_RDONLY)
try:
os.fsync(directory)
finally:
os.close(directory)
finally:
temporary.unlink(missing_ok=True)
return True
except (OSError, UnicodeError) as error:
raise UvmError(f"could not provision guest agent: {error}") from error
def _install_agent_files(self, disk: Path, files: Path, vm_id: str, username: str) -> None:
for binary in ("/usr/lib/systemd/systemd", "/usr/bin/apt-get"):
result = self._runner.run(("debugfs", "-R", f"stat {binary}", disk), capture=True)
if re.search(r"Type:\s+regular\b", result.stdout) is None:
raise UvmError("guest agent installation requires a systemd/apt image (Ubuntu 22.04 or newer)")
for directory in ("/etc", "/usr", "/usr/lib", "/usr/lib/uvm", "/etc/systemd",
"/etc/systemd/system", "/etc/systemd/system/multi-user.target.wants"):
self._agent_directory(disk, directory)
payloads = guest_agent_files()
payloads["/etc/uvm-agent.json"] = (json.dumps({"vm_id": vm_id, "username": username}) + "\n").encode("utf-8")
manifest_path = "/etc/uvm-agent-files.json"
previous = self._agent_file(disk, manifest_path, files / "manifest")
old_hashes = {}
if previous is not None:
try:
old_hashes = json.loads(previous.path.read_bytes())
except (ValueError, UnicodeError) as error:
raise UvmError("invalid guest agent ownership manifest") from error
if (not isinstance(old_hashes, dict) or set(old_hashes) - set(payloads)
or any(not isinstance(value, str) or re.fullmatch(r"[0-9a-f]{64}", value) is None
for value in old_hashes.values())):
raise UvmError("invalid guest agent ownership manifest")
hashes = {path: hashlib.sha256(content).hexdigest() for path, content in payloads.items()}
for index, (path, content) in enumerate(payloads.items()):
existing = self._agent_file(disk, path, files / f"existing-{index}")
if existing is not None:
digest = hashlib.sha256(existing.path.read_bytes()).hexdigest()
if digest not in {hashes[path], old_hashes.get(path)}:
raise UvmError(f"refusing to overwrite a customized guest agent file: {path}")
source = files / f"payload-{index}"
source.write_bytes(content)
self._write_guest_file(disk, path, _GuestFile(source, 0o600 if path.endswith(".json") else 0o644, 0, 0))
source = files / "new-manifest"
source.write_text(json.dumps(hashes, sort_keys=True) + "\n", encoding="utf-8")
self._write_guest_file(disk, manifest_path, _GuestFile(source, 0o600, 0, 0))
link = "/etc/systemd/system/multi-user.target.wants/uvm-agent.service"
target = "/etc/systemd/system/uvm-agent.service"
result = self._runner.run(("debugfs", "-R", f"stat {link}", disk), capture=True)
if "file not found" in (result.stdout + result.stderr).lower():
self._runner.run(("debugfs", "-w", "-R", f"symlink {link} {target}", disk), capture=True)
result = self._runner.run(("debugfs", "-R", f"stat {link}", disk), capture=True)
if (re.search(r"Type:\s+symlink\b", result.stdout) is None
or f'Fast link dest: "{target}"' not in result.stdout):
raise UvmError("could not enable guest agent without replacing a customized service link")
def _agent_directory(self, disk: Path, path: str) -> None:
result = self._runner.run(("debugfs", "-R", f"stat {path}", disk), capture=True)
if "file not found" in (result.stdout + result.stderr).lower():
self._create_guest_directory(disk, path, 0o755, 0, 0)
return
mode, owner = _MODE_PATTERN.search(result.stdout), _OWNER_PATTERN.search(result.stdout)
if (re.search(r"Type:\s+directory\b", result.stdout) is None or mode is None or owner is None
or int(owner.group(1)) != 0 or int(mode.group(1), 8) & 0o022):
raise UvmError(f"unsafe guest agent directory: {path}")
def _agent_file(self, disk: Path, path: str, destination: Path) -> _GuestFile | None:
result = self._runner.run(("debugfs", "-R", f"stat {path}", disk), capture=True)
if "file not found" in (result.stdout + result.stderr).lower():
return None
if re.search(r"Type:\s+regular\b", result.stdout) is None:
raise UvmError(f"unsafe guest agent file: {path}")
source = self._read_guest_file(disk, path, destination)
if (source.uid != 0 or source.mode & 0o022 or source.links != 1 or source.has_extended_attributes):
raise UvmError(f"unsafe guest agent file: {path}")
return source
@staticmethod
def _validate_storage(storage_mib: int) -> None:
if type(storage_mib) is not int or not MIN_STORAGE_MIB <= storage_mib <= MAX_STORAGE_MIB:
+13 -5
View File
@@ -14,17 +14,17 @@ from pathlib import Path
from .config import Settings
from .errors import UvmError
from .images import GUEST_NETWORK_TEMPLATE, GuestAssets
from .images import GUEST_NETWORK_TEMPLATE, GuestAssets, guest_agent_files
from .integrity import load_manifest, require_checksum, sha256_file, verify_file, write_manifest
from .state import StateStore
from .system import CommandRunner, check_kvm, ensure_data_directories, require_root
_UBUNTU_RELEASE = "jammy"
_ROOTFS_RECIPE = "uvm-ubuntu-jammy-v3\n"
_ROOTFS_RECIPE = "uvm-ubuntu-jammy-v4\n"
_GUEST_PACKAGES = (
"ca-certificates", "openssh-server", "sudo", "systemd-sysv", "udev",
"dbus", "libpam-systemd", "iproute2", "iputils-ping", "curl",
"dbus", "libpam-systemd", "iproute2", "iputils-ping", "curl", "python3",
)
# Keep the image mountable by the older Firecracker quickstart guest kernels.
_EXT4_FEATURES = (
@@ -397,7 +397,14 @@ class Installer:
machine_id.symlink_to("../../../etc/machine-id")
for key in (root / "etc/ssh").glob("ssh_host_*_key*"):
key.unlink()
self._runner.run(("systemctl", "--root", root, "enable", "ssh.service", "systemd-networkd.service"), capture=True)
for guest_path, contents in guest_agent_files().items():
path = root / guest_path.lstrip("/")
path.parent.mkdir(parents=True, exist_ok=True)
path.parent.chmod(0o755)
path.write_bytes(contents)
path.chmod(0o644)
self._runner.run(("systemctl", "--root", root, "enable", "ssh.service", "systemd-networkd.service",
"uvm-agent.service"), capture=True)
self._runner.run(("systemctl", "--root", root, "set-default", "multi-user.target"), capture=True)
@staticmethod
@@ -406,6 +413,7 @@ class Installer:
"etc/passwd", "etc/shadow", "etc/group", "etc/sudoers", "etc/ssh/sshd_config",
"usr/bin/sudo", "usr/bin/apt-get", "usr/sbin/sshd", "usr/lib/systemd/systemd",
"var/lib/dpkg/status", "var/lib/dpkg/info/apt.list", "var/lib/dpkg/info/sudo.list",
"usr/bin/python3", "usr/lib/uvm/agent.py", "etc/systemd/system/uvm-agent.service",
):
if not (root / relative).is_file():
raise UvmError(f"built Ubuntu rootfs is missing /{relative}")
@@ -414,7 +422,7 @@ class Installer:
fields = dict(line.split(": ", 1) for line in paragraph.splitlines() if ": " in line)
if fields.get("Status") == "install ok installed":
installed.add(fields.get("Package"))
if not {"apt", "sudo", "openssh-server", "systemd-sysv"}.issubset(installed):
if not {"apt", "sudo", "openssh-server", "systemd-sysv", "python3"}.issubset(installed):
raise UvmError("built Ubuntu rootfs has incomplete apt/dpkg package metadata")
if not (root / "usr/bin/sudo").stat().st_mode & 0o4000:
raise UvmError("built Ubuntu rootfs sudo is missing its setuid permission")
+42
View File
@@ -17,6 +17,8 @@ from .errors import ConflictError, NotFoundError, UvmError, ValidationError
from .firecracker.api import FirecrackerClient
from .firecracker.config import build_config, write_config
from .firecracker.process import FirecrackerProcessManager, ProcessInfo
from .guest_agent import DEFAULT_TIMEOUT_SECONDS
from .guest_exec import ExecResult, GuestSession, validate_exec
from .images import ImageStore
from .integrity import load_manifest, verify_file
from .known_hosts import KnownHostsManager
@@ -58,6 +60,7 @@ class LifecycleService:
network: NetworkManager, process: FirecrackerProcessManager, *,
known_hosts: KnownHostsManager,
client_factory: Callable[[Path, float], FirecrackerClient] = FirecrackerClient,
guest_connector: Callable[[Path, str, str, int], GuestSession] = GuestSession.connect,
clock: Callable[[], float] = time.time,
) -> None:
self._settings = settings
@@ -67,6 +70,7 @@ class LifecycleService:
self._process = process
self._known_hosts = known_hosts
self._client_factory = client_factory
self._guest_connector = guest_connector
self._clock = clock
def create(self, spec: VmSpec) -> VmRecord:
@@ -90,6 +94,7 @@ class LifecycleService:
disk = self._images.create_vm_disk(assets.rootfs, Path(vm.disk), storage_mib=spec.storage_mib)
self._images.provision_credentials(disk, vm.username, vm.password)
self._images.configure_guest_network(disk, vm.guest_ip)
self._images.provision_guest_agent(disk, vm.id, vm.username)
vm.disk_ready = True
self._save(vm)
self._boot(vm, assets.kernel)
@@ -494,6 +499,41 @@ class LifecycleService:
def find_for_ssh(self, identifier: str) -> VmRecord:
return self._state_store.find_by_id_or_ip(identifier)
def execute(
self, vm_id: str, command: str, *, sudo: bool = False,
timeout_seconds: float = DEFAULT_TIMEOUT_SECONDS,
) -> ExecResult:
"""Pin a running VM's guest connection under the lock, then wait unlocked."""
require_root()
validate_exec(command, sudo, timeout_seconds)
with self._state_store.operation_lock():
vm = self._load_for_operation(vm_id, exact_id=True)
self._not_deleting(vm)
if (vm.status != "running" or vm.operation is not None or vm.pending_resize is not None
or self._process.observe(vm) != "alive"):
raise ConflictError("guest execution requires a running VM with no pending lifecycle operation")
session = self._guest_connector(self._settings.guest_socket_path(vm.id), vm.id, vm.username, vm.pid)
try:
# This session stays attached to that VMM even if the VM is stopped,
# deleted, or restarted concurrently. Never reconnect/retry a command.
return session.execute(command, sudo=sudo, timeout_seconds=timeout_seconds)
finally:
session.close()
def install_guest_agent(self, vm_id: str) -> VmRecord:
"""Explicitly install/upgrade the agent on a cleanly stopped private disk."""
require_root()
with self._state_store.operation_lock():
vm = self._load_for_operation(vm_id, exact_id=True)
self._not_deleting(vm)
if (vm.status != "stopped" or vm.operation is not None or vm.pending_resize is not None
or self._process.observe(vm) != "gone"):
raise ConflictError("stop the VM and finish pending operations before installing the guest agent")
if not vm.disk_ready or not vm.disk:
raise ConflictError("guest agent installation requires a completely provisioned private disk")
self._images.provision_guest_agent(Path(vm.disk), vm.id, vm.username, install=True)
return vm
def log_path(self, identifier: str) -> Path:
vm = self._state_store.find_by_id_or_ip(identifier)
self._validate_paths(vm)
@@ -566,6 +606,8 @@ class LifecycleService:
directory = self._settings.vm_dir(vm.id).absolute()
if directory.is_symlink():
raise ConflictError("VM directory must not be a symbolic link")
if self._settings.guest_socket_path(vm.id).is_symlink():
raise ConflictError("unsafe guest vsock path")
for field, filename in (("socket", "firecracker.sock"), ("config", "config.json"),
("log", "firecracker.log"), ("disk", "rootfs.ext4")):
path = getattr(vm, field)
+29
View File
@@ -9,8 +9,11 @@ from ..api_models import (
PortSyncResponse,
PortUpdateRequest,
VmActionResultResponse,
VmAgentInstallResponse,
VmBulkActionRequest,
VmCreateRequest,
VmExecRequest,
VmExecResponse,
VmLogResponse,
VmResponse,
VmResizeRequest,
@@ -202,6 +205,32 @@ def sync_ports(
)
@router.post("/{vm_id}/exec", response_model=VmExecResponse)
def execute_in_vm(
vm_id: str,
request: VmExecRequest,
application: Application = Depends(get_authorized_application),
) -> VmExecResponse:
"""Execute a bounded, non-interactive guest command over vsock (exact VM ID)."""
result = application.lifecycle.execute(
vm_id, request.command, sudo=request.sudo, timeout_seconds=request.timeout_seconds,
)
return VmExecResponse(
exit_code=result.exit_code, stdout=result.stdout, stderr=result.stderr,
timed_out=result.timed_out, truncated=result.truncated,
)
@router.post("/{vm_id}/agent/install", response_model=VmAgentInstallResponse)
def install_guest_agent(
vm_id: str,
application: Application = Depends(get_authorized_application),
) -> VmAgentInstallResponse:
"""Install/upgrade the execution agent on a stopped VM's retained private disk."""
vm = application.lifecycle.install_guest_agent(vm_id)
return VmAgentInstallResponse(id=vm.id)
@router.get("/{identifier}/logs", response_model=VmLogResponse)
def get_vm_logs(
identifier: str,
+12 -2
View File
@@ -9,7 +9,10 @@ import ssl
from typing import Any
from .app import Application, build_application
from .errors import ConfigurationError, ConflictError, NotFoundError, UvmError, ValidationError
from .errors import (
ConfigurationError, ConflictError, GuestAgentUnavailableError, GuestExecutionError,
NotFoundError, UvmError, ValidationError,
)
def create_api(application: Application | None = None, *, host: str | None = None) -> Any:
@@ -49,9 +52,12 @@ def create_api(application: Application | None = None, *, host: str | None = Non
@api.exception_handler(UvmError)
async def handle_uvm_error(_request: Any, error: UvmError) -> Any:
detail: dict[str, Any] = {"message": str(error)}
if isinstance(error, GuestExecutionError):
detail["outcome_unknown"] = error.outcome_unknown
return JSONResponse(
status_code=_http_status_for(error),
content={"error": {"message": str(error)}},
content={"error": detail},
)
api.include_router(health_router)
@@ -130,6 +136,10 @@ def _validate_server_settings(
def _http_status_for(error: UvmError) -> int:
if isinstance(error, GuestAgentUnavailableError):
return 503
if isinstance(error, GuestExecutionError):
return 502
if isinstance(error, NotFoundError):
return 404
if isinstance(error, ConflictError):