feat: Add guest execution agent and vsock support

- Introduced a standalone guest execution agent to facilitate command execution within the guest VM over Firecracker's vsock.
- Enhanced FirecrackerClient to support vsock configuration, validating guest CID and UDS path.
- Updated configuration to include vsock settings by default.
- Refactored FirecrackerProcessManager to manage runtime sockets more effectively.
- Implemented guest agent installation and provisioning in the image store.
- Added API endpoints for executing commands in the VM and installing the guest agent.
- Improved error handling for guest execution failures and agent availability.
This commit is contained in:
2026-09-25 17:39:06 +05:30
parent dca7f78dae
commit b0bf99a6ee
25 changed files with 1893 additions and 30 deletions
+89
View File
@@ -0,0 +1,89 @@
"""Opt-in checks against an already-provisioned disposable VM on an isolated host."""
from __future__ import annotations
import json
import os
import unittest
from urllib.parse import quote
from urllib.request import Request, urlopen
@unittest.skipUnless(os.environ.get("UVM_EXEC_INTEGRATION") == "1", "set UVM_EXEC_INTEGRATION=1 on an isolated VM host")
class GuestExecIntegrationTests(unittest.TestCase):
@classmethod
def setUpClass(cls) -> None:
cls.api_url = os.environ.get("UVM_API_URL", "http://127.0.0.1:8000").rstrip("/")
cls.token = os.environ["UVM_API_TOKEN"]
cls.vm_id = os.environ["UVM_EXEC_VM_ID"]
cls.path = f"/vms/{quote(cls.vm_id, safe='')}"
vm = cls.request("GET", cls.path)
if (vm["id"] != cls.vm_id or vm["observed_status"] != "running"
or vm["username"] == "root" or vm["ports"] != []):
raise RuntimeError("provide a running disposable VM with a non-root login, installed agent, and empty TCP allowlist")
cls.username = vm["username"]
@classmethod
def request(cls, method, path, body=None):
request = Request(
cls.api_url + path, method=method,
data=json.dumps(body).encode("utf-8") if body is not None else None,
headers={"X-UVM-Token": cls.token, "Content-Type": "application/json"},
)
with urlopen(request, timeout=30) as response:
return json.load(response)
def execute(self, command, *, sudo=False, timeout=10):
return self.request("POST", self.path + "/exec", {
"command": command, "sudo": sudo, "timeout_seconds": timeout,
})
def test_user_and_root_execution_work_with_all_tcp_ports_closed(self) -> None:
ordinary = self.execute("id -u; whoami")
self.assertEqual(ordinary["exit_code"], 0)
uid, username = ordinary["stdout"].splitlines()
self.assertNotEqual(uid, "0")
self.assertEqual(username, self.username)
elevated = self.execute("id -u", sudo=True)
self.assertEqual(elevated["exit_code"], 0)
self.assertEqual(elevated["stdout"], "0\n")
def test_multiline_script_returns_output_and_nonzero_status(self) -> None:
result = self.execute("printf 'hello\\n'\nprintf 'problem\\n' >&2\nexit 7")
self.assertEqual(result, {
"exit_code": 7, "stdout": "hello\n", "stderr": "problem\n", "timed_out": False, "truncated": False,
})
def test_timeout_kills_a_descendant_that_started_a_new_session(self) -> None:
result = self.execute("""python3 - <<'PY'
import os, time
child = os.fork()
if child == 0:
os.setsid()
else:
print(child, flush=True)
time.sleep(30)
PY""", timeout=2)
self.assertTrue(result["timed_out"])
self.assertIsNone(result["exit_code"])
child = int(result["stdout"].strip())
checked = self.execute(f"""python3 - <<'PY'
from pathlib import Path
try:
state = Path('/proc/{child}/stat').read_text().rsplit(') ', 1)[1].split()[0]
except FileNotFoundError:
state = 'gone'
assert state in ('gone', 'Z'), state
PY""")
self.assertEqual(checked["exit_code"], 0, checked["stderr"])
def test_output_is_bounded_while_both_pipes_are_drained(self) -> None:
result = self.execute("""python3 - <<'PY'
import os
for _ in range(32):
os.write(1, b'o' * 65536)
os.write(2, b'e' * 65536)
PY""")
self.assertEqual(result["exit_code"], 0)
self.assertTrue(result["truncated"])
self.assertEqual(len(result["stdout"]) + len(result["stderr"]), 1024 * 1024)
+28
View File
@@ -78,6 +78,7 @@ class FirecrackerConfigTests(unittest.TestCase):
self.assertEqual(config["drives"][0]["path_on_host"], "/vm/rootfs.ext4")
self.assertIn("ip=10.42.0.2::10.42.0.1", config["boot-source"]["boot_args"])
self.assertNotIn("password", str(config))
self.assertEqual(config["vsock"], {"guest_cid": 3, "uds_path": "/var/lib/uvm/vms/vm-test/vsock.sock"})
def test_configures_firecracker_in_required_order_before_start(self) -> None:
vm = VmRecord(
@@ -107,6 +108,7 @@ class FirecrackerConfigTests(unittest.TestCase):
"/boot-source",
"/drives/rootfs",
"/network-interfaces/eth0",
"/vsock",
"/entropy",
"/actions",
],
@@ -124,6 +126,7 @@ class FirecrackerConfigTests(unittest.TestCase):
"drives": [{}],
"network-interfaces": [{}],
"entropy": {},
"vsock": {"guest_cid": 3, "uds_path": "/tmp/vsock.sock"},
}
client = RecordingFirecrackerClient()
@@ -147,6 +150,31 @@ class FirecrackerConfigTests(unittest.TestCase):
client.configure_and_start(config)
self.assertEqual(client.calls, [])
def test_missing_invalid_or_failed_vsock_configuration_prevents_guest_boot(self) -> None:
config = {
"machine-config": {"vcpu_count": 1, "mem_size_mib": 512}, "boot-source": {},
"drives": [{}], "network-interfaces": [{}], "entropy": {},
}
for vsock in (None, {}, {"guest_cid": True, "uds_path": "/tmp/vsock.sock"},
{"guest_cid": 2, "uds_path": "/tmp/vsock.sock"}, {"guest_cid": 3, "uds_path": ""}):
with self.subTest(vsock=vsock):
client = RecordingFirecrackerClient()
config["vsock"] = vsock
with self.assertRaisesRegex(FirecrackerError, "vsock"):
client.configure_and_start(config)
self.assertEqual(client.calls, [])
config["vsock"] = {"guest_cid": 3, "uds_path": "/tmp/vsock.sock"}
client = RecordingFirecrackerClient()
def request(method, path, body):
client.calls.append((method, path, body))
if path == "/vsock":
raise FirecrackerError("vsock unavailable")
with patch.object(client, "_request", side_effect=request), self.assertRaises(FirecrackerError):
client.configure_and_start(config)
self.assertNotIn("/actions", [path for _method, path, _body in client.calls])
def test_uses_the_configured_network_netmask_in_guest_boot_arguments(self) -> None:
settings = Settings(
network=IPv4Network("10.50.0.0/16"),
+199
View File
@@ -0,0 +1,199 @@
from __future__ import annotations
import hashlib
import json
import shutil
import sys
import tempfile
import unittest
from pathlib import Path
from unittest.mock import patch
from uvm.config import Settings
from uvm.errors import UvmError
from uvm.images import GUEST_AGENT_PREPARE, ImageStore, guest_agent_files
from uvm.system import CommandRunner
class GuestRuntimeBootstrapTests(unittest.TestCase):
def test_runtime_install_is_guest_only_skipped_when_ready_and_stops_on_apt_failure(self) -> None:
with tempfile.TemporaryDirectory() as temporary:
directory = Path(temporary)
python = directory / "python3"
apt = directory / "apt-get"
ready = directory / "ready"
python.write_text(
f"#!{sys.executable}\nimport os, sys\nfrom pathlib import Path\n"
"sys.exit(0 if Path(os.environ['READY']).exists() else 1)\n", encoding="utf-8",
)
apt.write_text(
f"#!{sys.executable}\nimport json, os, sys\nfrom pathlib import Path\n"
"print(json.dumps(sys.argv[1:]))\n"
"if sys.argv[-1] == 'update' and os.environ['FAIL_UPDATE'] == '1': sys.exit(100)\n"
"if sys.argv[-1] == 'python3': Path(os.environ['READY']).touch()\n", encoding="utf-8",
)
python.chmod(0o755)
apt.chmod(0o755)
script = GUEST_AGENT_PREPARE.replace("/usr/bin/python3", str(python)).replace("/usr/bin/apt-get", str(apt))
for installed, fail in ((True, False), (False, False), (False, True)):
with self.subTest(installed=installed, fail=fail):
ready.unlink(missing_ok=True)
if installed:
ready.touch()
result = CommandRunner(emit=None).run((
"env", f"READY={ready}", f"FAIL_UPDATE={int(fail)}", "/bin/sh", "-c", script,
), check=False, capture=True)
calls = [json.loads(line) for line in result.stdout.splitlines()]
self.assertEqual(result.returncode, 100 if fail else 0)
self.assertEqual(len(calls), 0 if installed else 1 if fail else 2)
if calls:
self.assertEqual(calls[0][-1], "update")
self.assertIn("APT::Update::Error-Mode=any", calls[0])
if not installed and not fail:
self.assertEqual(calls[-1][-2:], ["install", "python3"])
@unittest.skipUnless(all(shutil.which(tool) for tool in ("mkfs.ext4", "debugfs", "e2fsck", "cp")),
"e2fsprogs and coreutils are required for offline guest agent checks")
class GuestAgentImageTests(unittest.TestCase):
def setUp(self) -> None:
temporary = tempfile.TemporaryDirectory()
self.addCleanup(temporary.cleanup)
self.base = Path(temporary.name)
self.settings = Settings(base=self.base / "uvm")
self.settings.images_dir.mkdir(parents=True)
self.disk = self.base / "VM with spaces" / "rootfs.ext4"
self.disk.parent.mkdir()
self.runner = CommandRunner(emit=None)
self.images = ImageStore(self.settings, self.runner)
self.retained = {
"/etc/shadow": "worker:$6$retained-hash:1:0:99999:7:::\n",
"/etc/ssh/ssh_host_ed25519_key": "retained private SSH key\n",
"/home/worker/data": "retained application data\n",
}
def make_disk(self, marker="uvm-ubuntu-jammy-v3\n") -> None:
root = self.base / "tree"
payloads = {
**self.retained, "/etc/uvm-image": marker,
"/usr/lib/systemd/systemd": "systemd fixture", "/usr/bin/apt-get": "apt fixture",
}
for path, content in payloads.items():
target = root / path.lstrip("/")
target.parent.mkdir(parents=True, exist_ok=True)
target.write_text(content, encoding="utf-8")
target.chmod(0o644)
for directory in (root, *(path for path in root.rglob("*") if path.is_dir())):
directory.chmod(0o755)
with self.settings.rootfs_image.open("wb") as image:
image.truncate(32 * 1024 * 1024)
self.runner.run(("mkfs.ext4", "-q", "-F", "-b", "4096", "-d", root, self.settings.rootfs_image), capture=True)
# mkfs -d retains the creating user's ownership. Model the root-built
# guest while allowing this test itself to run without root.
paths = {"/"}
for path in payloads:
paths.add(path)
paths.update(str(parent) for parent in Path(path).parents)
commands = "\n".join(f"set_inode_field {path} {field} 0" for path in paths for field in ("uid", "gid"))
self.runner.run(("debugfs", "-w", self.settings.rootfs_image), input_text=commands + "\n", capture=True)
self.runner.run(("cp", "--sparse=always", self.settings.rootfs_image, self.disk), capture=True)
self.disk.chmod(0o600)
def guest(self, command: str, *, disk: Path | None = None) -> str:
return self.runner.run(("debugfs", "-R", command, disk or self.disk), capture=True).stdout
def snapshot(self) -> tuple[int, bytes]:
return self.disk.stat().st_ino, hashlib.sha256(self.disk.read_bytes()).digest()
def replace_guest_file(self, path: str, contents: str, *, disk: Path | None = None) -> None:
source = self.base / "edited"
source.write_text(contents, encoding="utf-8")
source.chmod(0o644)
self.runner.run(("debugfs", "-w", disk or self.disk), input_text=(
f"rm {path}\nwrite {source} {path}\nset_inode_field {path} uid 0\nset_inode_field {path} gid 0\n"
), capture=True)
def test_existing_guest_is_upgraded_atomically_with_private_identity_and_retained_data(self) -> None:
self.make_disk()
template = hashlib.sha256(self.settings.rootfs_image.read_bytes()).digest()
self.assertTrue(self.images.provision_guest_agent(self.disk, "vm-test", "worker", install=True))
self.assertEqual(json.loads(self.guest("cat /etc/uvm-agent.json")), {"vm_id": "vm-test", "username": "worker"})
self.assertRegex(self.guest("stat /etc/uvm-agent.json"), r"Mode:\s+0?600")
self.assertRegex(self.guest("stat /etc/uvm-agent.json"), r"User:\s+0\s+Group:\s+0")
for path, content in guest_agent_files().items():
self.assertEqual(self.guest(f"cat {path}"), content.decode("utf-8"))
self.assertIn('Fast link dest: "/etc/systemd/system/uvm-agent.service"',
self.guest("stat /etc/systemd/system/multi-user.target.wants/uvm-agent.service"))
self.assertEqual(self.guest("cat /etc/uvm-image"), "uvm-ubuntu-jammy-v3\n")
for path, content in self.retained.items():
self.assertEqual(self.guest(f"cat {path}"), content)
self.assertEqual(hashlib.sha256(self.settings.rootfs_image.read_bytes()).digest(), template)
self.assertEqual(self.disk.stat().st_mode & 0o777, 0o600)
self.runner.run(("e2fsck", "-f", "-n", self.disk), capture=True)
self.assertEqual(list(self.disk.parent.iterdir()), [self.disk])
self.images.provision_guest_agent(self.disk, "vm-test", "worker", install=True)
self.assertEqual(self.guest("cat /home/worker/data"), self.retained["/home/worker/data"])
def test_current_templates_are_configured_and_older_templates_require_explicit_install(self) -> None:
self.make_disk()
before = self.snapshot()
self.assertFalse(self.images.provision_guest_agent(self.disk, "vm-test", "worker"))
self.assertEqual(self.snapshot(), before)
self.replace_guest_file("/etc/uvm-image", "uvm-ubuntu-jammy-v4\n")
self.assertTrue(self.images.provision_guest_agent(self.disk, "vm-test", "worker"))
def test_agent_updates_use_recorded_hashes_but_preserve_guest_customizations(self) -> None:
self.make_disk()
self.images.provision_guest_agent(self.disk, "vm-test", "worker", install=True)
updated = guest_agent_files()
updated["/usr/lib/uvm/agent.py"] += b"\n# next packaged revision\n"
with patch("uvm.images.guest_agent_files", return_value=updated):
self.images.provision_guest_agent(self.disk, "vm-test", "worker", install=True)
self.assertTrue(self.guest("cat /usr/lib/uvm/agent.py").endswith("# next packaged revision\n"))
self.replace_guest_file("/usr/lib/uvm/agent.py", "guest customization\n")
before = self.snapshot()
with self.assertRaisesRegex(UvmError, "customized guest agent file"):
self.images.provision_guest_agent(self.disk, "vm-test", "worker", install=True)
self.assertEqual(self.snapshot(), before)
self.assertEqual(list(self.disk.parent.iterdir()), [self.disk])
def test_failure_during_provisioning_keeps_the_original_disk(self) -> None:
self.make_disk()
before = self.snapshot()
write = self.images._write_guest_file
def fail(candidate, path, source):
self.assertNotEqual(candidate, self.disk)
write(candidate, path, source)
raise UvmError("simulated write failure")
with patch.object(self.images, "_write_guest_file", side_effect=fail), self.assertRaisesRegex(UvmError, "write failure"):
self.images.provision_guest_agent(self.disk, "vm-test", "worker", install=True)
self.assertEqual(self.snapshot(), before)
self.assertEqual(list(self.disk.parent.iterdir()), [self.disk])
def test_guest_edits_revealed_by_journal_replay_are_checked_before_replacement(self) -> None:
self.make_disk()
self.images.provision_guest_agent(self.disk, "vm-test", "worker", install=True)
before = self.snapshot()
run = self.runner.run
def replay(command, **options):
result = run(command, **options)
if command[:3] == ("e2fsck", "-f", "-p"):
self.replace_guest_file("/usr/lib/uvm/agent.py", "journal-committed guest edit\n", disk=command[-1])
return result
with patch.object(self.runner, "run", side_effect=replay), self.assertRaisesRegex(UvmError, "customized"):
self.images.provision_guest_agent(self.disk, "vm-test", "worker", install=True)
self.assertEqual(self.snapshot(), before)
def test_unsafe_guest_directory_and_shared_disk_are_rejected(self) -> None:
self.make_disk()
with self.assertRaisesRegex(UvmError, "unsafe"):
self.images.provision_guest_agent(self.settings.rootfs_image, "vm-test", "worker", install=True)
self.runner.run(("debugfs", "-w", "-R", "set_inode_field /etc mode 040777", self.disk), capture=True)
before = self.snapshot()
with self.assertRaisesRegex(UvmError, "unsafe guest agent directory"):
self.images.provision_guest_agent(self.disk, "vm-test", "worker", install=True)
self.assertEqual(self.snapshot(), before)
+289
View File
@@ -0,0 +1,289 @@
from __future__ import annotations
import os
import pwd
import shlex
import socket
import struct
import sys
import tempfile
import threading
import time
import unittest
from contextlib import contextmanager
from pathlib import Path
from unittest.mock import Mock, patch
from uvm.errors import GuestAgentUnavailableError, GuestExecutionError, ValidationError
from uvm import guest_agent as agent
from uvm.guest_exec import GuestSession, validate_exec
class FakeScope:
"""Use real subprocesses/process groups without touching the host's cgroups."""
def __init__(self) -> None:
self.temporary = tempfile.TemporaryDirectory()
self.path = Path(self.temporary.name)
(self.path / "cgroup.procs").touch()
self.kills = 0
self.joined_pid = None
def kill(self) -> None:
self.kills += 1
self.joined_pid = (self.path / "cgroup.procs").read_text()
def close(self) -> None:
self.temporary.cleanup()
class GuestCommandTests(unittest.TestCase):
def setUp(self) -> None:
self.account = pwd.getpwuid(os.geteuid())
self.scope = FakeScope()
self.addCleanup(self.scope.temporary.cleanup)
def run_command(self, command, timeout=5):
return agent.run_command(command, self.account.pw_name, timeout, scope_factory=lambda: self.scope)
def test_multiline_script_has_separate_output_exit_status_and_no_stdin(self) -> None:
result = self.run_command("printf 'hello\\n'\nprintf 'problem\\n' >&2\nread input || exit 7")
self.assertEqual(result, {
"exit_code": 7, "stdout": "hello\n", "stderr": "problem\n", "timed_out": False, "truncated": False,
})
self.assertTrue(self.scope.joined_pid.isdigit())
self.assertGreater(self.scope.kills, 0)
def test_command_enters_scope_before_running_and_uses_account_home_and_environment(self) -> None:
with patch.dict(os.environ, {"UVM_API_TOKEN": "must-not-reach-guest-command"}):
result = self.run_command('printf "%s\\n" "$$" "$HOME" "$USER" "$UVM_API_TOKEN"; pwd; id -u')
self.assertEqual(result["exit_code"], 0)
self.assertEqual(result["stdout"].splitlines(), [
self.scope.joined_pid, self.account.pw_dir, self.account.pw_name, "", self.account.pw_dir, str(os.geteuid()),
])
def test_timeout_kills_the_shell_and_children_that_ignore_sigterm(self) -> None:
start = time.monotonic()
result = self.run_command("trap '' TERM; sleep 30 & printf '%s\\n' \"$!\"; wait", timeout=0.25)
self.assertTrue(result["timed_out"])
self.assertIsNone(result["exit_code"])
self.assertLess(time.monotonic() - start, 4)
child = int(result["stdout"])
try:
state = Path(f"/proc/{child}/stat").read_text().split(") ", 1)[1].split()[0]
except FileNotFoundError:
state = "gone"
self.assertIn(state, {"gone", "Z"})
def test_large_stdout_and_stderr_are_drained_without_unbounded_capture(self) -> None:
script = "import os; [(os.write(1, b'o'*65536), os.write(2, b'e'*65536)) for _ in range(32)]"
result = self.run_command(f"{shlex.quote(sys.executable)} -c {shlex.quote(script)}")
self.assertEqual(result["exit_code"], 0)
self.assertEqual(len(result["stdout"]) + len(result["stderr"]), agent.MAX_OUTPUT_BYTES)
self.assertTrue(result["truncated"])
self.assertFalse(result["timed_out"])
def test_non_utf8_output_is_replaced_without_losing_the_exit_code(self) -> None:
result = self.run_command("printf '\\377\\000'; exit 255")
self.assertEqual(result["stdout"], "\ufffd\0")
self.assertEqual(result["exit_code"], 255)
def test_cgroup_cleanup_failure_is_an_unknown_outcome_and_still_reaps_the_process(self) -> None:
with patch.object(self.scope, "kill", side_effect=OSError("cannot kill scope")):
with self.assertRaises(agent.ExecutionFailure) as raised:
self.run_command("exit 0")
self.assertTrue(raised.exception.outcome_unknown)
def test_privilege_drop_initializes_groups_before_uid_and_exec(self) -> None:
account = Mock(pw_name="worker", pw_uid=1001, pw_gid=1002, pw_dir="/home/worker")
events = []
with (
patch.object(agent.pwd, "getpwnam", return_value=account),
patch.object(agent.os, "geteuid", return_value=0),
patch.object(agent.os, "initgroups", side_effect=lambda *args: events.append(("groups", args))),
patch.object(agent.os, "setgid", side_effect=lambda value: events.append(("gid", value))),
patch.object(agent.os, "setuid", side_effect=lambda value: events.append(("uid", value))),
patch.object(agent.os, "chdir") as chdir,
patch.object(agent.os, "umask"),
patch.object(agent.os, "execve") as execute,
):
agent.child_main(self.scope.path, "worker", "printf x > file")
self.assertEqual(events, [("groups", ("worker", 1002)), ("gid", 1002), ("uid", 1001)])
chdir.assert_called_once_with("/home/worker")
self.assertEqual(execute.call_args.args[:2], ("/bin/sh", ["sh", "-c", "printf x > file"]))
self.assertEqual(execute.call_args.args[2]["HOME"], "/home/worker")
class GuestProtocolTests(unittest.TestCase):
def test_validates_byte_limits_types_finite_deadlines_and_text_before_transport(self) -> None:
for command, sudo, timeout in (
("", False, 1), (" \n", False, 1), ("a\0b", False, 1), ("\ud800", False, 1),
("x" * (agent.MAX_COMMAND_BYTES + 1), False, 1),
("\u00e9" * agent.MAX_COMMAND_BYTES, False, 1),
("id", "false", 1), ("id", 0, 1), ("id", False, True),
("id", False, 0), ("id", False, -1), ("id", False, float("nan")),
("id", False, float("inf")), ("id", False, 3601), ("id", False, "60"),
):
with self.subTest(command=command[:20], sudo=sudo, timeout=timeout), self.assertRaises(ValidationError):
validate_exec(command, sudo, timeout)
validate_exec("id", False, 0.1)
def test_oversized_frame_is_rejected_before_reading_a_body(self) -> None:
left, right = socket.socketpair()
with left, right:
left.sendall(struct.pack("!I", agent.MAX_REQUEST_BYTES + 1))
with self.assertRaises(agent.ProtocolError):
agent.receive_frame(right, agent.MAX_REQUEST_BYTES, time.monotonic() + 1)
def test_read_deadline_applies_to_the_whole_frame(self) -> None:
left, right = socket.socketpair()
with left, right:
left.sendall(struct.pack("!I", 5) + b"{")
start = time.monotonic()
with self.assertRaises(TimeoutError):
agent.receive_frame(right, 1024, time.monotonic() + 0.05)
self.assertLess(time.monotonic() - start, 0.5)
def test_agent_maps_sudo_to_root_and_rejects_bad_requests_without_launching(self) -> None:
config = {"vm_id": "vm-test", "username": "worker"}
for sudo in (False, True, "true"):
with self.subTest(sudo=sudo):
left, right = socket.socketpair()
result = {"exit_code": 0, "stdout": "", "stderr": "", "timed_out": False, "truncated": False}
runner = Mock(return_value=result)
thread = threading.Thread(target=agent.serve_connection, args=(right, config), kwargs={"runner": runner})
thread.start()
with left:
deadline = time.monotonic() + 2
self.assertEqual(agent.receive_frame(left, 4096, deadline), {"version": 1, **config})
agent.send_frame(left, {"version": 1, "command": "id", "sudo": sudo, "timeout_seconds": 2}, 4096, deadline)
reply = agent.receive_frame(left, 4096, deadline)
thread.join(2)
self.assertFalse(thread.is_alive())
if type(sudo) is bool:
self.assertEqual(reply, result)
runner.assert_called_once_with("id", "root" if sudo else "worker", 2)
else:
self.assertFalse(reply["outcome_unknown"])
runner.assert_not_called()
def test_guest_local_connections_are_rejected_before_a_worker_is_started(self) -> None:
connection = Mock()
listener = Mock()
listener.__enter__ = Mock(return_value=listener)
listener.__exit__ = Mock(return_value=False)
listener.accept.side_effect = [(connection, (3, 9999)), KeyboardInterrupt()]
with (
patch.object(agent.sys, "argv", ["agent.py"]), patch.object(agent.os, "geteuid", return_value=0),
patch.object(agent, "load_config", return_value={"vm_id": "vm-test", "username": "root"}),
patch.object(agent.socket, "socket", return_value=listener), patch.object(agent.threading, "Thread") as worker,
self.assertRaises(KeyboardInterrupt),
):
agent.main()
connection.close.assert_called_once()
worker.assert_not_called()
class GuestTransportTests(unittest.TestCase):
@contextmanager
def firecracker(self, handler):
with tempfile.TemporaryDirectory() as temporary:
path = Path(temporary) / "vsock.sock"
errors = []
with socket.socket(socket.AF_UNIX, socket.SOCK_STREAM) as listener:
listener.bind(str(path))
listener.listen(1)
listener.settimeout(3)
def serve():
try:
connection, _peer = listener.accept()
with connection:
handler(connection)
except (BrokenPipeError, EOFError, ConnectionResetError):
pass
except BaseException as error:
errors.append(error)
thread = threading.Thread(target=serve, daemon=True)
thread.start()
try:
yield path
finally:
thread.join(4)
self.assertFalse(thread.is_alive())
if errors:
raise errors[0]
def handshake(self, connection, **changes):
deadline = time.monotonic() + 2
self.assertEqual(agent.receive_exact(connection, len(b"CONNECT 1070\n"), deadline), b"CONNECT 1070\n")
# The acknowledgement and JSON frame can be coalesced on the wire.
connection.sendall(b"OK 1234\n")
agent.send_frame(connection, {"version": 1, "vm_id": "vm-test", "username": "worker", **changes}, 4096, deadline)
def test_round_trip_preserves_command_text_and_a_nonzero_exit(self) -> None:
command = "printf '%s\\n' 'quotes; $(literal)'\nexit 7"
expected = {"exit_code": 7, "stdout": "quotes; $(literal)\n", "stderr": "", "timed_out": False, "truncated": False}
def handle(connection):
self.handshake(connection)
request = agent.receive_frame(connection, agent.MAX_REQUEST_BYTES, time.monotonic() + 2)
self.assertEqual(request, {"version": 1, "command": command, "sudo": True, "timeout_seconds": 3})
agent.send_frame(connection, expected, agent.MAX_RESPONSE_BYTES, time.monotonic() + 2)
with self.firecracker(handle) as path:
session = GuestSession.connect(path, "vm-test", "worker", os.getpid())
try:
result = session.execute(command, sudo=True, timeout_seconds=3)
finally:
session.close()
self.assertEqual(result.exit_code, 7)
self.assertEqual(result.stdout, expected["stdout"])
def test_wrong_vm_or_process_identity_prevents_command_submission(self) -> None:
for changes, pid in (({"vm_id": "vm-other"}, os.getpid()), ({"version": 2}, os.getpid()),
({"username": "root"}, os.getpid()), ({}, os.getpid() + 100000)):
with self.subTest(changes=changes, pid=pid):
def handle(connection):
self.handshake(connection, **changes)
self.assertEqual(connection.recv(1), b"")
with self.firecracker(handle) as path, self.assertRaises(GuestAgentUnavailableError):
GuestSession.connect(path, "vm-test", "worker", pid)
def test_connection_loss_after_submission_has_unknown_outcome_without_retry(self) -> None:
submissions = []
def handle(connection):
self.handshake(connection)
submissions.append(agent.receive_frame(connection, agent.MAX_REQUEST_BYTES, time.monotonic() + 2))
with self.firecracker(handle) as path:
session = GuestSession.connect(path, "vm-test", "worker", os.getpid())
with self.assertRaises(GuestExecutionError) as raised:
try:
session.execute("touch example", sudo=False, timeout_seconds=1)
finally:
session.close()
self.assertTrue(raised.exception.outcome_unknown)
self.assertEqual(len(submissions), 1)
def test_malformed_result_is_not_reported_as_success(self) -> None:
for result in ({"exit_code": True, "stdout": "", "stderr": "", "timed_out": False, "truncated": False},
{"error": "bad", "outcome_unknown": "false"}, {"error": "\ud800", "outcome_unknown": False},
{"exit_code": 0}):
with self.subTest(result=result):
def handle(connection):
self.handshake(connection)
agent.receive_frame(connection, agent.MAX_REQUEST_BYTES, time.monotonic() + 2)
agent.send_frame(connection, result, agent.MAX_RESPONSE_BYTES, time.monotonic() + 2)
with self.firecracker(handle) as path:
session = GuestSession.connect(path, "vm-test", "worker", os.getpid())
with self.assertRaises(GuestExecutionError) as raised:
try:
session.execute("id", sudo=False, timeout_seconds=1)
finally:
session.close()
self.assertTrue(raised.exception.outcome_unknown)
+97
View File
@@ -0,0 +1,97 @@
from __future__ import annotations
from pathlib import Path
from unittest.mock import Mock
from tests.test_lifecycle import LifecycleTestCase
from uvm.errors import ConflictError, GuestAgentUnavailableError, GuestExecutionError, NotFoundError
from uvm.guest_exec import ExecResult
class GuestExecLifecycleTests(LifecycleTestCase):
def test_create_provisions_agent_before_the_disk_is_ready_or_the_guest_is_started(self) -> None:
def provision(disk, vm_id, username):
self.host.call("images.provision_agent", vm_id)
vm = self.store.get(vm_id)
self.assertFalse(vm.disk_ready)
self.assertEqual(self.process.observe(vm), "gone")
self.assertEqual(disk, Path(vm.disk))
self.assertEqual(username, "worker")
self.provision_agent.side_effect = provision
vm = self.create_vm(username="worker")
self.assert_call_order(vm, "images.provision_agent", "process.start")
def test_execution_pins_the_exact_vm_under_lock_and_waits_without_holding_it(self) -> None:
vm = self.create_vm(name="worker", ports=())
result = ExecResult(7, "output", "problem", False, False)
session = Mock()
def connect(path, vm_id, username, pid):
self.assertTrue(self.store.operation_active)
self.assertEqual((path, vm_id, username, pid), (self.settings.guest_socket_path(vm.id), vm.id, vm.username, vm.pid))
return session
def execute(command, **options):
self.assertFalse(self.store.operation_active)
self.assertFalse(self.store.transaction_active)
self.service.stop(vm.id)
return result
self.service._guest_connector = Mock(side_effect=connect)
session.execute.side_effect = execute
self.assertEqual(self.service.execute(vm.id, "exit 7", sudo=True, timeout_seconds=2), result)
session.execute.assert_called_once_with("exit 7", sudo=True, timeout_seconds=2)
session.close.assert_called_once()
def test_execution_rejects_aliases_stopped_unknown_and_pending_vms_before_connecting(self) -> None:
vm = self.create_vm(name="worker")
self.service._guest_connector = Mock()
for alias in (vm.name, vm.guest_ip):
with self.assertRaises(NotFoundError):
self.service.execute(alias, "id")
for changes in ({"status": "stopped"}, {"operation": "policy"}, {"operation": "delete"}):
self.persist(vm, **changes)
with self.assertRaises(ConflictError):
self.service.execute(vm.id, "id")
self.persist(vm, status="running", operation=None)
self.process.unknown.add(vm.id)
with self.assertRaises(ConflictError):
self.service.execute(vm.id, "id")
self.service._guest_connector.assert_not_called()
def test_unavailable_or_lost_agent_does_not_retry_or_mutate_vm_state(self) -> None:
vm = self.create_vm()
before = self.settings.state_path.read_bytes()
self.service._guest_connector = Mock(side_effect=GuestAgentUnavailableError("not ready"))
with self.assertRaises(GuestAgentUnavailableError):
self.service.execute(vm.id, "id")
self.service._guest_connector.assert_called_once()
session = Mock()
session.execute.side_effect = GuestExecutionError("connection lost", outcome_unknown=True)
self.service._guest_connector = Mock(return_value=session)
with self.assertRaises(GuestExecutionError):
self.service.execute(vm.id, "id")
session.execute.assert_called_once()
session.close.assert_called_once()
self.assertEqual(self.settings.state_path.read_bytes(), before)
def test_agent_upgrade_requires_a_stopped_proven_offline_disk_and_retains_identity(self) -> None:
vm = self.create_vm(username="worker")
self.provision_agent.reset_mock()
with self.assertRaises(ConflictError):
self.service.install_guest_agent(vm.id)
self.provision_agent.assert_not_called()
stopped = self.service.stop(vm.id)
before = stopped.to_dict()
def provision(disk, vm_id, username, *, install):
self.assertTrue(self.store.operation_active)
self.assertEqual(self.process.observe(self.store.get(vm_id)), "gone")
self.assertEqual((disk, username, install), (Path(vm.disk), "worker", True))
self.provision_agent.side_effect = provision
self.assertEqual(self.service.install_guest_agent(vm.id).to_dict(), before)
self.provision_agent.assert_called_once()
self.assertEqual(self.store.get(vm.id).to_dict(), before)
self.assertEqual(len(self.process.started), 1)
+5 -1
View File
@@ -216,13 +216,14 @@ class UbuntuBuildRunner:
return CommandResult(args, 100, stdout="Reading package lists...\n", stderr="E: unmet guest dependencies\n")
status = "\n\n".join(
f"Package: {package}\nStatus: install ok installed\nVersion: 1.0"
for package in ("apt", "sudo", "openssh-server", "systemd-sysv")
for package in ("apt", "sudo", "openssh-server", "systemd-sysv", "python3")
) + "\n"
files = {
"etc/sudoers": "root ALL=(ALL:ALL) ALL\n",
"etc/ssh/sshd_config": "UsePAM yes\n",
"etc/ssh/ssh_host_rsa_key": "template host key",
"usr/bin/sudo": "sudo binary",
"usr/bin/python3": "python binary",
"usr/sbin/sshd": "sshd binary", "usr/lib/systemd/systemd": "init binary",
"var/lib/dpkg/status": status,
"var/lib/dpkg/info/sudo.list": "/usr/bin/sudo\n",
@@ -319,6 +320,7 @@ class UbuntuRootfsBuildTests(unittest.TestCase):
self.assertLess(runner.calls.index(bootstrap), runner.calls.index(packages))
self.assertIn("sudo", packages)
self.assertIn("openssh-server", packages)
self.assertIn("python3", packages)
self.assertIn("--mount-proc", packages)
self.assertIn("--kill-child", packages)
self.assertEqual(settings.rootfs_image.stat().st_mode & 0o777, 0o600)
@@ -335,6 +337,8 @@ class UbuntuRootfsBuildTests(unittest.TestCase):
self.assertEqual(guest("cat /var/lib/dpkg/info/sudo.list").stdout, "/usr/bin/sudo\n")
self.assertRegex(guest("stat /usr/bin/sudo").stdout, r"Mode:\s+0?4755")
self.assertEqual(guest("cat /etc/uvm-image").stdout, _ROOTFS_RECIPE)
self.assertIn("socket.AF_VSOCK", guest("cat /usr/lib/uvm/agent.py").stdout)
self.assertIn("Delegate=yes", guest("cat /etc/systemd/system/uvm-agent.service").stdout)
self.assertIn("nameserver 1.1.1.1", guest("cat /etc/resolv.conf").stdout)
self.assertIn("KeepConfiguration=yes", guest("cat /etc/systemd/network/10-uvm.network").stdout)
self.assertIn("jammy-security main universe", guest("cat /etc/apt/sources.list").stdout)
+1
View File
@@ -310,6 +310,7 @@ class LifecycleTestCase(unittest.TestCase):
self.configure_network = self.enterContext(patch.object(
self.images, "configure_guest_network", side_effect=self.configure_guest_network,
))
self.provision_agent = self.enterContext(patch.object(self.images, "provision_guest_agent"))
self.copy_disk = self.enterContext(patch.object(
self.images, "create_vm_disk", wraps=self.images.create_vm_disk,
))
+26
View File
@@ -113,6 +113,28 @@ class ProcessTestCase(unittest.TestCase):
class FirecrackerProcessSafetyTests(ProcessTestCase):
def test_vsock_socket_is_removed_only_after_confirmed_process_exit(self) -> None:
path = self.manager._settings.guest_socket_path(self.vm.id)
path.parent.mkdir(parents=True)
path.touch()
self.vm.process_start_time = None
with self.assertRaises(FirecrackerError):
self.manager.terminate(self.vm)
self.assertTrue(path.exists())
self.vm.boot_id = OLD_BOOT_ID
self.manager.terminate(self.vm)
self.assertFalse(path.exists())
def test_unowned_vsock_socket_keeps_a_missing_pid_ambiguous(self) -> None:
path = self.manager._settings.guest_socket_path(self.vm.id)
path.parent.mkdir(parents=True)
path.touch()
self.vm.pid = self.vm.process_start_time = None
self.assertEqual(self.manager.observe(self.vm), "unknown")
with self.assertRaises(FirecrackerError):
self.manager.terminate(self.vm)
self.assertTrue(path.exists())
def test_observes_a_matching_identity_as_alive(self) -> None:
self.assertEqual(self.manager.observe(self.vm), "alive")
self.assertTrue(self.manager.is_alive(self.vm))
@@ -808,6 +830,10 @@ class FirecrackerLegacyAdoptionTests(ProcessTestCase):
def root_lstat(path: Path) -> os.stat_result:
metadata = list(real_lstat(path))
metadata[4] = 0
# Model protected system ancestors independently of the caller's
# TMPDIR permissions. Unsafe-path fixtures live inside self.directory.
if path in self.directory.parents:
metadata[0] &= ~0o022
return os.stat_result(metadata)
self.root_lstat = root_lstat
+93 -1
View File
@@ -12,7 +12,11 @@ from unittest.mock import Mock, patch
from uvm.cli import build_parser, main
from uvm.config import Settings
from uvm.domain import PortRule, VmRecord, VmResources, VmSpec
from uvm.errors import ConfigurationError, ConflictError, NotFoundError, UvmError, ValidationError
from uvm.errors import (
ConfigurationError, ConflictError, GuestAgentUnavailableError, GuestExecutionError,
NotFoundError, UvmError, ValidationError,
)
from uvm.guest_exec import ExecResult
from uvm.lifecycle import ListedVm, OperationResult
from uvm.server import _http_status_for, create_api, run_server
@@ -141,6 +145,8 @@ class ServerErrorTests(unittest.TestCase):
(ConfigurationError("invalid settings"), 422),
(UvmError("this operation needs root"), 403),
(UvmError("host operation failed"), 500),
(GuestAgentUnavailableError("not ready"), 503),
(GuestExecutionError("lost connection", outcome_unknown=True), 502),
):
with self.subTest(error=error):
self.assertEqual(_http_status_for(error), expected)
@@ -159,6 +165,7 @@ class FakeLifecycle:
def __init__(self) -> None:
self.last_spec: VmSpec | None = None
self.last_resize: dict | None = None
self.last_execution: dict | None = None
self.observed_status = "running"
self.policy_status = "enforced"
self.vm = VmRecord(
@@ -256,6 +263,19 @@ class FakeLifecycle:
self.inspect(identifier)
return SimpleNamespace(content=f"last {tail} log lines", truncated=False)
def execute(self, vm_id: str, command: str, *, sudo: bool, timeout_seconds: float) -> ExecResult:
if vm_id != self.vm.id:
raise NotFoundError(f"VM not found: {vm_id}")
self.last_execution = dict(vm_id=vm_id, command=command, sudo=sudo, timeout_seconds=timeout_seconds)
return ExecResult(7, "hello\n", "problem\n", False, False)
def install_guest_agent(self, vm_id: str) -> VmRecord:
if vm_id != self.vm.id:
raise NotFoundError(f"VM not found: {vm_id}")
if self.vm.status != "stopped":
raise ConflictError("stop the VM before installing the guest agent")
return self.vm
def diagnostics(self) -> list[dict]:
return [{"name": "kvm", "ok": True, "message": "available"}]
@@ -341,6 +361,8 @@ class ApiRouterTests(unittest.TestCase):
("DELETE", "/vms/test-vm", {}),
("POST", "/vms/test-vm/ports", {}),
("POST", "/vms/test-vm/ports/sync", {}),
("POST", "/vms/vm-test/exec", {"command": "id"}),
("POST", "/vms/vm-test/agent/install", {}),
("GET", "/diagnostics", {}),
("POST", "/exit", {}),
("POST", "/install", {}),
@@ -655,6 +677,74 @@ class ApiRouterTests(unittest.TestCase):
self.assertEqual(sync.json()["id"], "vm-test")
self.assertEqual(sync.json()["policy_status"], "inactive")
def test_exec_preserves_script_text_privileges_and_nonzero_exit_results(self) -> None:
command = "printf 'hello\\n'\nprintf 'problem\\n' >&2\nexit 7"
response = self.client.post("/vms/vm-test/exec", headers=self.headers,
json={"command": command, "sudo": True, "timeout_seconds": 12.5})
self.assertEqual(response.status_code, 200, response.text)
self.assertEqual(response.json(), {
"exit_code": 7, "stdout": "hello\n", "stderr": "problem\n", "timed_out": False, "truncated": False,
})
self.assertEqual(self.lifecycle.last_execution, {
"vm_id": "vm-test", "command": command, "sudo": True, "timeout_seconds": 12.5,
})
self.assertNotIn("guest-secret", response.text)
default = self.client.post("/vms/vm-test/exec", headers=self.headers, json={"command": "id"})
self.assertEqual(default.status_code, 200)
self.assertEqual(self.lifecycle.last_execution["sudo"], False)
self.assertEqual(self.lifecycle.last_execution["timeout_seconds"], 60)
def test_exec_rejects_malformed_requests_before_calling_the_service(self) -> None:
for payload in (
{}, {"command": ""}, {"command": " \n"}, {"command": "a\0b"}, {"command": 42},
{"command": "x" * 65537}, {"command": "\u00e9" * 40000},
{"command": "id", "sudo": "false"}, {"command": "id", "sudo": 1},
{"command": "id", "timeout_seconds": True}, {"command": "id", "timeout_seconds": "10"},
{"command": "id", "timeout_seconds": 0}, {"command": "id", "timeout_seconds": 3601},
{"command": "id", "username": "root"}, {"command": "id", "host_path": "/etc/passwd"},
):
with self.subTest(payload=str(payload)[:100]):
response = self.client.post("/vms/vm-test/exec", headers=self.headers, json=payload)
self.assertEqual(response.status_code, 422, response.text)
self.assertIsNone(self.lifecycle.last_execution)
def test_exec_errors_distinguish_vm_state_agent_readiness_and_unknown_outcomes(self) -> None:
for error, expected in ((NotFoundError("missing"), 404), (ConflictError("stopped"), 409),
(GuestAgentUnavailableError("not ready"), 503),
(GuestExecutionError("lost response", outcome_unknown=True), 502)):
with self.subTest(error=error), patch.object(self.lifecycle, "execute", side_effect=error):
response = self.client.post("/vms/vm-test/exec", headers=self.headers, json={"command": "id"})
self.assertEqual(response.status_code, expected)
if isinstance(error, GuestExecutionError):
self.assertTrue(response.json()["error"]["outcome_unknown"])
with patch.object(self.lifecycle, "execute", return_value=ExecResult(None, "partial", "", True, True)):
response = self.client.post("/vms/vm-test/exec", headers=self.headers, json={"command": "sleep 10"})
self.assertEqual(response.status_code, 200)
self.assertIsNone(response.json()["exit_code"])
self.assertTrue(response.json()["timed_out"])
self.assertTrue(response.json()["truncated"])
def test_exec_rejects_nonfinite_and_invalid_unicode_json_without_serialization_errors(self) -> None:
for body in ('{"command":"id","timeout_seconds":NaN}',
'{"command":"id","timeout_seconds":Infinity}',
'{"command":"\\ud800"}', '{"command":"id","sudo":NaN}'):
with self.subTest(body=body):
response = self.client.post("/vms/vm-test/exec", headers={**self.headers, "Content-Type": "application/json"},
content=body)
self.assertEqual(response.status_code, 422, response.text)
self.assertIsNone(self.lifecycle.last_execution)
def test_agent_install_requires_stopping_and_uses_the_existing_vm_disk(self) -> None:
blocked = self.client.post("/vms/vm-test/agent/install", headers=self.headers)
self.assertEqual(blocked.status_code, 409)
self.client.post("/vms/vm-test/stop", headers=self.headers)
with patch.object(self.lifecycle, "install_guest_agent", wraps=self.lifecycle.install_guest_agent) as install:
response = self.client.post("/vms/vm-test/agent/install", headers=self.headers)
self.assertEqual(response.status_code, 200)
self.assertEqual(response.json(), {"id": "vm-test", "status": "installed", "restart_required": True})
install.assert_called_once_with("vm-test")
self.assertEqual(self.lifecycle.vm.status, "stopped")
def test_bulk_action_resolves_targets_before_running_fixed_ids(self) -> None:
with (
patch.object(self.lifecycle, "resolve_targets", wraps=self.lifecycle.resolve_targets) as resolve,
@@ -786,6 +876,8 @@ class ApiRouterTests(unittest.TestCase):
("/vms/{vm_id}", "delete"),
("/vms/{vm_id}/ports", "post"),
("/vms/{vm_id}/ports/sync", "post"),
("/vms/{vm_id}/exec", "post"),
("/vms/{vm_id}/agent/install", "post"),
("/diagnostics", "get"),
},
)