kstyagi/nest
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity
Files
a4e9cefbc3021dc0fdcb33b7915deb506f59e7db
nest/tests/test_process.py
T
kstyagi b0bf99a6ee feat: Add guest execution agent and vsock support 
- Introduced a standalone guest execution agent to facilitate command execution within the guest VM over Firecracker's vsock.
- Enhanced FirecrackerClient to support vsock configuration, validating guest CID and UDS path.
- Updated configuration to include vsock settings by default.
- Refactored FirecrackerProcessManager to manage runtime sockets more effectively.
- Implemented guest agent installation and provisioning in the image store.
- Added API endpoints for executing commands in the VM and installing the guest agent.
- Improved error handling for guest execution failures and agent availability.
2026-09-25 17:39:06 +05:30

1318 lines
58 KiB
Python
Raw Blame History

from __future__ import annotations
import errno
import os
import signal
import socket
import stat
import subprocess
import sys
import tempfile
import unittest
from dataclasses import replace
from pathlib import Path
from types import SimpleNamespace
from unittest.mock import Mock, call, patch
from uvm.config import Settings
from uvm.domain import VmRecord
from uvm.errors import FirecrackerError
from uvm.firecracker import process as process_module
from uvm.firecracker.process import FirecrackerProcessManager, ProcessInfo
BOOT_ID = "6a2e1188-3925-456b-867c-855e7542b092"
OLD_BOOT_ID = "02e208af-7560-41ee-b8f9-df3579e57c71"
PID = 12345
PIDFD = 91
def make_vm(
directory: Path,
pid: int | None = PID,
start_time: str | None = "42",
*,
boot_id: str | None = BOOT_ID,
operation: str | None = None,
spawn_pending: bool = False,
) -> VmRecord:
return VmRecord(
id="vm-test",
cpu=1,
ram_mib=512,
guest_ip="10.42.0.2",
gateway="10.42.0.1",
tap="uvm-test",
mac="02:fc:00:00:00:01",
pid=pid,
process_start_time=start_time,
boot_id=boot_id,
operation=operation,
spawn_pending=spawn_pending,
socket=str(directory / "firecracker.sock"),
config=str(directory / "config.json"),
log=str(directory / "firecracker.log"),
disk=str(directory / "rootfs.ext4"),
)
class FakeClock:
def __init__(self) -> None:
self.now = 0.0
def monotonic(self) -> float:
return self.now
def sleep(self, duration: float) -> None:
self.now += duration
class ProcessTestCase(unittest.TestCase):
def setUp(self) -> None:
temporary_directory = tempfile.TemporaryDirectory()
self.addCleanup(temporary_directory.cleanup)
self.directory = Path(temporary_directory.name)
self.clock = FakeClock()
self.child = Mock(pid=PID)
self.child.poll.return_value = None
self.popen = Mock(return_value=self.child)
self.manager = FirecrackerProcessManager(
Settings(
base=self.directory,
api_socket_timeout_s=0.1,
terminate_timeout_s=0.1,
),
popen=self.popen,
sleep=self.clock.sleep,
monotonic=self.clock.monotonic,
)
self.vm = make_vm(self.directory)
self.boot_id = self.enterContext(patch.object(self.manager, "_boot_id", return_value=BOOT_ID))
self.stat = self.enterContext(
patch.object(self.manager, "_process_stat", return_value=("S", "42"))
)
# Every signal/handle boundary is mocked for the synthetic PID and fd.
self.kill = Mock()
self.close = Mock()
self.enterContext(
patch.object(
process_module,
"os",
SimpleNamespace(
kill=self.kill, close=self.close, getpid=os.getpid, fsencode=os.fsencode,
),
)
)
self.open_pidfd = self.enterContext(patch.object(process_module, "_open_pidfd", return_value=PIDFD))
self.signal_pidfd = self.enterContext(patch.object(process_module, "_signal_pidfd"))
self.exited = self.enterContext(patch.object(process_module, "_pidfd_exited", return_value=False))
def tearDown(self) -> None:
for invocation in self.kill.call_args_list:
self.assertEqual(invocation.args[1], 0, "never signal a bare PID")
class FirecrackerProcessSafetyTests(ProcessTestCase):
def test_vsock_socket_is_removed_only_after_confirmed_process_exit(self) -> None:
path = self.manager._settings.guest_socket_path(self.vm.id)
path.parent.mkdir(parents=True)
path.touch()
self.vm.process_start_time = None
with self.assertRaises(FirecrackerError):
self.manager.terminate(self.vm)
self.assertTrue(path.exists())
self.vm.boot_id = OLD_BOOT_ID
self.manager.terminate(self.vm)
self.assertFalse(path.exists())
def test_unowned_vsock_socket_keeps_a_missing_pid_ambiguous(self) -> None:
path = self.manager._settings.guest_socket_path(self.vm.id)
path.parent.mkdir(parents=True)
path.touch()
self.vm.pid = self.vm.process_start_time = None
self.assertEqual(self.manager.observe(self.vm), "unknown")
with self.assertRaises(FirecrackerError):
self.manager.terminate(self.vm)
self.assertTrue(path.exists())
def test_observes_a_matching_identity_as_alive(self) -> None:
self.assertEqual(self.manager.observe(self.vm), "alive")
self.assertTrue(self.manager.is_alive(self.vm))
def test_refuses_to_signal_an_unidentified_legacy_pid(self) -> None:
self.vm.process_start_time = None
Path(self.vm.socket).touch()
with self.assertRaisesRegex(FirecrackerError, "identity is ambiguous"):
self.manager.terminate(self.vm)
self.assertTrue(Path(self.vm.socket).exists())
self.signal_pidfd.assert_not_called()
self.open_pidfd.assert_not_called()
def test_unreadable_legacy_proc_with_a_possible_live_pid_is_unknown(self) -> None:
self.vm.process_start_time = None
self.stat.return_value = None
with self.assertRaises(FirecrackerError):
self.manager.terminate(self.vm)
self.kill.assert_called_once_with(PID, 0)
self.signal_pidfd.assert_not_called()
def test_allows_cleanup_when_an_unidentified_legacy_pid_is_already_gone(self) -> None:
self.vm.process_start_time = None
self.stat.return_value = None
self.kill.side_effect = ProcessLookupError
Path(self.vm.socket).touch()
self.manager.terminate(self.vm)
self.assertFalse(Path(self.vm.socket).exists())
self.signal_pidfd.assert_not_called()
def test_rejects_non_positive_pids_without_signaling_or_unlinking(self) -> None:
Path(self.vm.socket).touch()
for pid in (0, -1):
with self.subTest(pid=pid):
self.vm.pid = pid
self.assertEqual(self.manager.observe(self.vm), "unknown")
with self.assertRaisesRegex(FirecrackerError, "invalid persisted"):
self.manager.terminate(self.vm)
self.assertTrue(Path(self.vm.socket).exists())
self.kill.assert_not_called()
self.signal_pidfd.assert_not_called()
def test_a_prior_boot_proves_exit_even_if_pid_and_start_time_match(self) -> None:
self.vm.boot_id = OLD_BOOT_ID
Path(self.vm.socket).touch()
self.assertEqual(self.manager.observe(self.vm), "gone")
self.manager.terminate(self.vm)
self.assertFalse(Path(self.vm.socket).exists())
self.stat.assert_not_called()
self.signal_pidfd.assert_not_called()
def test_missing_saved_or_current_boot_identity_is_unknown(self) -> None:
for saved_boot, current_boot in ((None, BOOT_ID), (BOOT_ID, None), (None, None)):
with self.subTest(saved=saved_boot, current=current_boot):
self.vm.boot_id = saved_boot
self.boot_id.return_value = current_boot
self.assertEqual(self.manager.observe(self.vm), "unknown")
self.assertTrue(self.manager.is_alive(self.vm))
with self.assertRaises(FirecrackerError):
self.manager.terminate(self.vm)
self.signal_pidfd.assert_not_called()
def test_malformed_saved_tokens_do_not_prove_pid_reuse(self) -> None:
Path(self.vm.socket).touch()
for boot_id in (None, BOOT_ID):
for start_time in ("invalid", "-42", " 42", "42\0", "\u0664\u0662", 42):
with self.subTest(boot=boot_id, start_time=start_time):
self.vm.boot_id = boot_id
self.vm.process_start_time = start_time
self.assertEqual(self.manager.observe(self.vm), "unknown")
with self.assertRaisesRegex(FirecrackerError, "identity is ambiguous"):
self.manager.terminate(self.vm)
self.assertTrue(Path(self.vm.socket).exists())
self.signal_pidfd.assert_not_called()
def test_unreadable_proc_is_unknown_unless_pid_is_proven_missing(self) -> None:
self.stat.return_value = None
for error in (None, PermissionError(), OSError(errno.EIO, "unavailable"), OverflowError()):
with self.subTest(error=error):
self.kill.side_effect = error
self.assertEqual(self.manager.observe(self.vm), "unknown")
self.kill.side_effect = ProcessLookupError
self.assertEqual(self.manager.observe(self.vm), "gone")
self.assertFalse(self.manager.is_alive(self.vm))
def test_pid_reuse_is_gone_and_never_signaled(self) -> None:
self.stat.return_value = ("S", "99")
Path(self.vm.socket).touch()
self.assertEqual(self.manager.observe(self.vm), "gone")
self.manager.terminate(self.vm)
self.assertFalse(Path(self.vm.socket).exists())
self.open_pidfd.assert_not_called()
self.signal_pidfd.assert_not_called()
def test_zombie_and_dead_processes_are_gone_when_the_group_has_exited(self) -> None:
self.exited.return_value = True
for state in ("Z", "X", "x"):
with self.subTest(state=state):
self.stat.return_value = (state, "42")
self.assertEqual(self.manager.observe(self.vm), "gone")
self.assertEqual(self.close.call_count, 3)
self.signal_pidfd.assert_not_called()
def test_zombie_leader_with_live_threads_is_not_safe_to_delete(self) -> None:
self.stat.return_value = ("Z", "42")
Path(self.vm.socket).touch()
self.assertEqual(self.manager.observe(self.vm), "unknown")
with self.assertRaises(FirecrackerError):
self.manager.terminate(self.vm)
self.assertTrue(Path(self.vm.socket).exists())
self.signal_pidfd.assert_not_called()
def test_zombie_observation_fails_closed_without_a_safe_handle(self) -> None:
self.stat.return_value = ("Z", "42")
self.open_pidfd.side_effect = OSError(errno.ENOSYS, "unsupported")
self.assertEqual(self.manager.observe(self.vm), "unknown")
def test_no_pid_during_spawning_is_unknown_even_without_a_socket(self) -> None:
self.vm.pid = None
self.vm.process_start_time = None
self.vm.operation = "spawning"
for saved_boot in (None, BOOT_ID):
with self.subTest(boot=saved_boot):
self.vm.boot_id = saved_boot
self.assertEqual(self.manager.observe(self.vm), "unknown")
with self.assertRaises(FirecrackerError):
self.manager.terminate(self.vm)
with self.assertRaises(FirecrackerError):
self.manager.start(self.vm)
self.popen.assert_not_called()
self.signal_pidfd.assert_not_called()
def test_pending_spawn_survives_persisted_stop_and_delete_intent_without_a_pid(self) -> None:
for operation in ("stop", "delete"):
for saved_boot, current_boot in ((None, BOOT_ID), (BOOT_ID, BOOT_ID), (OLD_BOOT_ID, None)):
with self.subTest(operation=operation, saved_boot=saved_boot, current_boot=current_boot):
vm = make_vm(
self.directory, pid=None, start_time=None, boot_id=saved_boot,
operation=operation, spawn_pending=True,
)
vm = VmRecord.from_dict(vm.to_dict())
self.boot_id.return_value = current_boot
self.assertEqual(self.manager.observe(vm), "unknown")
with self.assertRaisesRegex(FirecrackerError, "identity is ambiguous"):
self.manager.terminate(vm)
with self.assertRaises(FirecrackerError):
self.manager.start(vm)
self.assertTrue(vm.spawn_pending)
self.popen.assert_not_called()
self.signal_pidfd.assert_not_called()
def test_prior_boot_proves_a_pending_spawn_gone_despite_stop_or_delete_intent(self) -> None:
for operation in ("stop", "delete"):
with self.subTest(operation=operation):
vm = make_vm(
self.directory, pid=None, start_time=None, boot_id=OLD_BOOT_ID,
operation=operation, spawn_pending=True,
)
Path(vm.socket).touch()
self.assertEqual(self.manager.observe(vm), "gone")
self.manager.terminate(vm)
self.assertFalse(Path(vm.socket).exists())
self.signal_pidfd.assert_not_called()
def test_known_pid_observation_is_independent_of_spawn_pending(self) -> None:
self.vm.spawn_pending = True
self.vm.operation = "delete"
self.assertEqual(self.manager.observe(self.vm), "alive")
self.stat.return_value = None
self.kill.side_effect = ProcessLookupError
self.assertEqual(self.manager.observe(self.vm), "gone")
self.manager.terminate(self.vm)
def test_no_pid_spawning_from_a_proven_prior_boot_is_gone(self) -> None:
self.vm.pid = None
self.vm.process_start_time = None
self.vm.operation = "spawning"
self.vm.boot_id = OLD_BOOT_ID
Path(self.vm.socket).touch()
self.assertEqual(self.manager.observe(self.vm), "gone")
self.manager.terminate(self.vm)
self.assertFalse(Path(self.vm.socket).exists())
self.signal_pidfd.assert_not_called()
def test_no_pid_spawning_cannot_infer_a_reboot_from_an_unreadable_boot_id(self) -> None:
self.vm.pid = None
self.vm.process_start_time = None
self.vm.operation = "spawning"
self.vm.boot_id = OLD_BOOT_ID
self.boot_id.return_value = None
self.assertEqual(self.manager.observe(self.vm), "unknown")
def test_a_clean_unspawned_record_is_gone(self) -> None:
self.vm.pid = None
self.vm.process_start_time = None
self.vm.boot_id = None
self.assertEqual(self.manager.observe(self.vm), "gone")
self.manager.terminate(self.vm)
self.signal_pidfd.assert_not_called()
def test_an_unowned_socket_is_not_assumed_stale(self) -> None:
self.vm.pid = None
self.vm.process_start_time = None
Path(self.vm.socket).touch()
self.assertEqual(self.manager.observe(self.vm), "unknown")
with self.assertRaises(FirecrackerError):
self.manager.start(self.vm)
with self.assertRaises(FirecrackerError):
self.manager.terminate(self.vm)
self.assertTrue(Path(self.vm.socket).exists())
self.popen.assert_not_called()
def test_an_unreadable_unowned_socket_is_unknown(self) -> None:
self.vm.pid = None
self.vm.process_start_time = None
with patch.object(Path, "lstat", side_effect=PermissionError):
self.assertEqual(self.manager.observe(self.vm), "unknown")
def test_term_exit_is_confirmed_before_unlinking_and_identity_is_retained(self) -> None:
Path(self.vm.socket).touch()
self.exited.side_effect = [False, True]
self.signal_pidfd.side_effect = lambda *_args: self.assertTrue(Path(self.vm.socket).exists())
self.manager.terminate(self.vm)
self.signal_pidfd.assert_called_once_with(PIDFD, signal.SIGTERM)
self.close.assert_called_once_with(PIDFD)
self.assertFalse(Path(self.vm.socket).exists())
self.assertEqual((self.vm.pid, self.vm.process_start_time, self.vm.boot_id), (PID, "42", BOOT_ID))
self.kill.assert_not_called()
def test_forced_stop_waits_for_exit_even_with_zero_term_timeout(self) -> None:
Path(self.vm.socket).touch()
self.exited.side_effect = [False, False, False, True]
self.signal_pidfd.side_effect = lambda *_args: self.assertTrue(Path(self.vm.socket).exists())
self.manager.terminate(self.vm, timeout=0)
self.assertEqual(
self.signal_pidfd.call_args_list,
[call(PIDFD, signal.SIGTERM), call(PIDFD, signal.SIGKILL)],
)
self.assertGreater(self.clock.now, 0)
self.assertFalse(Path(self.vm.socket).exists())
def test_unconfirmed_kill_retains_socket_and_identity(self) -> None:
Path(self.vm.socket).touch()
with self.assertRaisesRegex(FirecrackerError, "could not confirm exit.*SIGKILL"):
self.manager.terminate(self.vm, timeout=0)
self.assertTrue(Path(self.vm.socket).exists())
self.assertEqual((self.vm.pid, self.vm.process_start_time, self.vm.boot_id), (PID, "42", BOOT_ID))
self.assertEqual(self.clock.now, 1.0)
self.close.assert_called_once_with(PIDFD)
def test_signal_errors_preserve_the_socket(self) -> None:
Path(self.vm.socket).touch()
for failing_signal in (signal.SIGTERM, signal.SIGKILL):
with self.subTest(signal=failing_signal):
def fail_signal(_pidfd: int, signal_number: int) -> None:
if signal_number == failing_signal:
raise PermissionError("denied")
self.signal_pidfd.side_effect = fail_signal
with self.assertRaisesRegex(FirecrackerError, "could not safely stop"):
self.manager.terminate(self.vm, timeout=0)
self.assertTrue(Path(self.vm.socket).exists())
def test_signal_lookup_error_still_requires_exit_confirmation(self) -> None:
Path(self.vm.socket).touch()
self.signal_pidfd.side_effect = ProcessLookupError
self.exited.side_effect = [False, True]
self.manager.terminate(self.vm)
self.assertFalse(Path(self.vm.socket).exists())
self.assertEqual(self.exited.call_count, 2)
def test_signal_lookup_error_does_not_hide_an_unconfirmed_exit(self) -> None:
Path(self.vm.socket).touch()
self.signal_pidfd.side_effect = ProcessLookupError
with self.assertRaisesRegex(FirecrackerError, "could not confirm exit"):
self.manager.terminate(self.vm, timeout=0)
self.assertTrue(Path(self.vm.socket).exists())
def test_pid_disappearing_before_handle_acquisition_is_safe_to_clean(self) -> None:
Path(self.vm.socket).touch()
self.open_pidfd.side_effect = ProcessLookupError
self.manager.terminate(self.vm)
self.assertFalse(Path(self.vm.socket).exists())
self.signal_pidfd.assert_not_called()
self.close.assert_not_called()
def test_unavailable_or_denied_pidfd_does_not_fall_back_to_pid_signals(self) -> None:
Path(self.vm.socket).touch()
for error in (OSError(errno.ENOSYS, "unsupported"), PermissionError("denied")):
with self.subTest(error=error):
self.open_pidfd.side_effect = error
with self.assertRaisesRegex(FirecrackerError, "safely open a pidfd"):
self.manager.terminate(self.vm)
self.assertTrue(Path(self.vm.socket).exists())
self.kill.assert_not_called()
self.signal_pidfd.assert_not_called()
def test_unavailable_pidfd_signaling_does_not_fall_back_to_pid_signals(self) -> None:
Path(self.vm.socket).touch()
self.signal_pidfd.side_effect = OSError(errno.ENOSYS, "unsupported")
with self.assertRaises(FirecrackerError):
self.manager.terminate(self.vm)
self.assertTrue(Path(self.vm.socket).exists())
self.kill.assert_not_called()
self.close.assert_called_once_with(PIDFD)
def test_identity_is_rechecked_after_opening_the_pidfd(self) -> None:
Path(self.vm.socket).touch()
self.stat.side_effect = [("S", "42"), ("S", "99")]
self.manager.terminate(self.vm)
self.signal_pidfd.assert_not_called()
self.close.assert_called_once_with(PIDFD)
self.assertFalse(Path(self.vm.socket).exists())
def test_identity_becoming_unreadable_after_open_is_not_signaled(self) -> None:
Path(self.vm.socket).touch()
self.stat.side_effect = [("S", "42"), None]
with self.assertRaisesRegex(FirecrackerError, "identity became ambiguous"):
self.manager.terminate(self.vm)
self.signal_pidfd.assert_not_called()
self.close.assert_called_once_with(PIDFD)
self.assertTrue(Path(self.vm.socket).exists())
def test_an_already_exited_handle_needs_no_signal(self) -> None:
Path(self.vm.socket).touch()
self.exited.return_value = True
self.manager.terminate(self.vm)
self.signal_pidfd.assert_not_called()
self.assertFalse(Path(self.vm.socket).exists())
def test_poll_failure_retains_the_socket_and_closes_the_handle(self) -> None:
Path(self.vm.socket).touch()
self.exited.side_effect = OSError(errno.EIO, "poll failed")
with self.assertRaises(FirecrackerError):
self.manager.terminate(self.vm)
self.assertTrue(Path(self.vm.socket).exists())
self.close.assert_called_once_with(PIDFD)
self.signal_pidfd.assert_not_called()
def test_socket_unlink_errors_are_not_suppressed(self) -> None:
self.vm.boot_id = OLD_BOOT_ID
with patch.object(Path, "unlink", side_effect=PermissionError("denied")):
with self.assertRaisesRegex(FirecrackerError, "could not remove Firecracker socket"):
self.manager.terminate(self.vm)
def test_timeout_override_controls_term_grace_not_exit_confirmation(self) -> None:
with patch.object(self.manager, "_wait_for_exit", side_effect=[False, True]) as wait:
self.manager.terminate(self.vm, timeout=0.025)
self.assertEqual(wait.call_args_list, [call(PIDFD, 0.025), call(PIDFD, 1.0)])
def test_invalid_timeouts_are_rejected_before_signaling(self) -> None:
for timeout in (-1, float("nan"), float("inf")):
with self.subTest(timeout=timeout):
with self.assertRaisesRegex(FirecrackerError, "timeout must be"):
self.manager.terminate(self.vm, timeout=timeout)
self.signal_pidfd.assert_not_called()
def test_observation_reaps_only_owned_children(self) -> None:
self.manager._children[PID] = self.child
self.child.poll.return_value = 0
self.stat.return_value = None
self.kill.side_effect = ProcessLookupError
self.assertEqual(self.manager.observe(self.vm), "gone")
self.child.poll.assert_called_once_with()
self.assertNotIn(PID, self.manager._children)
class FirecrackerProcessStartTests(ProcessTestCase):
def setUp(self) -> None:
super().setUp()
self.vm = make_vm(self.directory, pid=None, start_time=None, boot_id=None)
self.wait = self.enterContext(patch.object(self.manager, "_wait_for_socket"))
def test_identity_is_in_memory_and_published_before_socket_readiness(self) -> None:
events = []
self.vm.operation = "boot"
persisted = [replace(self.vm, operation="spawning", boot_id=BOOT_ID, spawn_pending=True).to_dict()]
self.assertEqual(self.manager.observe(VmRecord.from_dict(persisted[0])), "unknown")
def on_spawn(info: ProcessInfo) -> None:
events.append("spawn")
self.assertEqual(info, ProcessInfo(PID, "42", BOOT_ID))
self.assertEqual((self.vm.pid, self.vm.process_start_time, self.vm.boot_id), (PID, "42", BOOT_ID))
self.assertTrue(self.vm.spawn_pending)
self.wait.assert_not_called()
self.vm.operation = "configuring"
self.vm.spawn_pending = False
persisted.append(self.vm.to_dict())
self.wait.side_effect = lambda *_args: events.append("socket")
info = self.manager.start(self.vm, on_spawn=on_spawn)
self.assertEqual(info, ProcessInfo(PID, "42", BOOT_ID))
self.assertEqual(events, ["spawn", "socket"])
self.assertEqual(self.vm.operation, "configuring")
self.assertFalse(self.vm.spawn_pending)
self.assertEqual(VmRecord.from_dict(persisted[1]), self.vm)
self.assertTrue(self.popen.call_args.kwargs["start_new_session"])
self.signal_pidfd.assert_called_once_with(PIDFD, 0)
self.open_pidfd.assert_called_once_with(os.getpid())
self.assertIsNone(ProcessInfo(PID, "42").boot_id)
def test_start_clears_pending_after_identity_only_when_no_callback_is_supplied(self) -> None:
def spawn(*_args, **_kwargs):
self.assertTrue(self.vm.spawn_pending)
self.assertEqual(self.vm.operation, "spawning")
self.assertIsNone(self.vm.pid)
return self.child
self.popen.side_effect = spawn
for on_spawn in (None, Mock()):
with self.subTest(callback=on_spawn is not None):
self.vm = make_vm(self.directory, pid=None, start_time=None, boot_id=None)
self.wait.side_effect = lambda *_args: self.assertEqual(
self.vm.spawn_pending, on_spawn is not None,
)
self.manager.start(self.vm, on_spawn=on_spawn)
self.assertEqual(self.vm.spawn_pending, on_spawn is not None)
self.assertEqual(self.manager.observe(self.vm), "alive")
def test_start_refuses_live_or_ambiguous_existing_identity(self) -> None:
self.vm.pid = PID
self.vm.boot_id = BOOT_ID
Path(self.vm.socket).touch()
for start_time in ("42", None):
with self.subTest(start_time=start_time):
self.vm.process_start_time = start_time
with self.assertRaisesRegex(FirecrackerError, "refusing to start"):
self.manager.start(self.vm)
self.assertTrue(Path(self.vm.socket).exists())
self.popen.assert_not_called()
self.open_pidfd.assert_not_called()
def test_start_replaces_only_a_proven_stale_socket(self) -> None:
self.vm.boot_id = OLD_BOOT_ID
self.vm.operation = "spawning"
Path(self.vm.socket).touch()
def on_spawn(_info: ProcessInfo) -> None:
self.assertFalse(Path(self.vm.socket).exists())
self.manager.start(self.vm, on_spawn=on_spawn)
self.assertEqual(self.vm.boot_id, BOOT_ID)
self.assertEqual(self.vm.pid, PID)
def test_failing_persistence_callback_cleans_up_without_losing_identity(self) -> None:
failure = RuntimeError("persistence failed")
self.exited.side_effect = [False, True]
self.child.poll.side_effect = [None, None, -signal.SIGTERM]
def on_spawn(_info: ProcessInfo) -> None:
self.assertEqual((self.vm.pid, self.vm.process_start_time, self.vm.boot_id), (PID, "42", BOOT_ID))
Path(self.vm.socket).touch()
raise failure
with self.assertRaises(RuntimeError) as raised:
self.manager.start(self.vm, on_spawn=on_spawn)
self.assertIs(raised.exception, failure)
self.wait.assert_not_called()
self.assertFalse(Path(self.vm.socket).exists())
self.assertEqual((self.vm.pid, self.vm.process_start_time, self.vm.boot_id), (PID, "42", BOOT_ID))
self.assertNotIn(PID, self.manager._children)
self.assertEqual(self.signal_pidfd.call_args_list, [call(PIDFD, 0), call(PIDFD, signal.SIGTERM)])
def test_interrupt_during_startup_terminates_the_child_process(self) -> None:
self.wait.side_effect = KeyboardInterrupt
self.exited.side_effect = [False, True]
with self.assertRaises(KeyboardInterrupt):
self.manager.start(self.vm)
self.assertEqual(self.signal_pidfd.call_args_list, [call(PIDFD, 0), call(PIDFD, signal.SIGTERM)])
self.assertEqual(self.vm.pid, PID)
def test_incomplete_spawn_cleanup_surfaces_both_failures_and_preserves_resources(self) -> None:
failure = RuntimeError("persistence failed")
def on_spawn(_info: ProcessInfo) -> None:
Path(self.vm.socket).touch()
raise failure
with self.assertRaisesRegex(FirecrackerError, "persistence failed.*cleanup incomplete") as raised:
self.manager.start(self.vm, on_spawn=on_spawn)
self.assertIs(raised.exception.__cause__, failure)
self.wait.assert_not_called()
self.assertTrue(Path(self.vm.socket).exists())
self.assertEqual((self.vm.pid, self.vm.process_start_time, self.vm.boot_id), (PID, "42", BOOT_ID))
def test_identity_capture_failure_retains_partial_identity_and_reports_incomplete_cleanup(self) -> None:
self.stat.return_value = None
on_spawn = Mock()
def spawn(*_args, **_kwargs):
Path(self.vm.socket).touch()
return self.child
self.popen.side_effect = spawn
with self.assertRaisesRegex(FirecrackerError, "cleanup incomplete.*identity is ambiguous"):
self.manager.start(self.vm, on_spawn=on_spawn)
on_spawn.assert_not_called()
self.wait.assert_not_called()
self.assertEqual((self.vm.pid, self.vm.process_start_time, self.vm.boot_id), (PID, None, BOOT_ID))
self.assertTrue(self.vm.spawn_pending)
self.assertTrue(Path(self.vm.socket).exists())
self.signal_pidfd.assert_called_once_with(PIDFD, 0)
def test_popen_oserror_restores_prior_operation_and_metadata(self) -> None:
self.popen.side_effect = FileNotFoundError(errno.ENOENT, "missing binary")
for pid, start_time, boot_id, operation, spawn_pending in (
(None, None, None, None, False),
(None, None, BOOT_ID, "boot", False),
(PID, "42", OLD_BOOT_ID, "boot", False),
(None, None, OLD_BOOT_ID, "spawning", True),
(None, None, OLD_BOOT_ID, "boot", True),
):
with self.subTest(pid=pid, boot=boot_id, operation=operation):
self.vm = make_vm(
self.directory, pid, start_time, boot_id=boot_id, operation=operation,
spawn_pending=spawn_pending,
)
previous = self.vm.to_dict()
on_spawn = Mock()
with self.assertRaisesRegex(FirecrackerError, "could not start Firecracker"):
self.manager.start(self.vm, on_spawn=on_spawn)
self.assertEqual(self.vm.to_dict(), previous)
self.assertEqual(self.manager.observe(self.vm), "gone")
on_spawn.assert_not_called()
self.wait.assert_not_called()
self.assertEqual(self.manager._children, {})
def test_uncertain_popen_failure_preserves_the_spawning_marker(self) -> None:
for error in (
KeyboardInterrupt(), SystemExit(), RuntimeError("uncertain spawn"),
InterruptedError(errno.EINTR, "interrupted spawn"),
):
with self.subTest(error=type(error).__name__):
self.vm = make_vm(
self.directory, pid=None, start_time=None, boot_id=None, operation="boot",
)
self.popen.side_effect = error
exception_type = FirecrackerError if isinstance(error, OSError) else type(error)
with self.assertRaises(exception_type):
self.manager.start(self.vm)
self.assertEqual(self.vm.operation, "spawning")
self.assertTrue(self.vm.spawn_pending)
self.assertEqual(self.vm.boot_id, BOOT_ID)
self.assertIsNone(self.vm.pid)
self.assertIsNone(self.vm.process_start_time)
self.assertEqual(self.manager.observe(self.vm), "unknown")
def test_post_spawn_oserror_does_not_restore_the_old_identity(self) -> None:
self.vm.operation = "boot"
self.exited.side_effect = [False, True]
def on_spawn(_info: ProcessInfo) -> None:
self.vm.operation = "configuring"
raise OSError("persistence failed")
with self.assertRaisesRegex(FirecrackerError, "persistence failed"):
self.manager.start(self.vm, on_spawn=on_spawn)
self.assertEqual(self.vm.operation, "configuring")
self.assertEqual((self.vm.pid, self.vm.process_start_time, self.vm.boot_id), (PID, "42", BOOT_ID))
self.wait.assert_not_called()
def test_unsupported_pidfd_prevents_spawn_and_preserves_the_old_record(self) -> None:
self.vm.boot_id = OLD_BOOT_ID
Path(self.vm.socket).touch()
self.open_pidfd.side_effect = OSError(errno.ENOSYS, "unsupported")
with self.assertRaisesRegex(FirecrackerError, "safe pidfd support is required"):
self.manager.start(self.vm)
self.popen.assert_not_called()
self.assertTrue(Path(self.vm.socket).exists())
self.assertEqual(self.vm.boot_id, OLD_BOOT_ID)
self.assertIsNone(self.vm.operation)
def test_unknown_current_boot_prevents_spawn(self) -> None:
self.boot_id.return_value = None
with self.assertRaisesRegex(FirecrackerError, "current host boot identity"):
self.manager.start(self.vm)
self.popen.assert_not_called()
self.open_pidfd.assert_not_called()
def test_socket_wait_checks_exit_before_socket_presence(self) -> None:
self.child.poll.return_value = 0
with patch.object(Path, "is_socket", return_value=True) as is_socket:
with self.assertRaisesRegex(FirecrackerError, "exited early"):
FirecrackerProcessManager._wait_for_socket(
self.manager, self.child, Path(self.vm.socket), Path(self.vm.log)
)
is_socket.assert_not_called()
def test_socket_wait_does_not_accept_a_regular_file(self) -> None:
Path(self.vm.socket).touch()
with self.assertRaisesRegex(FirecrackerError, "timed out waiting"):
FirecrackerProcessManager._wait_for_socket(
self.manager, self.child, Path(self.vm.socket), Path(self.vm.log)
)
def test_socket_timeout_occurs_after_identity_publication_and_confirms_cleanup(self) -> None:
self.wait.side_effect = FirecrackerError("timed out waiting for Firecracker API socket")
self.exited.side_effect = [False, True]
on_spawn = Mock()
with self.assertRaisesRegex(FirecrackerError, "timed out waiting"):
self.manager.start(self.vm, on_spawn=on_spawn)
on_spawn.assert_called_once_with(ProcessInfo(PID, "42", BOOT_ID))
self.assertEqual(self.vm.pid, PID)
class FirecrackerLegacyAdoptionTests(ProcessTestCase):
def setUp(self) -> None:
super().setUp()
self.runtime = self.manager._settings.vm_dir(self.vm.id)
self.runtime.mkdir(parents=True, mode=0o700)
self.runtime.parent.chmod(0o755)
self.vm = make_vm(self.runtime, boot_id=None)
self.binary = self.manager._settings.firecracker_binary
self.binary.parent.mkdir(mode=0o755)
self.binary.touch(mode=0o755)
listener = socket.socket(socket.AF_UNIX)
self.addCleanup(listener.close)
listener.bind(self.vm.socket)
# Production runs as root; preserve real file types/modes in unprivileged tests.
real_lstat = Path.lstat
def root_lstat(path: Path) -> os.stat_result:
metadata = list(real_lstat(path))
metadata[4] = 0
# Model protected system ancestors independently of the caller's
# TMPDIR permissions. Unsafe-path fixtures live inside self.directory.
if path in self.directory.parents:
metadata[0] &= ~0o022
return os.stat_result(metadata)
self.root_lstat = root_lstat
self.enterContext(patch.object(Path, "lstat", autospec=True, side_effect=root_lstat))
self.executable = self.enterContext(
patch.object(Path, "readlink", autospec=True, return_value=self.binary)
)
self.samefile = self.enterContext(patch.object(Path, "samefile", autospec=True, return_value=True))
self.expected_cmdline = b"\0".join(
(os.fsencode(self.binary), b"--api-sock", os.fsencode(self.vm.socket), b"")
)
self.cmdline = self.enterContext(
patch.object(Path, "read_bytes", autospec=True, return_value=self.expected_cmdline)
)
def test_verified_v1_identity_can_be_persisted_and_explicitly_stopped(self) -> None:
self.vm.status = "running"
legacy = self.vm.to_dict()
for name in (
"boot_id", "operation", "disk_ready", "runtime_owned", "tap_owned",
"ports", "network", "bridge", "name", "spawn_pending",
):
legacy.pop(name)
self.vm = VmRecord.from_dict(legacy)
previous = self.vm.to_dict()
self.assertEqual(self.manager.observe(self.vm), "unknown")
with self.assertRaisesRegex(FirecrackerError, "identity is ambiguous"):
self.manager.terminate(self.vm)
self.assertTrue(self.manager.adopt_legacy_identity(self.vm))
self.assertEqual(self.vm.to_dict(), previous | {"boot_id": BOOT_ID})
self.signal_pidfd.assert_not_called()
self.kill.assert_not_called()
self.close.assert_called_once_with(PIDFD)
persisted = VmRecord.from_dict(self.vm.to_dict())
self.assertEqual(self.manager.observe(persisted), "alive")
self.exited.side_effect = [False, True]
self.manager.terminate(persisted)
self.signal_pidfd.assert_called_once_with(PIDFD, signal.SIGTERM)
self.assertFalse(Path(self.vm.socket).exists())
def test_pidfd_pins_the_process_before_any_identity_read_until_final_exit_check(self) -> None:
events = Mock()
for name, boundary in (
("open", self.open_pidfd), ("stat", self.stat), ("exe", self.executable),
("samefile", self.samefile), ("cmdline", self.cmdline), ("boot", self.boot_id),
("exited", self.exited), ("close", self.close),
):
events.attach_mock(boundary, name)
self.close.side_effect = lambda _fd: self.assertIsNone(self.vm.boot_id)
self.assertTrue(self.manager.adopt_legacy_identity(self.vm))
proc = Path(f"/proc/{PID}")
self.assertEqual(events.mock_calls, [
call.open(PID), call.stat(PID), call.exe(proc / "exe"),
call.samefile(proc / "exe", self.binary), call.cmdline(proc / "cmdline"),
call.boot(), call.exited(PIDFD), call.close(PIDFD),
])
def test_only_missing_boot_id_with_a_valid_pid_and_original_token_is_eligible(self) -> None:
for boot_id, pid, start_time in (
(BOOT_ID, PID, "42"), (OLD_BOOT_ID, PID, "42"), ("", PID, "42"),
(None, None, "42"), (None, 0, "42"), (None, -1, "42"),
(None, 2**32 + PID, "42"), (None, True, "42"),
(None, PID, None), (None, PID, ""), (None, PID, "invalid"),
(None, PID, "-42"), (None, PID, " 42"), (None, PID, "42\0"),
(None, PID, "\u0664\u0662"), (None, PID, 42),
):
with self.subTest(boot=boot_id, pid=pid, start_time=start_time):
vm = replace(self.vm, boot_id=boot_id, pid=pid, process_start_time=start_time)
previous = vm.to_dict()
self.assertFalse(self.manager.adopt_legacy_identity(vm))
self.assertEqual(vm.to_dict(), previous)
self.open_pidfd.assert_not_called()
self.signal_pidfd.assert_not_called()
self.stat.assert_not_called()
def test_missing_mismatched_or_dead_proc_identity_is_never_adopted(self) -> None:
for process_stat in (None, ("S", "99"), ("Z", "42"), ("X", "42"), ("x", "42")):
with self.subTest(stat=process_stat):
self.stat.return_value = process_stat
self.assertFalse(self.manager.adopt_legacy_identity(self.vm))
self.assertIsNone(self.vm.boot_id)
self.assertEqual(self.close.call_count, 5)
self.cmdline.assert_not_called()
self.signal_pidfd.assert_not_called()
def test_unavailable_or_denied_pidfd_is_not_replaced_by_a_pid_probe(self) -> None:
for error in (ProcessLookupError(), PermissionError(), OSError(errno.ENOSYS, "unsupported")):
with self.subTest(error=error):
self.open_pidfd.side_effect = error
self.assertFalse(self.manager.adopt_legacy_identity(self.vm))
self.assertIsNone(self.vm.boot_id)
self.stat.assert_not_called()
self.kill.assert_not_called()
self.close.assert_not_called()
def test_executable_must_match_both_the_configured_path_and_inode(self) -> None:
for executable, samefile in (
(Path("/usr/bin/sleep"), True),
(Path(str(self.binary) + " (deleted)"), True),
(Path("firecracker"), True),
(self.binary, False),
):
with self.subTest(executable=executable, samefile=samefile):
self.executable.return_value = executable
self.samefile.return_value = samefile
self.assertFalse(self.manager.adopt_legacy_identity(self.vm))
self.assertIsNone(self.vm.boot_id)
self.cmdline.assert_not_called()
self.signal_pidfd.assert_not_called()
def test_cmdline_must_match_all_three_arguments_and_nul_boundaries_exactly(self) -> None:
binary, api_socket = str(self.binary), self.vm.socket
commands = [
["firecracker", "--api-sock", api_socket],
["/usr/bin/sleep", "--api-sock", api_socket],
[binary, "--api-sock", api_socket + ".other"],
[binary, "--api-sock", str(self.runtime.parent / "other-vm" / "firecracker.sock")],
[binary, "--api-sock", "firecracker.sock"],
[binary, "--api-sock", api_socket, "--extra"],
[binary, f"--api-sock={api_socket}"],
[binary, api_socket, "--api-sock"],
[binary, "--api-sock"],
["/bin/sh", binary, "--api-sock", api_socket],
]
invalid_cmdlines = [
b"\0".join(os.fsencode(argument) for argument in command) + b"\0"
for command in commands
] + [b"", self.expected_cmdline[:-1], self.expected_cmdline + b"\0"]
for cmdline in invalid_cmdlines:
with self.subTest(cmdline=cmdline):
self.cmdline.return_value = cmdline
self.assertFalse(self.manager.adopt_legacy_identity(self.vm))
self.assertIsNone(self.vm.boot_id)
self.assertTrue(Path(self.vm.socket).exists())
self.signal_pidfd.assert_not_called()
def test_socket_record_must_be_the_exact_protected_path_for_its_vm(self) -> None:
for vm_id, api_socket in (
(self.vm.id, "firecracker.sock"),
(self.vm.id, str(self.directory / "firecracker.sock")),
(self.vm.id, str(self.runtime) + "/../vm-test/firecracker.sock"),
(self.vm.id, str(self.runtime) + "//firecracker.sock"),
("another-vm", self.vm.socket),
("../vm-test", str(self.directory / "vm-test/firecracker.sock")),
(".", str(self.runtime.parent / "firecracker.sock")),
("", str(self.runtime.parent / "firecracker.sock")),
):
with self.subTest(vm_id=vm_id, socket=api_socket):
vm = replace(self.vm, id=vm_id, socket=api_socket)
self.assertFalse(self.manager.adopt_legacy_identity(vm))
self.assertIsNone(vm.boot_id)
self.cmdline.assert_not_called()
self.assertTrue(Path(self.vm.socket).exists())
def test_missing_regular_or_symlink_socket_is_not_adopted(self) -> None:
socket_path = Path(self.vm.socket)
for kind in ("missing", "regular", "symlink"):
with self.subTest(kind=kind):
socket_path.unlink(missing_ok=True)
if kind == "regular":
socket_path.touch()
elif kind == "symlink":
socket_path.symlink_to(self.binary)
self.assertFalse(self.manager.adopt_legacy_identity(self.vm))
self.assertIsNone(self.vm.boot_id)
self.cmdline.assert_not_called()
def test_unprotected_or_symlinked_directories_and_binary_are_not_adopted(self) -> None:
for path, mode, uid in (
(self.directory, stat.S_IFDIR | 0o777, 0),
(self.runtime.parent, stat.S_IFDIR | 0o775, 0),
(self.runtime, stat.S_IFDIR | 0o755, 0),
(self.binary.parent, stat.S_IFDIR | 0o777, 0),
(self.runtime.parent, stat.S_IFLNK | 0o777, 0),
(self.directory, stat.S_IFDIR | 0o700, 1001),
(self.runtime, stat.S_IFDIR | 0o700, 1001),
(self.binary, stat.S_IFREG | 0o777, 0),
(self.binary, stat.S_IFREG | 0o755, 1001),
(self.binary, stat.S_IFLNK | 0o777, 0),
(Path(self.vm.socket), stat.S_IFSOCK | 0o700, 1001),
):
with self.subTest(path=path, mode=oct(mode), uid=uid):
def untrusted_lstat(candidate: Path) -> os.stat_result:
metadata = list(self.root_lstat(candidate))
if candidate == path:
metadata[0], metadata[4] = mode, uid
return os.stat_result(metadata)
with patch.object(Path, "lstat", new=untrusted_lstat):
self.assertFalse(self.manager.adopt_legacy_identity(self.vm))
self.assertIsNone(self.vm.boot_id)
self.cmdline.assert_not_called()
def test_unreadable_process_or_filesystem_evidence_does_not_adopt(self) -> None:
for method in ("lstat", "readlink", "samefile", "read_bytes"):
with self.subTest(method=method):
with patch.object(Path, method, side_effect=PermissionError("denied")):
self.assertFalse(self.manager.adopt_legacy_identity(self.vm))
self.assertIsNone(self.vm.boot_id)
self.assertEqual(self.close.call_count, 4)
self.signal_pidfd.assert_not_called()
def test_boot_exit_and_handle_errors_leave_identity_unmodified(self) -> None:
for name, failure in (
("boot", patch.object(self.manager, "_boot_id", return_value=None)),
("exited", patch.object(process_module, "_pidfd_exited", return_value=True)),
("poll", patch.object(process_module, "_pidfd_exited", side_effect=OSError("poll failed"))),
("close", patch.object(process_module.os, "close", side_effect=OSError("close failed"))),
):
with self.subTest(boundary=name), failure:
self.assertFalse(self.manager.adopt_legacy_identity(self.vm))
self.assertIsNone(self.vm.boot_id)
self.signal_pidfd.assert_not_called()
def test_exit_during_proc_reads_is_detected_on_the_original_pidfd(self) -> None:
def read_and_exit(_path: Path) -> bytes:
self.exited.return_value = True
return self.expected_cmdline
self.cmdline.side_effect = read_and_exit
self.assertFalse(self.manager.adopt_legacy_identity(self.vm))
self.assertIsNone(self.vm.boot_id)
self.exited.assert_called_once_with(PIDFD)
self.close.assert_called_once_with(PIDFD)
class FirecrackerProcessBoundaryTests(unittest.TestCase):
def test_public_host_boot_id_delegates_without_caching(self) -> None:
manager = FirecrackerProcessManager(Settings())
with patch.object(manager, "_boot_id", side_effect=[BOOT_ID, None]) as boot_id:
self.assertEqual(manager.host_boot_id(), BOOT_ID)
self.assertIsNone(manager.host_boot_id())
self.assertEqual(boot_id.call_args_list, [call(), call()])
def test_proc_stat_handles_spaces_and_parentheses_in_the_command(self) -> None:
stat = f"{PID} (a tricky (name) with ) parentheses) S " + " ".join(["0"] * 18 + ["42", "0"])
manager = FirecrackerProcessManager(Settings())
with patch.object(Path, "read_text", return_value=stat):
self.assertEqual(manager._process_stat(PID), ("S", "42"))
self.assertEqual(manager.process_start_time(PID), "42")
def test_unreadable_or_malformed_proc_stat_has_no_identity(self) -> None:
manager = FirecrackerProcessManager(Settings())
for stat in (
"",
"12345 broken",
"12345 (name) S 0",
"12345 (name) ? " + "0 " * 20,
"12345 name) S " + "0 " * 20,
"99999 (name) S " + "0 " * 20,
):
with self.subTest(stat=stat), patch.object(Path, "read_text", return_value=stat):
self.assertIsNone(manager.process_start_time(PID))
for error in (PermissionError(), FileNotFoundError(), UnicodeError()):
with self.subTest(error=error), patch.object(Path, "read_text", side_effect=error):
self.assertIsNone(manager.process_start_time(PID))
def test_proc_start_time_must_be_numeric(self) -> None:
stat = f"{PID} (name) S " + " ".join(["0"] * 18 + ["invalid"])
with patch.object(Path, "read_text", return_value=stat):
self.assertIsNone(FirecrackerProcessManager(Settings()).process_start_time(PID))
def test_boot_id_is_read_from_the_kernel_and_empty_or_unreadable_is_unknown(self) -> None:
with patch.object(Path, "read_text", return_value=f"{BOOT_ID}\n") as read:
self.assertEqual(FirecrackerProcessManager._boot_id(), BOOT_ID)
read.assert_called_once_with(encoding="ascii")
with patch.object(Path, "read_text", return_value="\n"):
self.assertIsNone(FirecrackerProcessManager._boot_id())
with patch.object(Path, "read_text", side_effect=PermissionError):
self.assertIsNone(FirecrackerProcessManager._boot_id())
def test_native_pidfd_bindings_are_preferred(self) -> None:
with (
patch.object(os, "pidfd_open", return_value=PIDFD, create=True) as open_pidfd,
patch.object(signal, "pidfd_send_signal", create=True) as send_signal,
patch.object(process_module, "_libc_pidfd") as libc,
):
self.assertEqual(process_module._open_pidfd(PID), PIDFD)
process_module._signal_pidfd(PIDFD, signal.SIGTERM)
open_pidfd.assert_called_once_with(PID, 0)
send_signal.assert_called_once_with(PIDFD, signal.SIGTERM)
libc.assert_not_called()
def test_libc_pidfd_bindings_cover_python_builds_without_native_bindings(self) -> None:
libc = SimpleNamespace(pidfd_open=Mock(return_value=PIDFD), pidfd_send_signal=Mock(return_value=0))
with (
patch.object(os, "pidfd_open", None, create=True),
patch.object(signal, "pidfd_send_signal", None, create=True),
patch.object(process_module.ctypes, "CDLL", return_value=libc),
):
self.assertEqual(process_module._open_pidfd(PID), PIDFD)
process_module._signal_pidfd(PIDFD, signal.SIGTERM)
libc.pidfd_open.assert_called_once_with(PID, 0)
libc.pidfd_send_signal.assert_called_once_with(PIDFD, signal.SIGTERM, None, 0)
def test_libc_pidfd_errors_preserve_errno(self) -> None:
libc = SimpleNamespace(pidfd_open=Mock(return_value=-1))
with (
patch.object(os, "pidfd_open", None, create=True),
patch.object(process_module.ctypes, "CDLL", return_value=libc),
patch.object(process_module.ctypes, "get_errno", return_value=errno.ESRCH),
):
with self.assertRaises(ProcessLookupError):
process_module._open_pidfd(PID)
def test_missing_libc_pidfd_binding_fails_closed(self) -> None:
with (
patch.object(os, "pidfd_open", None, create=True),
patch.object(process_module.ctypes, "CDLL", return_value=SimpleNamespace()),
):
with self.assertRaises(OSError) as raised:
process_module._open_pidfd(PID)
self.assertEqual(raised.exception.errno, errno.ENOSYS)
def test_invalid_pidfd_poll_events_are_errors_not_exit_confirmation(self) -> None:
poller = Mock()
poller.poll.return_value = [(PIDFD, process_module.select.POLLNVAL)]
with patch.object(process_module.select, "poll", return_value=poller):
with self.assertRaises(OSError):
process_module._pidfd_exited(PIDFD)
class FirecrackerOwnedChildTests(unittest.TestCase):
def setUp(self) -> None:
temporary_directory = tempfile.TemporaryDirectory()
self.addCleanup(temporary_directory.cleanup)
self.directory = Path(temporary_directory.name)
self.children: list[subprocess.Popen] = []
self.addCleanup(self.cleanup_children)
try:
pidfd = process_module._open_pidfd(os.getpid())
try:
process_module._signal_pidfd(pidfd, 0)
finally:
os.close(pidfd)
except (OSError, NotImplementedError) as error:
self.skipTest(f"safe pidfd support unavailable: {error}")
self.manager = FirecrackerProcessManager(
Settings(base=self.directory, api_socket_timeout_s=1, terminate_timeout_s=0.1),
popen=self.spawn_child,
)
self.vm = make_vm(self.directory, pid=None, start_time=None, boot_id=None)
self.ignore_term = False
self.create_socket = True
def spawn_child(self, command, **kwargs) -> subprocess.Popen:
script = (
"import signal, socket, sys, time\n"
"if sys.argv[2] == 'ignore': signal.signal(signal.SIGTERM, signal.SIG_IGN)\n"
"if sys.argv[3] == 'socket':\n"
" listener = socket.socket(socket.AF_UNIX)\n"
" listener.bind(sys.argv[1])\n"
" listener.listen()\n"
"time.sleep(10)\n"
)
child = subprocess.Popen(
[
sys.executable, "-B", "-c", script, command[-1],
"ignore" if self.ignore_term else "default",
"socket" if self.create_socket else "no-socket",
],
**kwargs,
)
self.children.append(child)
return child
def cleanup_children(self) -> None:
for child in self.children:
if child.poll() is None:
# These are exclusively the harmless children created by this test.
child.kill()
child.wait(timeout=5)
def test_real_child_start_stop_and_restart_keep_the_retained_disk(self) -> None:
disk = self.directory / "rootfs.ext4"
disk.write_bytes(b"retained guest writes")
published = []
for _ in range(2):
info = self.manager.start(self.vm, on_spawn=published.append)
self.assertEqual(info.pid, self.children[-1].pid)
self.assertEqual(info.boot_id, self.manager._boot_id())
self.assertEqual(self.manager.observe(self.vm), "alive")
self.assertTrue(Path(self.vm.socket).is_socket())
self.manager.terminate(self.vm)
self.assertEqual(self.manager.observe(self.vm), "gone")
self.assertIsNotNone(self.children[-1].returncode)
self.assertNotIn(info.pid, self.manager._children)
self.assertFalse(Path(self.vm.socket).exists())
self.assertEqual(disk.read_bytes(), b"retained guest writes")
self.assertEqual(len(published), 2)
self.assertNotEqual(published[0].pid, published[1].pid)
def test_real_child_is_reaped_when_persistence_fails(self) -> None:
failure = RuntimeError("save failed")
def on_spawn(info: ProcessInfo) -> None:
self.assertEqual((self.vm.pid, self.vm.process_start_time, self.vm.boot_id), (info.pid, info.start_time, info.boot_id))
raise failure
with self.assertRaises(RuntimeError) as raised:
self.manager.start(self.vm, on_spawn=on_spawn)
self.assertIs(raised.exception, failure)
self.assertEqual(self.vm.pid, self.children[0].pid)
self.assertIsNotNone(self.vm.process_start_time)
self.assertIsNotNone(self.children[0].returncode)
self.assertEqual(self.manager.observe(self.vm), "gone")
self.assertFalse(Path(self.vm.socket).exists())
def test_real_popen_exec_failure_restores_a_proven_gone_record(self) -> None:
manager = FirecrackerProcessManager(self.manager._settings)
self.vm.operation = "boot"
previous = self.vm.to_dict()
on_spawn = Mock()
with self.assertRaisesRegex(FirecrackerError, "could not start Firecracker"):
manager.start(self.vm, on_spawn=on_spawn)
self.assertEqual(self.vm.to_dict(), previous)
self.assertEqual(manager.observe(self.vm), "gone")
self.assertEqual(manager._children, {})
on_spawn.assert_not_called()
def test_real_child_ignoring_term_is_killed_and_reaped_before_socket_removal(self) -> None:
self.ignore_term = True
self.manager.start(self.vm)
self.manager.terminate(self.vm, timeout=0)
self.assertEqual(self.children[0].returncode, -signal.SIGKILL)
self.assertFalse(Path(self.vm.socket).exists())
self.assertEqual(self.manager.observe(self.vm), "gone")
def test_real_socket_timeout_cleans_up_and_reaps_the_child(self) -> None:
self.create_socket = False
self.ignore_term = True
on_spawn = Mock()
with self.assertRaisesRegex(FirecrackerError, "timed out waiting"):
self.manager.start(self.vm, on_spawn=on_spawn)
on_spawn.assert_called_once()
self.assertIsNotNone(self.vm.pid)
self.assertEqual(self.children[0].returncode, -signal.SIGKILL)
self.assertEqual(self.manager.observe(self.vm), "gone")
def test_real_unreaped_zombie_is_gone_without_signaling(self) -> None:
child = subprocess.Popen([sys.executable, "-B", "-c", "pass"])
self.children.append(child)
self.vm.pid = child.pid
self.vm.process_start_time = self.manager.process_start_time(child.pid)
self.vm.boot_id = self.manager._boot_id()
os.waitid(os.P_PID, child.pid, os.WEXITED | os.WNOWAIT)
with patch.object(process_module, "_signal_pidfd") as send_signal:
self.assertEqual(self.manager.observe(self.vm), "gone")
self.manager.terminate(self.vm)
send_signal.assert_not_called()
child.wait(timeout=5)
Reference in New Issue View Git Blame Copy Permalink