661 lines
38 KiB
Python
661 lines
38 KiB
Python
from __future__ import annotations
|
|
|
|
from dataclasses import replace
|
|
from ipaddress import IPv4Address
|
|
from pathlib import Path
|
|
from unittest.mock import patch
|
|
|
|
from tests.test_lifecycle import LifecycleTestCase
|
|
from uvm.domain import PortRule
|
|
from uvm.errors import ConflictError, FirecrackerError, NotFoundError, UvmError, ValidationError
|
|
from uvm.lifecycle import LifecycleService, OperationResult
|
|
from uvm.state import StateStore
|
|
|
|
|
|
class LifecycleLookupTests(LifecycleTestCase):
|
|
def test_vm_resources_are_distinct_and_aliases_resolve_to_the_same_record(self) -> None:
|
|
first = self.create_vm(name="web")
|
|
second = self.create_vm(name="database", guest_ip=IPv4Address("10.42.0.9"))
|
|
third = self.create_vm()
|
|
for field in ("id", "guest_ip", "mac", "tap", "pid", "disk"):
|
|
self.assertEqual(len({getattr(vm, field) for vm in (first, second, third)}), 3, field)
|
|
self.assertEqual((first.guest_ip, second.guest_ip, third.guest_ip),
|
|
("10.42.0.2", "10.42.0.9", "10.42.0.3"))
|
|
for identifier in (second.id, second.name, second.guest_ip):
|
|
with self.subTest(identifier=identifier):
|
|
self.assertEqual(self.service.inspect(identifier).vm.id, second.id)
|
|
self.assertEqual(self.service.find_for_ssh(identifier).id, second.id)
|
|
self.assertEqual(self.service.log_path(identifier), Path(second.log))
|
|
selected = self.service.resolve_targets([
|
|
second.name, first.guest_ip, second.id, first.name, third.id, second.guest_ip,
|
|
])
|
|
self.assertEqual([vm.id for vm in selected], [second.id, first.id, third.id])
|
|
self.service.stop(second.name)
|
|
self.assertEqual(self.process.observe(self.store.get(second.id)), "gone")
|
|
self.assertEqual(self.process.observe(first), "alive")
|
|
self.assertEqual(self.process.observe(third), "alive")
|
|
self.assertEqual(self.network.firewall.status(first), "enforced")
|
|
self.assertEqual(self.network.firewall.status(third), "enforced")
|
|
|
|
def test_name_and_stopped_ip_collisions_do_not_reserve_or_provision_again(self) -> None:
|
|
vm = self.create_vm(name="reserved")
|
|
self.service.stop(vm.id)
|
|
before = self.settings.state_path.read_bytes()
|
|
for kwargs, message in (
|
|
({"name": vm.name}, "VM name already exists"),
|
|
({"name": vm.id}, "must not impersonate generated VM IDs"),
|
|
({"guest_ip": IPv4Address(vm.guest_ip)}, "IP already allocated"),
|
|
):
|
|
with self.subTest(kwargs=kwargs), self.assertRaisesRegex(UvmError, message):
|
|
self.create_vm(**kwargs)
|
|
self.assertEqual(self.settings.state_path.read_bytes(), before)
|
|
self.assertEqual(self.provision.call_count, 1)
|
|
self.assertEqual(self.copy_disk.call_count, 1)
|
|
self.assertEqual(len(self.process.started), 1)
|
|
|
|
def test_generated_id_names_are_rejected_before_state_or_host_work(self) -> None:
|
|
with self.assertRaisesRegex(ValidationError, "must not impersonate generated VM IDs"):
|
|
self.create_vm(name="vm-0123456789abcdef0123456789abcdef")
|
|
self.assertFalse(self.settings.state_path.exists())
|
|
self.assertEqual(self.store.operation_locks, 0)
|
|
self.assertEqual(self.host.events, [])
|
|
self.check_kvm.assert_not_called()
|
|
self.provision.assert_not_called()
|
|
self.copy_disk.assert_not_called()
|
|
|
|
def test_generated_id_retries_id_name_and_tap_collisions(self) -> None:
|
|
with patch("uvm.lifecycle.new_vm_id", return_value="vm-first-same1234567"):
|
|
first = self.create_vm(name="reserved-name")
|
|
collision = "vm-other-same1234567"
|
|
self.assertEqual(self.network.tap_name(collision), first.tap)
|
|
with patch("uvm.lifecycle.new_vm_id", side_effect=[
|
|
first.id, first.name, collision, "vm-new-unique12345",
|
|
]) as generate:
|
|
second = self.create_vm()
|
|
self.assertEqual(generate.call_count, 4)
|
|
self.assertEqual(second.id, "vm-new-unique12345")
|
|
self.assertNotEqual(second.tap, first.tap)
|
|
self.assertNotEqual(second.guest_ip, first.guest_ip)
|
|
self.assertEqual(self.network.taps, {first.tap: first.id, second.tap: second.id})
|
|
|
|
def test_ambiguous_persisted_name_ip_or_legacy_id_never_selects_an_arbitrary_vm(self) -> None:
|
|
with patch("uvm.lifecycle.new_vm_id", return_value="vm-legacy"):
|
|
first = self.create_vm(name="first")
|
|
second = self.create_vm(name="second")
|
|
for identifier in (first.guest_ip, first.id, first.name):
|
|
with self.subTest(identifier=identifier):
|
|
self.persist(second, name=identifier)
|
|
before = self.settings.state_path.read_bytes()
|
|
self.host.events.clear()
|
|
for method in (self.service.inspect, self.service.find_for_ssh,
|
|
self.service.log_path, self.service.stop):
|
|
with self.assertRaisesRegex(ConflictError, "ambiguous VM identifier"):
|
|
method(identifier)
|
|
with self.assertRaisesRegex(ConflictError, "ambiguous VM identifier"):
|
|
self.service.resolve_targets([identifier])
|
|
self.assertEqual(self.settings.state_path.read_bytes(), before)
|
|
self.assertEqual(self.host.events, [])
|
|
|
|
def test_exact_generated_id_wins_over_a_malformed_legacy_name(self) -> None:
|
|
first = self.create_vm(name="first")
|
|
second = self.create_vm(name="second")
|
|
self.persist(second, name=first.id)
|
|
before = self.settings.state_path.read_bytes()
|
|
second_record = self.store.get(second.id).to_dict()
|
|
self.host.events.clear()
|
|
self.assertEqual(self.service.inspect(first.id).vm.id, first.id)
|
|
self.assertEqual(self.service.find_for_ssh(first.id).id, first.id)
|
|
self.assertEqual(self.service.log_path(first.id), Path(first.log))
|
|
self.assertEqual([vm.id for vm in self.service.resolve_targets([first.id])], [first.id])
|
|
self.assertEqual(self.settings.state_path.read_bytes(), before)
|
|
self.assertEqual(self.service.stop(first.id).id, first.id)
|
|
self.service.destroy(first.id)
|
|
self.assertEqual(self.store.get(second.id).to_dict(), second_record)
|
|
self.assertEqual(self.host.actions(second.id), [])
|
|
|
|
before = self.settings.state_path.read_bytes()
|
|
self.host.events.clear()
|
|
for method in (self.service.inspect, self.service.find_for_ssh,
|
|
self.service.log_path, self.service.stop):
|
|
with self.assertRaises(NotFoundError):
|
|
method(first.id)
|
|
with self.assertRaises(NotFoundError):
|
|
self.service.resolve_targets([first.id])
|
|
self.assertEqual(self.settings.state_path.read_bytes(), before)
|
|
self.assertEqual(self.host.events, [])
|
|
|
|
def test_target_resolution_rejects_invalid_or_missing_targets_without_mutation(self) -> None:
|
|
vm = self.create_vm(name="web")
|
|
before = self.settings.state_path.read_bytes()
|
|
before_locks = self.store.operation_locks
|
|
self.host.events.clear()
|
|
with self.assertRaises(ValidationError):
|
|
self.service.resolve_targets([])
|
|
with self.assertRaises(ValidationError):
|
|
self.service.resolve_targets([vm.id], all_vms=True)
|
|
with self.assertRaises(NotFoundError):
|
|
self.service.resolve_targets([vm.name, "missing"])
|
|
self.assertEqual(self.settings.state_path.read_bytes(), before)
|
|
self.assertEqual(self.store.operation_locks, before_locks)
|
|
self.assertEqual(self.host.events, [])
|
|
|
|
|
|
class LifecyclePortTests(LifecycleTestCase):
|
|
def test_set_ports_requires_an_exact_registry_id_not_a_name_or_ip(self) -> None:
|
|
vm = self.create_vm(name="web")
|
|
before = self.settings.state_path.read_bytes()
|
|
self.host.events.clear()
|
|
for identifier in (vm.name, vm.guest_ip):
|
|
with self.subTest(identifier=identifier), self.assertRaises(NotFoundError):
|
|
self.service.set_ports(identifier, ())
|
|
self.assertEqual(self.settings.state_path.read_bytes(), before)
|
|
self.assertEqual(self.host.events, [])
|
|
self.assertEqual(self.service.set_ports(vm.id, ()).ports, ())
|
|
|
|
def test_frozen_legacy_id_cannot_retarget_policy_after_name_reuse(self) -> None:
|
|
with patch("uvm.lifecycle.new_vm_id", return_value="vm-legacy"):
|
|
original = self.create_vm(name="original")
|
|
selected = self.service.inspect(original.name).vm.id
|
|
self.service.destroy(original.id)
|
|
replacement = self.create_vm(name=original.id, ports=(PortRule(22), PortRule(80)))
|
|
self.assertEqual(self.service.inspect(selected).vm.id, replacement.id)
|
|
before = self.settings.state_path.read_bytes()
|
|
before_tables = self.network.firewall.tables.copy()
|
|
self.host.events.clear()
|
|
for mode, ports in (("open", (PortRule(443),)), ("close", (PortRule(80),)), ("set", ())):
|
|
with self.subTest(mode=mode), self.assertRaises(NotFoundError):
|
|
self.service.set_ports(selected, ports, mode=mode)
|
|
self.assertEqual(self.settings.state_path.read_bytes(), before)
|
|
self.assertEqual(self.network.firewall.tables, before_tables)
|
|
self.assertEqual(self.host.events, [])
|
|
self.assertIn(replacement.tap, self.network.active_taps)
|
|
self.assertEqual(self.process.observe(replacement), "alive")
|
|
|
|
def test_stopped_port_edits_and_sync_are_metadata_only(self) -> None:
|
|
vm = self.create_vm()
|
|
stopped = self.service.stop(vm.id)
|
|
self.assertFalse(stopped.tap_owned)
|
|
for mode, ports, expected in (
|
|
("open", (PortRule(443, "HTTPS"), PortRule(80, "HTTP")),
|
|
(PortRule(22), PortRule(80, "HTTP"), PortRule(443, "HTTPS"))),
|
|
("close", (PortRule(80),), (PortRule(22), PortRule(443, "HTTPS"))),
|
|
("set", (), ()),
|
|
):
|
|
with self.subTest(mode=mode):
|
|
self.host.events.clear()
|
|
updated = self.service.set_ports(vm.id, ports, mode=mode)
|
|
self.assertEqual(updated.ports, expected)
|
|
self.assertEqual((updated.status, updated.operation, updated.pid), ("stopped", None, None))
|
|
self.assertFalse(updated.tap_owned)
|
|
self.assertEqual(self.store.get(vm.id).ports, expected)
|
|
self.assertFalse(set(self.host.actions(vm.id)) - {"process.observe", "process.adopt_legacy_identity"})
|
|
self.host.events.clear()
|
|
self.service.sync_ports(vm.id)
|
|
self.assertFalse(set(self.host.actions(vm.id)) - {"process.observe", "process.adopt_legacy_identity"})
|
|
self.assertEqual(self.network.firewall.tables, {})
|
|
self.assertEqual(len(self.process.started), 1)
|
|
self.assertEqual(self.provision.call_count, 1)
|
|
|
|
def test_live_port_modes_persist_sorted_rules_without_flushing_connections(self) -> None:
|
|
vm = self.create_vm(ports=(PortRule(22), PortRule(80, "HTTP")))
|
|
other = self.create_vm(ports=(PortRule(443, "HTTPS"),))
|
|
before_other = self.store.get(other.id).to_dict()
|
|
for mode, ports, expected in (
|
|
("open", (PortRule(443, "HTTPS"), PortRule(80)),
|
|
(PortRule(22), PortRule(80, "HTTP"), PortRule(443, "HTTPS"))),
|
|
("close", (PortRule(80), PortRule(9000)), (PortRule(22), PortRule(443, "HTTPS"))),
|
|
("set", (), ()),
|
|
):
|
|
with self.subTest(mode=mode):
|
|
self.host.events.clear()
|
|
updated = self.service.set_ports(vm.id, ports, mode=mode)
|
|
self.assertEqual(updated.ports, expected)
|
|
self.assertEqual(updated.pid, vm.pid)
|
|
self.assertEqual(updated.status, "running")
|
|
self.assertIsNone(updated.operation)
|
|
pending = self.host.records["firewall.apply", vm.id]
|
|
self.assertEqual(pending.ports, expected)
|
|
self.assertEqual(pending.operation, "policy")
|
|
self.assert_call_order(vm, "firewall.apply", "network.activate_tap")
|
|
self.assertFalse(set(self.host.actions(vm.id)) & {
|
|
"firewall.cleanup_connections", "firewall.remove", "network.delete_vm_tap",
|
|
"process.terminate", "process.start", "client.configure",
|
|
})
|
|
self.assertEqual(self.network.firewall.status(updated), "enforced")
|
|
self.assertEqual(self.store.get(other.id).to_dict(), before_other)
|
|
self.assertEqual(self.network.firewall.status(other), "enforced")
|
|
self.assertIn(other.tap, self.network.active_taps)
|
|
|
|
def test_failed_live_close_persists_pending_policy_quarantines_and_never_reopens_old_ports(self) -> None:
|
|
for failure in ("firewall.apply", "firewall.readback", "network.activate_tap"):
|
|
with self.subTest(failure=failure):
|
|
vm = self.create_vm(ports=(PortRule(22), PortRule(80, "HTTP")))
|
|
self.host.events.clear()
|
|
self.host.failures[failure, vm.id] = UvmError("policy update failed")
|
|
with self.assertRaisesRegex(UvmError, "policy pending.*policy update failed"):
|
|
self.service.set_ports(vm.id, (PortRule(80),), mode="close")
|
|
pending = self.store.get(vm.id)
|
|
self.assertEqual((pending.status, pending.operation), ("failed", "policy"))
|
|
self.assertEqual(pending.ports, (PortRule(22),))
|
|
self.assertEqual(pending.pid, vm.pid)
|
|
self.assertIn("policy update failed", pending.last_error)
|
|
self.assertEqual(self.host.records["firewall.apply", vm.id].ports, pending.ports)
|
|
self.assertEqual(self.host.records["firewall.quarantine", vm.id].ports, pending.ports)
|
|
self.assertEqual(self.network.firewall.status(pending), "quarantined")
|
|
self.assertNotIn(vm.tap, self.network.active_taps)
|
|
self.assertFalse(set(self.host.actions(vm.id)) & {
|
|
"process.terminate", "firewall.cleanup_connections", "firewall.remove",
|
|
})
|
|
self.host.events.clear()
|
|
synced = self.service.sync_ports(vm.id)
|
|
self.assertEqual(synced.ports, (PortRule(22),))
|
|
self.assertEqual((synced.status, synced.operation, synced.last_error), ("running", None, None))
|
|
self.assertEqual(self.network.firewall.status(synced), "enforced")
|
|
self.assertIn(vm.tap, self.network.active_taps)
|
|
self.assert_call_order(vm, "firewall.apply", "network.activate_tap")
|
|
|
|
def test_failed_close_still_quarantines_when_tap_deactivation_fails(self) -> None:
|
|
vm = self.create_vm(ports=(PortRule(22), PortRule(80, "HTTP")))
|
|
self.host.events.clear()
|
|
self.host.failures["firewall.apply", vm.id] = UvmError("policy rejected")
|
|
self.host.failures["network.deactivate_tap", vm.id] = UvmError("link-down failed")
|
|
with self.assertRaisesRegex(UvmError, "policy pending"):
|
|
self.service.set_ports(vm.id, (PortRule(80),), mode="close")
|
|
pending = self.store.get(vm.id)
|
|
self.assertEqual(pending.ports, (PortRule(22),))
|
|
self.assertEqual(pending.operation, "policy")
|
|
self.assertIn("link-down failed", pending.last_error)
|
|
self.assertTrue(self.network.has_tap(pending))
|
|
self.assertIn(vm.tap, self.network.active_taps)
|
|
self.assertEqual(self.network.firewall.status(pending), "quarantined")
|
|
self.assert_call_order(vm, "firewall.apply", "network.deactivate_tap", "firewall.quarantine")
|
|
|
|
def test_policy_failure_never_quarantines_a_missing_or_foreign_tap(self) -> None:
|
|
for owner in (None, "operator-owned"):
|
|
with self.subTest(owner=owner):
|
|
vm = self.create_vm(ports=(PortRule(22), PortRule(80)))
|
|
if owner is None:
|
|
del self.network.taps[vm.tap]
|
|
self.network.active_taps.discard(vm.tap)
|
|
else:
|
|
self.network.taps[vm.tap] = owner
|
|
with self.assertRaisesRegex(ConflictError, "unowned TAP"):
|
|
self.network.has_tap(vm)
|
|
before_taps = self.network.taps.copy()
|
|
before_active = self.network.active_taps.copy()
|
|
before_tables = self.network.firewall.tables.copy()
|
|
self.host.events.clear()
|
|
with self.assertRaisesRegex(UvmError, "policy pending"):
|
|
self.service.set_ports(vm.id, (PortRule(80),), mode="close")
|
|
pending = self.store.get(vm.id)
|
|
self.assertEqual((pending.ports, pending.operation), ((PortRule(22),), "policy"))
|
|
self.assertEqual(pending.pid, vm.pid)
|
|
self.assertEqual(self.network.taps, before_taps)
|
|
self.assertEqual(self.network.active_taps, before_active)
|
|
self.assertEqual(self.network.firewall.tables, before_tables)
|
|
self.assertFalse(set(self.host.actions(vm.id)) & {
|
|
"firewall.apply", "firewall.quarantine", "firewall.remove",
|
|
"firewall.cleanup_connections", "network.delete_vm_tap", "process.terminate",
|
|
})
|
|
|
|
def test_policy_fallback_rechecks_ownership_after_a_tap_is_replaced(self) -> None:
|
|
vm = self.create_vm(ports=(PortRule(22), PortRule(80)))
|
|
before_tables = self.network.firewall.tables.copy()
|
|
apply = self.network.firewall.apply
|
|
|
|
def replace_tap_and_fail(record, *, quarantine=False):
|
|
if not quarantine:
|
|
self.network.taps[record.tap] = "operator-owned"
|
|
self.host.failures["firewall.apply", record.id] = UvmError("policy rejected")
|
|
return apply(record, quarantine=quarantine)
|
|
|
|
self.host.events.clear()
|
|
with patch.object(self.network.firewall, "apply", side_effect=replace_tap_and_fail) as applied:
|
|
with self.assertRaisesRegex(UvmError, "policy pending.*unowned TAP"):
|
|
self.service.set_ports(vm.id, (PortRule(80),), mode="close")
|
|
self.assertEqual(applied.call_count, 1)
|
|
self.assertEqual(self.network.taps[vm.tap], "operator-owned")
|
|
self.assertIn(vm.tap, self.network.active_taps)
|
|
self.assertEqual(self.network.firewall.tables, before_tables)
|
|
self.assertNotIn("firewall.quarantine", self.host.actions(vm.id))
|
|
self.assertEqual(self.store.get(vm.id).ports, (PortRule(22),))
|
|
|
|
def test_failed_close_keeps_tap_down_if_quarantine_also_fails(self) -> None:
|
|
vm = self.create_vm(ports=(PortRule(22), PortRule(80)))
|
|
self.host.events.clear()
|
|
self.host.failures["firewall.apply", vm.id] = UvmError("policy rejected")
|
|
self.host.failures["firewall.quarantine", vm.id] = UvmError("quarantine rejected")
|
|
with self.assertRaisesRegex(UvmError, "policy pending"):
|
|
self.service.set_ports(vm.id, (PortRule(80),), mode="close")
|
|
pending = self.store.get(vm.id)
|
|
self.assertEqual(pending.ports, (PortRule(22),))
|
|
self.assertEqual(pending.operation, "policy")
|
|
self.assertNotIn(vm.tap, self.network.active_taps)
|
|
self.assertIn(vm.id, self.network.firewall.tables)
|
|
self.assertNotIn("network.activate_tap", self.host.actions(vm.id))
|
|
self.assertNotIn("firewall.remove", self.host.actions(vm.id))
|
|
|
|
def test_interrupted_close_retains_new_intent_and_quarantines(self) -> None:
|
|
for interrupt in (KeyboardInterrupt(), SystemExit(143)):
|
|
with self.subTest(interrupt=type(interrupt).__name__):
|
|
vm = self.create_vm(ports=(PortRule(22), PortRule(80)))
|
|
self.host.failures["firewall.apply", vm.id] = interrupt
|
|
with self.assertRaises(type(interrupt)):
|
|
self.service.set_ports(vm.id, (PortRule(80),), mode="close")
|
|
pending = self.store.get(vm.id)
|
|
self.assertEqual((pending.ports, pending.operation), ((PortRule(22),), "policy"))
|
|
self.assertNotIn(vm.tap, self.network.active_taps)
|
|
self.assertEqual(self.network.firewall.status(pending), "quarantined")
|
|
|
|
def test_legacy_policy_requires_explicit_set_while_stopped(self) -> None:
|
|
vm = self.create_vm()
|
|
self.persist(vm, ports=None)
|
|
before = self.settings.state_path.read_bytes()
|
|
for mode in ("open", "close", "set"):
|
|
with self.subTest(mode=mode), self.assertRaisesRegex(ConflictError, "legacy VM policy"):
|
|
self.service.set_ports(vm.id, (PortRule(22),), mode=mode)
|
|
with self.assertRaisesRegex(ConflictError, "legacy VM has no port policy"):
|
|
self.service.start(vm.id)
|
|
with self.assertRaisesRegex(ConflictError, "legacy VM has no managed policy"):
|
|
self.service.sync_ports(vm.id)
|
|
self.assertEqual(self.settings.state_path.read_bytes(), before)
|
|
self.service.stop(vm.id)
|
|
self.assertEqual(self.service.inspect(vm.id).policy_status, "unmanaged")
|
|
for mode in ("open", "close"):
|
|
with self.subTest(mode=mode), self.assertRaises(ConflictError):
|
|
self.service.set_ports(vm.id, (), mode=mode)
|
|
migrated = self.service.set_ports(vm.id, (), mode="set")
|
|
self.assertEqual(migrated.ports, ())
|
|
self.assertEqual(self.service.inspect(vm.id).policy_status, "inactive")
|
|
self.assertEqual(self.service.start(vm.id).status, "running")
|
|
|
|
def test_invalid_port_requests_fail_before_lock_state_or_host_mutation(self) -> None:
|
|
vm = self.create_vm()
|
|
before = self.settings.state_path.read_bytes()
|
|
before_locks = self.store.operation_locks
|
|
self.host.events.clear()
|
|
for ports, mode in (
|
|
([PortRule(22)], "set"), (("22",), "set"),
|
|
((PortRule(80, "HTTP"), PortRule(80, "TCP")), "set"),
|
|
((PortRule(22),), "replace"),
|
|
):
|
|
with self.subTest(ports=ports, mode=mode), self.assertRaises(ValidationError):
|
|
self.service.set_ports(vm.id, ports, mode=mode)
|
|
self.assertEqual(self.store.operation_locks, before_locks)
|
|
self.assertEqual(self.settings.state_path.read_bytes(), before)
|
|
self.assertEqual(self.host.events, [])
|
|
|
|
def test_start_of_running_vm_repairs_policy_and_egress_without_reboot_or_reprovision(self) -> None:
|
|
vm = self.create_vm()
|
|
del self.network.firewall.tables[vm.id]
|
|
self.host.events.clear()
|
|
started = self.service.start(vm.id)
|
|
self.assertEqual(started.pid, vm.pid)
|
|
self.assertEqual(self.network.firewall.status(started), "enforced")
|
|
self.assertNotIn("process.start", self.host.actions(vm.id))
|
|
self.assertNotIn("process.terminate", self.host.actions(vm.id))
|
|
self.assertNotIn("client.configure", self.host.actions(vm.id))
|
|
self.assertIn(("network.ensure_bridge", None), self.host.events)
|
|
self.assertLess(self.host.events.index(("firewall.apply", vm.id)),
|
|
self.host.events.index(("network.ensure_bridge", None)))
|
|
self.assertEqual(self.provision.call_count, 1)
|
|
self.assertEqual(self.copy_disk.call_count, 1)
|
|
|
|
def test_live_start_rejects_changed_network_settings_before_repairing_network(self) -> None:
|
|
vm = self.create_vm()
|
|
self.service._settings = replace(self.settings, bridge="another-bridge")
|
|
before = self.settings.state_path.read_bytes()
|
|
self.host.events.clear()
|
|
with self.assertRaisesRegex(ConflictError, "retained network"):
|
|
self.service.start(vm.id)
|
|
self.assertEqual(self.settings.state_path.read_bytes(), before)
|
|
self.assertNotIn(("network.ensure_bridge", None), self.host.events)
|
|
self.assertNotIn("firewall.apply", self.host.actions(vm.id))
|
|
|
|
def test_live_start_can_retry_an_egress_setup_failure_with_the_same_process(self) -> None:
|
|
vm = self.create_vm()
|
|
self.host.failures["network.ensure_bridge", None] = UvmError("egress setup failed")
|
|
with self.assertRaisesRegex(UvmError, "egress setup failed"):
|
|
self.service.start(vm.id)
|
|
self.assertEqual(self.process.observe(self.store.get(vm.id)), "alive")
|
|
self.assertEqual(self.network.firewall.status(vm), "enforced")
|
|
self.assertEqual(self.service.start(vm.id).pid, vm.pid)
|
|
self.assertEqual(len(self.process.started), 1)
|
|
self.assertEqual(self.provision.call_count, 1)
|
|
|
|
|
|
class LifecycleBulkTests(LifecycleTestCase):
|
|
def test_all_bulk_actions_take_one_outer_lock_and_use_only_selected_ids(self) -> None:
|
|
untouched = self.create_vm()
|
|
untouched_record = self.store.get(untouched.id).to_dict()
|
|
for action in ("start", "stop", "restart", "delete", "ports-sync"):
|
|
with self.subTest(action=action):
|
|
first, second = self.create_vm(), self.create_vm()
|
|
ids = (first.id, second.id)
|
|
if action == "start":
|
|
for vm_id in ids:
|
|
self.service.stop(vm_id)
|
|
before_locks = self.store.operation_locks
|
|
self.host.events.clear()
|
|
self.process.terminated.clear()
|
|
results = self.service.run_many(action, ids, timeout=0.5)
|
|
self.assertEqual(results, [OperationResult(vm_id, action, "succeeded") for vm_id in ids])
|
|
self.assertEqual(self.store.operation_locks - before_locks, 1)
|
|
self.assertFalse(self.store.operation_active)
|
|
self.assertFalse(self.store.transaction_active)
|
|
self.assertEqual(self.store.get(untouched.id).to_dict(), untouched_record)
|
|
self.assertEqual(self.host.actions(untouched.id), [])
|
|
if action in {"stop", "restart", "delete"}:
|
|
self.assertEqual([(vm_id, timeout) for vm_id, _, timeout in self.process.terminated
|
|
if timeout is not None], [(vm_id, 0.5) for vm_id in ids])
|
|
for vm in (first, second):
|
|
if action == "delete":
|
|
self.assertNotIn(vm.id, self.store.load().vms)
|
|
self.assertFalse(self.settings.vm_dir(vm.id).exists())
|
|
else:
|
|
current = self.store.get(vm.id)
|
|
self.assertEqual(current.status, "stopped" if action == "stop" else "running")
|
|
if action in {"start", "restart"}:
|
|
self.assertNotEqual(current.pid, vm.pid)
|
|
|
|
def test_partial_failure_reports_each_target_and_continues(self) -> None:
|
|
first, failed, last = self.create_vm(), self.create_vm(), self.create_vm()
|
|
self.host.failures["process.terminate", failed.id] = FirecrackerError("exit unconfirmed")
|
|
results = self.service.run_many("stop", (first.id, failed.id, last.id), timeout=0)
|
|
self.assertEqual([result.id for result in results], [first.id, failed.id, last.id])
|
|
self.assertEqual([result.outcome for result in results], ["succeeded", "failed", "succeeded"])
|
|
self.assertIn(failed.id, results[1].error)
|
|
self.assertIn("exit unconfirmed", results[1].error)
|
|
self.assertIsNone(results[0].error)
|
|
self.assertIsNone(results[2].error)
|
|
self.assertEqual(self.store.get(first.id).status, "stopped")
|
|
self.assertEqual(self.store.get(last.id).status, "stopped")
|
|
retained = self.store.get(failed.id)
|
|
self.assertEqual(retained.pid, failed.pid)
|
|
self.assertTrue(Path(failed.disk).exists())
|
|
self.assertIn(failed.tap, self.network.taps)
|
|
self.assertEqual(self.network.firewall.status(retained), "quarantined")
|
|
retry = self.service.run_many("stop", (failed.id,))
|
|
self.assertEqual(retry, [OperationResult(failed.id, "stop", "succeeded")])
|
|
|
|
def test_fixed_snapshot_excludes_later_vms_and_reused_name_or_ip(self) -> None:
|
|
removed = self.create_vm(name="web")
|
|
retained = self.create_vm(name="database")
|
|
selected = tuple(vm.id for vm in self.service.resolve_targets([], all_vms=True))
|
|
self.service.destroy(removed.id)
|
|
replacement = self.create_vm(name=removed.name, guest_ip=IPv4Address(removed.guest_ip))
|
|
later = self.create_vm(name="later")
|
|
before = {vm.id: self.store.get(vm.id).to_dict() for vm in (replacement, later)}
|
|
results = self.service.run_many("stop", selected)
|
|
self.assertEqual({result.id: result.outcome for result in results},
|
|
{removed.id: "skipped", retained.id: "succeeded"})
|
|
for vm in (replacement, later):
|
|
self.assertEqual(self.store.get(vm.id).to_dict(), before[vm.id])
|
|
self.assertEqual(self.process.observe(vm), "alive")
|
|
self.assertEqual(self.store.get(retained.id).status, "stopped")
|
|
|
|
def test_bulk_exact_ids_do_not_retarget_a_new_vm_named_after_a_deleted_id(self) -> None:
|
|
with patch("uvm.lifecycle.new_vm_id", return_value="vm-legacy"):
|
|
original = self.create_vm(name="original")
|
|
selected = tuple(vm.id for vm in self.service.resolve_targets([original.name]))
|
|
self.service.destroy(original.id)
|
|
replacement = self.create_vm(name=original.id)
|
|
self.assertEqual(self.service.inspect(original.id).vm.id, replacement.id)
|
|
before = self.store.get(replacement.id).to_dict()
|
|
self.host.events.clear()
|
|
results = self.service.run_many("delete", selected)
|
|
self.assertEqual(results, [OperationResult(original.id, "delete", "skipped", "already absent")])
|
|
self.assertEqual(self.store.get(replacement.id).to_dict(), before)
|
|
self.assertEqual(self.host.actions(replacement.id), [])
|
|
self.assertTrue(Path(replacement.disk).exists())
|
|
|
|
def test_interrupt_preserves_current_resources_and_skips_remaining_targets(self) -> None:
|
|
first, interrupted, last = self.create_vm(), self.create_vm(), self.create_vm()
|
|
last_record = self.store.get(last.id).to_dict()
|
|
self.host.events.clear()
|
|
self.host.failures["process.terminate", interrupted.id] = KeyboardInterrupt()
|
|
results = self.service.run_many("stop", (first.id, interrupted.id, last.id))
|
|
self.assertEqual(results, [
|
|
OperationResult(first.id, "stop", "succeeded"),
|
|
OperationResult(interrupted.id, "stop", "interrupted", "interrupted"),
|
|
OperationResult(last.id, "stop", "skipped", "not attempted after interruption"),
|
|
])
|
|
self.assertEqual(self.store.get(first.id).status, "stopped")
|
|
pending = self.store.get(interrupted.id)
|
|
self.assertEqual((pending.status, pending.operation), ("failed", "stop"))
|
|
self.assertEqual(pending.pid, interrupted.pid)
|
|
self.assertTrue(Path(interrupted.disk).exists())
|
|
self.assertIn(interrupted.tap, self.network.taps)
|
|
self.assertEqual(self.network.firewall.status(pending), "quarantined")
|
|
self.assertEqual(self.store.get(last.id).to_dict(), last_record)
|
|
self.assertEqual(self.host.actions(last.id), [])
|
|
self.assertFalse(self.store.operation_active)
|
|
self.assertEqual(self.service.stop(last.id).status, "stopped")
|
|
|
|
def test_empty_bulk_requests_have_no_vm_or_host_effects(self) -> None:
|
|
self.assertEqual(self.service.resolve_targets([], all_vms=True), [])
|
|
for action in ("start", "stop", "restart", "delete", "ports-sync"):
|
|
with self.subTest(action=action):
|
|
self.assertEqual(self.service.run_many(action, ()), [])
|
|
self.assertEqual(self.store.load().vms, {})
|
|
self.assertFalse(self.settings.state_path.exists())
|
|
self.assertEqual(self.host.events, [])
|
|
self.provision.assert_not_called()
|
|
self.copy_disk.assert_not_called()
|
|
|
|
def test_invalid_bulk_action_fails_before_lock_or_mutation(self) -> None:
|
|
with self.assertRaisesRegex(ValidationError, "invalid lifecycle action"):
|
|
self.service.run_many("create", ("vm-missing",))
|
|
self.assertEqual(self.store.operation_locks, 0)
|
|
self.assertEqual(self.host.events, [])
|
|
|
|
|
|
class LifecycleInspectionTests(LifecycleTestCase):
|
|
def test_unknown_pid_inspection_is_read_only_and_never_adopts_or_cleans_up(self) -> None:
|
|
vm = self.create_vm(name="legacy-identity")
|
|
self.persist(vm, boot_id=None)
|
|
before = self.settings.state_path.read_bytes()
|
|
before_files = {path: (path.stat().st_ino, path.read_bytes())
|
|
for path in self.settings.vm_dir(vm.id).iterdir()}
|
|
before_locks = self.store.operation_locks
|
|
self.host.events.clear()
|
|
self.require_root.reset_mock()
|
|
self.check_kvm.reset_mock()
|
|
with patch.object(self.store, "transaction", side_effect=AssertionError("read must not persist")):
|
|
inspected = self.service.inspect(vm.name)
|
|
listed, = self.service.list_vms()
|
|
self.assertEqual(self.service.find_for_ssh(vm.guest_ip).id, vm.id)
|
|
self.assertEqual(self.service.log_path(vm.id), Path(vm.log))
|
|
self.assertEqual([item.id for item in self.service.resolve_targets([vm.name])], [vm.id])
|
|
self.assertEqual(inspected.observed_status, "unknown")
|
|
self.assertEqual(listed.observed_status, "unknown")
|
|
self.assertEqual(inspected.vm.status, "running")
|
|
self.assertIsNone(inspected.vm.boot_id)
|
|
self.assertEqual(self.store.operation_locks, before_locks)
|
|
self.assertEqual(self.settings.state_path.read_bytes(), before)
|
|
self.assertEqual({path: (path.stat().st_ino, path.read_bytes())
|
|
for path in self.settings.vm_dir(vm.id).iterdir()}, before_files)
|
|
self.assertFalse(set(self.host.actions(vm.id)) - {"process.observe", "firewall.status"})
|
|
self.require_root.assert_not_called()
|
|
self.check_kvm.assert_not_called()
|
|
|
|
def test_mutation_adopts_legacy_identity_under_lock_before_termination(self) -> None:
|
|
vm = self.create_vm()
|
|
self.persist(vm, boot_id=None)
|
|
self.host.events.clear()
|
|
stopped = self.service.stop(vm.id)
|
|
self.assertEqual(stopped.status, "stopped")
|
|
self.assertEqual(self.host.records["process.terminate", vm.id].boot_id, self.process.host_boot_id())
|
|
self.assert_call_order(vm, "process.adopt_legacy_identity", "process.terminate")
|
|
|
|
def test_unknown_identity_blocks_start_and_port_edits_without_mutation(self) -> None:
|
|
vm = self.create_vm()
|
|
self.process.unknown.add(vm.id)
|
|
before = self.settings.state_path.read_bytes()
|
|
self.host.events.clear()
|
|
for method in (self.service.start, self.service.sync_ports):
|
|
with self.assertRaisesRegex(ConflictError, "unknown"):
|
|
method(vm.id)
|
|
with self.assertRaisesRegex(ConflictError, "known process identity"):
|
|
self.service.set_ports(vm.id, ())
|
|
self.assertEqual(self.settings.state_path.read_bytes(), before)
|
|
self.assertFalse(set(self.host.actions(vm.id)) - {"process.observe", "process.adopt_legacy_identity"})
|
|
|
|
def test_observed_dead_or_running_status_never_rewrites_desired_state(self) -> None:
|
|
dead = self.create_vm()
|
|
alive = self.create_vm()
|
|
del self.process.running[dead.id]
|
|
self.persist(alive, status="stopped")
|
|
before = self.settings.state_path.read_bytes()
|
|
observations = {item.vm.id: item for item in self.service.list_vms()}
|
|
self.assertEqual(observations[dead.id].observed_status, "dead")
|
|
self.assertEqual(observations[dead.id].vm.status, "running")
|
|
self.assertEqual(observations[alive.id].observed_status, "running")
|
|
self.assertEqual(observations[alive.id].vm.status, "stopped")
|
|
self.assertEqual(self.settings.state_path.read_bytes(), before)
|
|
|
|
def test_empty_inspection_does_not_create_an_uninstalled_registry(self) -> None:
|
|
settings = replace(self.settings, base=self.settings.base / "not-installed")
|
|
service = LifecycleService(settings, StateStore(settings), self.images, self.network, self.process,
|
|
known_hosts=self.known_hosts)
|
|
self.assertEqual(service.list_vms(), [])
|
|
self.assertEqual(service.resolve_targets([], all_vms=True), [])
|
|
for method in (service.inspect, service.find_for_ssh, service.log_path):
|
|
with self.assertRaises(NotFoundError):
|
|
method("missing")
|
|
self.assertFalse(settings.base.exists())
|
|
self.assertEqual(self.host.events, [])
|
|
|
|
def test_read_log_tail_is_bounded_and_marks_an_oversized_line(self) -> None:
|
|
vm = self.create_vm(name="log-reader")
|
|
path = Path(vm.log)
|
|
path.write_bytes(b"old\n" * 100_000 + b"last\n" * 2)
|
|
|
|
tail = self.service.read_log_tail(vm.id, tail=2)
|
|
|
|
self.assertEqual(tail.content, "last\n" * 2)
|
|
self.assertFalse(tail.truncated)
|
|
with self.assertRaisesRegex(ValidationError, "non-negative"):
|
|
self.service.read_log_tail(vm.id, tail=-1)
|
|
|
|
path.write_bytes(b"x" * 300_000)
|
|
truncated = self.service.read_log_tail(vm.id, tail=1)
|
|
self.assertEqual(len(truncated.content), 262_144)
|
|
self.assertTrue(truncated.truncated)
|
|
|
|
def test_diagnostics_collects_failures_without_host_commands_or_lifecycle_effects(self) -> None:
|
|
self.check_kvm.side_effect = UvmError("KVM unavailable")
|
|
self.host.failures["firewall.check_available", None] = UvmError("nft unavailable")
|
|
with patch("uvm.lifecycle.shutil.which", return_value=None):
|
|
checks = {item["name"]: item for item in self.service.diagnostics()}
|
|
self.assertTrue(checks["root"]["ok"])
|
|
self.assertTrue(checks["guest assets"]["ok"])
|
|
self.assertFalse(checks["kvm"]["ok"])
|
|
self.assertEqual(checks["kvm"]["message"], "KVM unavailable")
|
|
self.assertFalse(checks["private firewall"]["ok"])
|
|
self.assertEqual(checks["private firewall"]["message"], "nft unavailable")
|
|
self.assertFalse(checks["nft"]["ok"])
|
|
self.assertEqual(self.store.operation_locks, 0)
|
|
self.assertFalse(self.settings.state_path.exists())
|
|
self.assertEqual(self.host.events, [("firewall.check_available", None)])
|