kstyagi/nest
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: implement resource resizing for VMs with storage management

Browse Source 
- Added KnownHostsManager to handle cleanup of SSH known_hosts entries upon VM deletion.
- Introduced disk_size_bytes method in ImageStore to read logical disk capacity.
- Enhanced LifecycleService to manage VM resource resizing, including CPU, RAM, and storage.
- Updated application configuration to support known_hosts_files for SSH management.
- Modified dashboard to include resource resizing options in the UI.
- Improved error handling and validation for resource inputs in the dashboard.
- Removed obsolete .pyc files from the repository.
This commit is contained in:
kstyagi
2026-09-21 20:37:52 +05:30
parent fb80ce7c8a
commit 4a46685a4c
40 changed files with 1048 additions and 63 deletions
Download Patch File Download Diff File Expand all files Collapse all files
tests/test_cli.py
+1
View File
@@ -108,6 +108,7 @@ class CliCreateTests(unittest.TestCase):
command="create",
cpu="1",
ram="512",
storage="30G",
host_ip=None,
username="admin",
password="custom-secret",
tests/test_cli_commands.py
+34 -3
View File
@@ -30,7 +30,7 @@ class CliTestCase(unittest.TestCase):
def setUp(self) -> None:
self.vm = replace(
make_vm(), name="web", ports=(PortRule(22), PortRule(80, "HTTP")),
status="running", disk_ready=True,
status="running", disk_ready=True, storage_mib=30 * 1024,
)
self.lifecycle = create_autospec(LifecycleService, instance=True, spec_set=True)
self.listed = ListedVm(vm=self.vm, observed_status="running", policy_status="enforced")
@@ -45,8 +45,9 @@ class CliTestCase(unittest.TestCase):
]
self.lifecycle.create.side_effect = lambda spec: replace(
self.vm, cpu=spec.cpu, ram_mib=spec.ram_mib, username=spec.username,
password=spec.password, name=spec.name, ports=spec.ports,
password=spec.password, name=spec.name, ports=spec.ports, storage_mib=spec.storage_mib,
)
self.lifecycle.resize.return_value = self.vm
self.application = SimpleNamespace(
settings=Settings(), lifecycle=self.lifecycle,
state_store=Mock(spec_set=("operation_lock",)),
@@ -186,10 +187,38 @@ class CliParsingTests(CliTestCase):
self.assertEqual(self.invoke([])[0], 2)
class CliResizeCommandTests(CliTestCase):
def test_resize_forwards_resources_and_timeout_without_changing_identity(self) -> None:
self.lifecycle.resize.return_value = replace(self.vm, cpu=2, ram_mib=4096, storage_mib=50 * 1024)
code, stdout, stderr = self.invoke([
"resize", "web", "--cpu", "2", "--ram", "4G", "--storage", "50G", "--timeout", "0.5", "--json",
])
self.assertEqual(code, 0, stderr)
self.lifecycle.resize.assert_called_once_with("web", cpu=2, ram_mib=4096, storage_mib=51200, timeout=0.5)
result = json.loads(stdout)
self.assertEqual(result["storage_mib"], 51200)
self.assertEqual(result["guest_ip"], self.vm.guest_ip)
self.assertEqual(result["username"], self.vm.username)
self.assertNotIn("password", result)
self.assertNotIn(self.vm.password, stdout + stderr)
def test_partial_resize_preserves_omitted_fields(self) -> None:
code, _stdout, stderr = self.invoke(["resize", "web", "--storage", "60G"])
self.assertEqual(code, 0, stderr)
self.lifecycle.resize.assert_called_once_with("web", cpu=None, ram_mib=None, storage_mib=61440, timeout=None)
def test_invalid_resize_requests_are_rejected_before_lifecycle_calls(self) -> None:
for options in ([], ["--storage", "29G"], ["--ram", "127M"], ["--cpu", "nan"],
["--cpu", "0"], ["--cpu", "2", "--timeout", "-1"], ["--username", "new-user"]):
with self.subTest(options=options):
self.assertEqual(self.invoke(["resize", "web", *options])[0], 2)
self.lifecycle.resize.assert_not_called()
class CliCreateCommandTests(CliTestCase):
def test_create_combines_default_ports_and_stdin_credentials(self) -> None:
code, stdout, stderr = self.invoke([
"create", "--name", "web", "--cpu", "0.5", "--ram", "1G",
"create", "--name", "web", "--cpu", "0.5", "--ram", "1G", "--storage", "50G",
"--guest-ip", "10.42.0.8", "--username", "admin", "--password-stdin",
"--ports", "80:http,443:https", "--ports", "8080:tcp", "--json",
], stdin=io.StringIO(" stdin-secret \r\n"))
@@ -199,6 +228,7 @@ class CliCreateCommandTests(CliTestCase):
self.assertEqual(spec.name, "web")
self.assertEqual(spec.cpu, 0.5)
self.assertEqual(spec.ram_mib, 1024)
self.assertEqual(spec.storage_mib, 50 * 1024)
self.assertEqual(spec.guest_ip, IPv4Address("10.42.0.8"))
self.assertEqual(spec.username, "admin")
self.assertEqual(spec.password, " stdin-secret ")
@@ -222,6 +252,7 @@ class CliCreateCommandTests(CliTestCase):
code, stdout, stderr = self.invoke(["create", *options, "--json"])
self.assertEqual(code, 0, stderr)
self.assertEqual(self.lifecycle.create.call_args.args[0].ports, expected)
self.assertEqual(self.lifecycle.create.call_args.args[0].storage_mib, 30 * 1024)
self.assertNotIn("WARNING", stdout)
self.assertIn("default password", stderr)
self.assertEqual(json.loads(stdout)["policy_status"], "enforced")
tests/test_lifecycle.py
+35
View File
@@ -1,6 +1,7 @@
from __future__ import annotations
import json
import shutil
import tempfile
import unittest
from collections.abc import Callable, Iterable, Iterator
@@ -80,6 +81,16 @@ class FakeHost:
return [action for action, target in self.events if target == vm_id]
class FakeKnownHosts:
def __init__(self, host: FakeHost) -> None:
self.host = host
def remove(self, vm: VmRecord) -> None:
self.host.call("known_hosts.remove", vm.id)
if Path(vm.socket).parent.exists():
raise AssertionError("known_hosts cleanup must follow VM runtime deletion")
class FakeFirewall:
def __init__(self, host: FakeHost, network: FakeNetwork) -> None:
self.host = host
@@ -282,12 +293,26 @@ class LifecycleTestCase(unittest.TestCase):
self.store = TrackedStateStore(self.settings)
self.host = FakeHost(self.store)
self.images = ImageStore(self.settings)
self.disk_sizes: dict[Path, int] = {}
self.enterContext(patch.object(self.images, "_copy_sparse", side_effect=shutil.copy2))
self.filesystem_resize = self.enterContext(patch.object(
self.images, "_resize_filesystem",
side_effect=lambda disk, capacity: self.disk_sizes.update({disk: capacity * 1024 * 1024}),
))
self.enterContext(patch.object(
self.images, "disk_size_bytes",
side_effect=lambda disk: self.disk_sizes.get(disk, disk.stat().st_size),
))
self.resize_disk = self.enterContext(patch.object(
self.images, "resize_vm_disk", side_effect=self.resize_vm_disk,
))
self.provision = self.enterContext(patch.object(self.images, "provision_credentials"))
self.copy_disk = self.enterContext(patch.object(
self.images, "create_vm_disk", wraps=self.images.create_vm_disk,
))
self.network = FakeNetwork(self.settings, self.host)
self.process = FakeProcess(self.host)
self.known_hosts = FakeKnownHosts(self.host)
self.clients: dict[str, FakeClient] = {}
self.service = LifecycleService(
self.settings,
@@ -295,6 +320,7 @@ class LifecycleTestCase(unittest.TestCase):
self.images,
self.network, # type: ignore[arg-type]
self.process, # type: ignore[arg-type]
known_hosts=self.known_hosts, # type: ignore[arg-type]
client_factory=self.make_client, # type: ignore[arg-type]
)
self.require_root = self.enterContext(patch("uvm.lifecycle.require_root"))
@@ -307,6 +333,13 @@ class LifecycleTestCase(unittest.TestCase):
self.clients[client.vm_id] = client
return client
def resize_vm_disk(self, disk: Path, storage_mib: int) -> None:
vm_id = disk.parent.name
self.host.call("images.resize_disk", vm_id)
if self.process.observe(self.store.get(vm_id)) != "gone":
raise AssertionError("cannot resize a running or ambiguous VM")
self.disk_sizes[disk] = storage_mib * 1024 * 1024
def create_vm(self, **kwargs) -> VmRecord:
return self.service.create(VmSpec(cpu=1, ram_mib=512, **kwargs))
@@ -510,6 +543,7 @@ class LifecycleTests(LifecycleTestCase):
settings = replace(self.settings, **changed)
service = LifecycleService(
settings, self.store, self.images, self.network, self.process,
known_hosts=self.known_hosts,
client_factory=self.make_client,
)
self.host.events.clear()
@@ -542,6 +576,7 @@ class LifecycleTests(LifecycleTestCase):
settings = replace(self.settings, network=IPv4Network(network), gateway=IPv4Address(gateway))
service = LifecycleService(
settings, self.store, self.images, self.network, self.process,
known_hosts=self.known_hosts,
client_factory=self.make_client,
)
with (
tests/test_lifecycle_commands.py
+2 -1
View File
@@ -591,7 +591,8 @@ class LifecycleInspectionTests(LifecycleTestCase):
def test_empty_inspection_does_not_create_an_uninstalled_registry(self) -> None:
settings = replace(self.settings, base=self.settings.base / "not-installed")
service = LifecycleService(settings, StateStore(settings), self.images, self.network, self.process)
service = LifecycleService(settings, StateStore(settings), self.images, self.network, self.process,
known_hosts=self.known_hosts)
self.assertEqual(service.list_vms(), [])
self.assertEqual(service.resolve_targets([], all_vms=True), [])
for method in (service.inspect, service.find_for_ssh, service.log_path):
tests/test_resize.py
+195
View File
@@ -0,0 +1,195 @@
from __future__ import annotations
from dataclasses import replace
from pathlib import Path
from tests.test_lifecycle import LifecycleTestCase
from uvm.config import MIN_STORAGE_MIB
from uvm.domain import PortRule, VmResources, VmSpec
from uvm.errors import ConflictError, FirecrackerError, UvmError, ValidationError
class LifecycleResizeTests(LifecycleTestCase):
def test_creation_provisions_the_default_or_requested_storage(self) -> None:
for capacity in (MIN_STORAGE_MIB, 80 * 1024):
with self.subTest(capacity=capacity):
vm = self.create_vm(storage_mib=capacity)
self.copy_disk.assert_called_with(
self.settings.rootfs_image, Path(vm.disk), storage_mib=capacity,
)
self.filesystem_resize.assert_called_with(Path(vm.disk), capacity)
self.assertEqual(vm.storage_mib, capacity)
self.assertEqual(self.store.get(vm.id).storage_mib, capacity)
self.assertEqual(VmSpec(cpu=1, ram_mib=512).storage_mib, MIN_STORAGE_MIB)
def test_stop_resize_start_preserves_disk_data_credentials_and_network_identity(self) -> None:
vm = self.create_vm(name="database", username="admin", password="same-secret",
ports=(PortRule(22), PortRule(5432)))
sibling = self.create_vm(name="sibling")
sibling_before = self.store.get(sibling.id).to_dict()
original = replace(vm)
disk = Path(vm.disk)
contents = b"guest data\nunchanged password hash\noriginal SSH host keys\n"
disk.write_bytes(contents)
for cpu, ram_mib, storage_mib in ((2, 4096, 50 * 1024), (0.5, 256, MIN_STORAGE_MIB)):
with self.subTest(cpu=cpu, storage=storage_mib):
previous_pid = self.store.get(vm.id).pid
locks = self.store.operation_locks
self.host.events.clear()
resized = self.service.resize(vm.name, cpu=cpu, ram_mib=ram_mib,
storage_mib=storage_mib, timeout=0.5)
self.assertEqual(self.store.operation_locks - locks, 1)
self.assertEqual((resized.cpu, resized.ram_mib, resized.storage_mib), (cpu, ram_mib, storage_mib))
self.assertEqual(resized.status, "running")
self.assertIsNone(resized.pending_resize)
self.assertIsNone(resized.operation)
self.assertNotEqual(resized.pid, previous_pid)
for field in ("id", "name", "guest_ip", "gateway", "network", "bridge", "mac", "tap",
"disk", "username", "password", "ports", "created_at"):
self.assertEqual(getattr(resized, field), getattr(original, field), field)
self.assertEqual(disk.read_bytes(), contents)
actions = self.host.actions(vm.id)
self.assertLess(actions.index("process.gone"), actions.index("images.resize_disk"))
self.assertLess(actions.index("images.resize_disk"), actions.index("process.start"))
stopped = self.host.records["process.terminate", vm.id]
self.assertEqual(stopped.pending_resize, VmResources(cpu, ram_mib, storage_mib))
self.assertEqual(stopped.operation, "resize")
config = self.clients[vm.id].config
self.assertEqual(config["machine-config"], {"vcpu_count": max(1, int(cpu)), "mem_size_mib": ram_mib})
self.assertEqual(config["drives"][0]["path_on_host"], vm.disk)
self.assertEqual(self.store.get(vm.id).to_dict(), resized.to_dict())
self.assertEqual(self.store.get(sibling.id).to_dict(), sibling_before)
self.assertEqual(self.copy_disk.call_count, 2)
self.assertEqual(self.provision.call_count, 2)
self.assertEqual(self.resize_disk.call_count, 2)
def test_cpu_or_memory_only_resize_keeps_storage_and_boots_an_initially_stopped_vm(self) -> None:
vm = self.create_vm(storage_mib=50 * 1024)
self.service.stop(vm.id)
first = self.service.resize(vm.guest_ip, cpu=2)
self.assertEqual((first.cpu, first.ram_mib, first.storage_mib), (2, 512, 50 * 1024))
second = self.service.resize(vm.id, ram_mib=4096)
self.assertEqual((second.cpu, second.ram_mib, second.storage_mib), (2, 4096, 50 * 1024))
self.assertEqual(second.status, "running")
self.resize_disk.assert_not_called()
self.assertEqual(self.provision.call_count, 1)
def test_invalid_resize_requests_do_not_change_state_or_stop_the_machine(self) -> None:
vm = self.create_vm()
before = self.settings.state_path.read_bytes()
self.host.events.clear()
for changes in ({}, {"cpu": 0}, {"cpu": float("nan")}, {"cpu": True},
{"ram_mib": 127}, {"ram_mib": 512.0}, {"storage_mib": 29 * 1024},
{"storage_mib": True}, {"storage_mib": 2**63}, {"cpu": 2, "timeout": -1}):
with self.subTest(changes=changes), self.assertRaises(ValidationError):
self.service.resize(vm.id, **changes)
self.assertEqual(self.settings.state_path.read_bytes(), before)
self.assertEqual(self.host.events, [])
self.resize_disk.assert_not_called()
def test_resize_rejects_unsafe_or_unbootable_vm_before_stopping(self) -> None:
for problem in ("unknown", "delete", "legacy", "unprovisioned", "missing", "topology", "kernel"):
with self.subTest(problem=problem):
vm = self.create_vm()
if problem == "unknown":
self.process.unknown.add(vm.id)
elif problem == "delete":
self.persist(vm, operation="delete")
elif problem == "legacy":
self.persist(vm, ports=None)
elif problem == "unprovisioned":
self.persist(vm, disk_ready=False)
elif problem == "missing":
Path(vm.disk).unlink()
elif problem == "topology":
self.persist(vm, bridge="different")
else:
self.settings.kernel_image.unlink()
before = self.settings.state_path.read_bytes()
self.host.events.clear()
with self.assertRaises(UvmError):
self.service.resize(vm.id, cpu=2)
self.assertEqual(self.settings.state_path.read_bytes(), before)
self.assertNotIn("process.terminate", self.host.actions(vm.id))
self.resize_disk.assert_not_called()
if problem == "topology":
self.persist(vm, bridge="uvm0")
if problem == "kernel":
self.settings.kernel_image.write_bytes(b"kernel")
def test_failed_stop_keeps_old_resources_and_persists_retryable_target(self) -> None:
vm = self.create_vm()
self.host.failures["process.terminate", vm.id] = FirecrackerError("exit unconfirmed")
with self.assertRaisesRegex(UvmError, "failed to resize.*exit unconfirmed"):
self.service.resize(vm.id, cpu=2, ram_mib=4096, storage_mib=50 * 1024)
retained = self.store.get(vm.id)
self.assertEqual((retained.cpu, retained.ram_mib, retained.storage_mib), (1, 512, MIN_STORAGE_MIB))
self.assertEqual(retained.pending_resize, VmResources(2, 4096, 50 * 1024))
self.assertEqual(retained.pid, vm.pid)
self.resize_disk.assert_not_called()
self.assertEqual(len(self.process.started), 1)
with self.assertRaisesRegex(ConflictError, "resize is pending"):
self.service.start(vm.id)
retried = self.service.resize(vm.id, cpu=2)
self.assertEqual((retried.cpu, retried.ram_mib, retried.storage_mib), (2, 4096, 50 * 1024))
def test_failed_disk_resize_is_stopped_and_start_retries_persisted_target(self) -> None:
vm = self.create_vm(storage_mib=50 * 1024)
self.host.failures["images.resize_disk", vm.id] = UvmError("filesystem does not fit")
with self.assertRaisesRegex(UvmError, "filesystem does not fit"):
self.service.resize(vm.id, cpu=2, storage_mib=MIN_STORAGE_MIB)
retained = self.store.get(vm.id)
self.assertEqual((retained.cpu, retained.storage_mib), (1, 50 * 1024))
self.assertEqual(retained.pending_resize, VmResources(2, 512, MIN_STORAGE_MIB))
self.assertIsNone(retained.pid)
self.assertEqual(len(self.process.started), 1)
self.assertEqual(Path(vm.disk).read_bytes(), b"rootfs")
restarted = self.service.start(vm.id)
self.assertEqual((restarted.cpu, restarted.storage_mib), (2, MIN_STORAGE_MIB))
self.assertIsNone(restarted.pending_resize)
self.assertEqual(self.provision.call_count, 1)
def test_failed_boot_retains_applied_resources_for_a_later_start(self) -> None:
vm = self.create_vm()
self.host.failures["client.configure", vm.id] = FirecrackerError("boot failed")
with self.assertRaisesRegex(UvmError, "boot failed"):
self.service.resize(vm.id, cpu=2, storage_mib=50 * 1024)
retained = self.store.get(vm.id)
self.assertEqual((retained.cpu, retained.storage_mib), (2, 50 * 1024))
self.assertIsNone(retained.pending_resize)
self.assertTrue(retained.disk_ready)
restarted = self.service.start(vm.id)
self.assertEqual(restarted.status, "running")
self.resize_disk.assert_called_once_with(Path(vm.disk), 50 * 1024)
def test_legacy_disk_gets_minimum_capacity_on_its_first_resource_resize(self) -> None:
vm = self.create_vm()
self.persist(vm, storage_mib=None)
resized = self.service.resize(vm.id, cpu=2)
self.assertEqual(resized.storage_mib, MIN_STORAGE_MIB)
self.resize_disk.assert_called_once_with(Path(vm.disk), MIN_STORAGE_MIB)
def test_retry_can_change_target_after_disk_publication_before_metadata_save(self) -> None:
for recorded_gib, published_gib in ((30, 50), (50, 30)):
with self.subTest(recorded=recorded_gib, published=published_gib):
vm = self.create_vm(storage_mib=recorded_gib * 1024)
self.service.stop(vm.id)
self.persist(vm, operation="resize", pending_resize=VmResources(2, 4096, published_gib * 1024))
self.disk_sizes[Path(vm.disk)] = published_gib * 1024**3
self.resize_disk.reset_mock()
result = self.service.resize(vm.id, storage_mib=recorded_gib * 1024)
self.resize_disk.assert_called_once_with(Path(vm.disk), recorded_gib * 1024)
self.assertEqual(result.storage_mib * 1024 * 1024, self.disk_sizes[Path(vm.disk)])
self.assertIsNone(result.pending_resize)
def test_interrupted_resize_can_be_stopped_then_resumed_or_deleted(self) -> None:
vm = self.create_vm()
self.host.failures["images.resize_disk", vm.id] = KeyboardInterrupt()
with self.assertRaises(KeyboardInterrupt):
self.service.resize(vm.id, storage_mib=50 * 1024)
stopped = self.service.stop(vm.id)
self.assertEqual(stopped.pending_resize.storage_mib, 50 * 1024)
self.assertEqual(stopped.operation, "resize")
self.service.destroy(vm.id)
self.assertNotIn(vm.id, self.store.load().vms)
self.assertFalse(Path(vm.disk).exists())
tests/test_server.py
+80 -2
View File
@@ -11,7 +11,7 @@ from unittest.mock import patch
from uvm.cli import build_parser, main
from uvm.config import Settings
from uvm.domain import PortRule, VmRecord, VmSpec
from uvm.domain import PortRule, VmRecord, VmResources, VmSpec
from uvm.errors import ConfigurationError, ConflictError, NotFoundError, UvmError, ValidationError
from uvm.lifecycle import ListedVm, OperationResult
from uvm.server import _http_status_for, create_api, run_server
@@ -22,6 +22,7 @@ def fake_application(*, api_token: str | None = None):
app_name="uvm",
default_vcpu=1,
default_ram_mib=512,
default_storage_mib=30 * 1024,
default_ssh_user="root",
api_token=api_token,
api_tls_cert=None,
@@ -145,6 +146,7 @@ if FASTAPI_AVAILABLE:
class FakeLifecycle:
def __init__(self) -> None:
self.last_spec: VmSpec | None = None
self.last_resize: dict | None = None
self.observed_status = "running"
self.policy_status = "enforced"
self.vm = VmRecord(
@@ -152,6 +154,7 @@ class FakeLifecycle:
name="test-vm",
cpu=1,
ram_mib=512,
storage_mib=30 * 1024,
guest_ip="10.42.0.2",
gateway="10.42.0.1",
tap="uvm-test",
@@ -187,6 +190,7 @@ class FakeLifecycle:
self.last_spec = spec
self.vm.cpu = spec.cpu
self.vm.ram_mib = spec.ram_mib
self.vm.storage_mib = spec.storage_mib
if spec.guest_ip is not None:
self.vm.guest_ip = str(spec.guest_ip)
self.vm.username = spec.username
@@ -222,6 +226,16 @@ class FakeLifecycle:
self.policy_status = "inactive"
return self.vm
def resize(self, identifier: str, *, cpu=None, ram_mib=None, storage_mib=None, timeout=None) -> VmRecord:
self.inspect(identifier)
self.last_resize = dict(cpu=cpu, ram_mib=ram_mib, storage_mib=storage_mib, timeout=timeout)
for key, value in (("cpu", cpu), ("ram_mib", ram_mib), ("storage_mib", storage_mib)):
if value is not None:
setattr(self.vm, key, value)
self.vm.pending_resize = None
self.vm.operation = None
return self.start(identifier)
def destroy(self, identifier: str, *, timeout: float | None = None) -> VmRecord:
self.inspect(identifier)
return self.vm
@@ -319,6 +333,7 @@ class ApiRouterTests(unittest.TestCase):
("POST", "/vms/actions", {"action": "start", "identifiers": ["vm-test"]}),
("POST", "/vms/test-vm/start", {}),
("POST", "/vms/test-vm/restart", {}),
("POST", "/vms/test-vm/resize", {"cpu": 2}),
("POST", "/vms/test-vm/stop", {}),
("DELETE", "/vms/test-vm", {}),
("POST", "/vms/test-vm/ports", {}),
@@ -376,6 +391,68 @@ class ApiRouterTests(unittest.TestCase):
self.assertEqual(self.lifecycle.last_spec.password, "root")
self.assertIsNone(self.lifecycle.last_spec.name)
self.assertEqual(self.lifecycle.last_spec.ports, (PortRule(22),))
self.assertEqual(self.lifecycle.last_spec.storage_mib, 30 * 1024)
self.assertEqual(response.json()["storage_mib"], 30 * 1024)
def test_create_accepts_custom_storage_and_rejects_capacities_below_minimum(self) -> None:
for capacity, expected in ((50, 50 * 1024), ("50G", 50 * 1024), ("1T", 1024 * 1024)):
with self.subTest(capacity=capacity):
response = self.client.post("/vms", headers=self.headers, json={"storage": capacity})
self.assertEqual(response.status_code, 201, response.text)
self.assertEqual(self.lifecycle.last_spec.storage_mib, expected)
self.assertEqual(response.json()["storage_mib"], expected)
self.lifecycle.last_spec = None
for capacity in ("29G", 29, "30M", "bad", True, None, [], {}):
with self.subTest(capacity=capacity):
response = self.client.post("/vms", headers=self.headers, json={"storage": capacity})
self.assertEqual(response.status_code, 422, response.text)
self.assertIsNone(self.lifecycle.last_spec)
def test_resize_accepts_partial_resources_and_returns_the_same_vm_identity(self) -> None:
before = self.lifecycle.vm.to_dict()
response = self.client.post("/vms/test-vm/resize?timeout=0.5", headers=self.headers,
json={"cpu": 2, "ram": "4G", "storage": "50G"})
self.assertEqual(response.status_code, 200, response.text)
self.assertEqual(self.lifecycle.last_resize, {"cpu": 2, "ram_mib": 4096, "storage_mib": 51200, "timeout": 0.5})
for field in ("id", "name", "guest_ip", "gateway", "mac", "username"):
self.assertEqual(response.json()[field], before[field])
self.assertEqual(response.json()["status"], "running")
self.assertEqual(response.json()["policy_status"], "enforced")
self.assertEqual(response.json()["storage_mib"], 51200)
self.assertIsNone(response.json()["pending_resize"])
self.assertNotIn("password", response.json())
self.assertEqual(self.lifecycle.vm.password, before["password"])
partial = self.client.post("/vms/10.42.0.2/resize", headers=self.headers, json={"ram": "2G"})
self.assertEqual(partial.status_code, 200)
self.assertEqual(self.lifecycle.last_resize, {"cpu": None, "ram_mib": 2048, "storage_mib": None, "timeout": None})
self.assertEqual(partial.json()["storage_mib"], 51200)
def test_resize_rejects_invalid_or_non_resource_edits_before_lifecycle_calls(self) -> None:
for payload in ({}, {"cpu": None}, {"cpu": 0}, {"cpu": True}, {"ram": "127M"},
{"storage": "29G"}, {"storage": False}, {"storage": []},
{"cpu": 2, "username": "new-user"}, {"cpu": 2, "password": "new-secret"},
{"cpu": 2, "guest_ip": "10.42.0.9"}):
with self.subTest(payload=payload):
response = self.client.post("/vms/vm-test/resize", headers=self.headers, json=payload)
self.assertEqual(response.status_code, 422, response.text)
self.assertIsNone(self.lifecycle.last_resize)
for timeout in ("-1", "nan", "inf"):
response = self.client.post(f"/vms/vm-test/resize?timeout={timeout}", headers=self.headers, json={"cpu": 2})
self.assertEqual(response.status_code, 422)
self.assertIsNone(self.lifecycle.last_resize)
def test_resize_errors_and_pending_resources_are_exposed_without_credentials(self) -> None:
for error, expected in ((NotFoundError("missing"), 404), (ConflictError("unknown process"), 409),
(UvmError("filesystem does not fit"), 500)):
with patch.object(self.lifecycle, "resize", side_effect=error):
response = self.client.post("/vms/vm-test/resize", headers=self.headers, json={"storage": "30G"})
self.assertEqual(response.status_code, expected)
self.assertEqual(response.json(), {"error": {"message": str(error)}})
target = VmResources(2, 4096, 51200)
self.lifecycle.vm.pending_resize = target
response = self.client.get("/vms/vm-test", headers=self.headers)
self.assertEqual(response.json()["pending_resize"], target.to_dict())
self.assertNotIn("guest-secret", response.text)
def test_create_accepts_valid_names(self) -> None:
for name in (None, "a", "web-1", "a" * 63):
@@ -454,7 +531,7 @@ class ApiRouterTests(unittest.TestCase):
def test_list_and_get_whitelist_fields_and_preserve_observed_policy(self) -> None:
expected_fields = {
"id", "name", "cpu", "ram_mib", "guest_ip", "gateway", "mac", "username",
"id", "name", "cpu", "ram_mib", "storage_mib", "pending_resize", "guest_ip", "gateway", "mac", "username",
"status", "observed_status", "ports", "policy_status", "operation", "pid",
"created_at", "updated_at", "last_error",
}
@@ -683,6 +760,7 @@ class ApiRouterTests(unittest.TestCase):
("/vms/{identifier}/logs", "get"),
("/vms/{vm_id}/start", "post"),
("/vms/{vm_id}/restart", "post"),
("/vms/{vm_id}/resize", "post"),
("/vms/{vm_id}/stop", "post"),
("/vms/{vm_id}", "delete"),
("/vms/{vm_id}/ports", "post"),
tests/test_state.py
+26 -11
View File
@@ -9,7 +9,7 @@ from pathlib import Path
from unittest.mock import patch
from uvm.config import Settings
from uvm.domain import STATE_SCHEMA_VERSION, PortRule, State, VmRecord, VmSpec, new_vm_id
from uvm.domain import STATE_SCHEMA_VERSION, PortRule, State, VmRecord, VmResources, VmSpec, new_vm_id
from uvm.errors import ConflictError, NotFoundError, StateError
from uvm.state import StateStore
@@ -61,13 +61,13 @@ class StateStoreTests(unittest.TestCase):
def test_legacy_state_is_loaded_and_migrated_on_next_write(self) -> None:
self.base.mkdir(parents=True)
for version in (None, 1):
for version in (None, 1, 2):
with self.subTest(version=version):
raw_vm = make_vm().to_dict()
for field in (
"disk", "username", "password", "updated_at", "name", "ports",
"disk_ready", "runtime_owned", "tap_owned", "operation", "boot_id",
"spawn_pending", "network", "bridge",
"spawn_pending", "network", "bridge", "storage_mib", "pending_resize",
):
raw_vm.pop(field)
legacy = {"vms": {"vm-test": raw_vm}}
@@ -77,7 +77,7 @@ class StateStoreTests(unittest.TestCase):
loaded = self.store.load()
vm = loaded.vms["vm-test"]
self.assertEqual(loaded.schema_version, 2)
self.assertEqual(loaded.schema_version, 3)
self.assertEqual(vm.disk, "")
self.assertEqual(vm.username, "root")
self.assertIsNone(vm.password)
@@ -85,6 +85,8 @@ class StateStoreTests(unittest.TestCase):
self.assertIsNone(vm.ports)
self.assertIsNone(vm.network)
self.assertIsNone(vm.bridge)
self.assertIsNone(vm.storage_mib)
self.assertIsNone(vm.pending_resize)
self.assertFalse(vm.disk_ready)
self.assertTrue(vm.runtime_owned)
self.assertFalse(vm.tap_owned)
@@ -99,7 +101,7 @@ class StateStoreTests(unittest.TestCase):
state.vms["vm-test"].status = "stopped"
persisted = json.loads(self.settings.state_path.read_text(encoding="utf-8"))
self.assertEqual(persisted["schema_version"], 2)
self.assertEqual(persisted["schema_version"], 3)
self.assertEqual(persisted["next_mac_index"], 2)
self.assertEqual(persisted["vms"]["vm-test"]["status"], "stopped")
self.assertIsNone(persisted["vms"]["vm-test"]["ports"])
@@ -114,16 +116,17 @@ class StateStoreTests(unittest.TestCase):
runtime_owned=True, tap_owned=True, operation="configuring",
boot_id="host-boot-id", spawn_pending=True, pid=123,
process_start_time="456789", network="10.42.0.0/24", bridge="uvm0",
last_error="interrupted boot",
last_error="interrupted boot", storage_mib=30 * 1024,
pending_resize=VmResources(cpu=2, ram_mib=4096, storage_mib=50 * 1024),
)
with self.store.transaction() as state:
state.vms[vm.id] = vm
loaded = self.store.load()
self.assertEqual(loaded.schema_version, 2)
self.assertEqual(loaded.schema_version, 3)
self.assertEqual(loaded.vms[vm.id], vm)
raw = json.loads(self.settings.state_path.read_text(encoding="utf-8"))
self.assertEqual(raw["schema_version"], 2)
self.assertEqual(raw["schema_version"], 3)
self.assertEqual(
raw["vms"][vm.id]["ports"],
None if ports is None else [port.to_dict() for port in ports],
@@ -151,7 +154,7 @@ class StateStoreTests(unittest.TestCase):
def test_load_rejects_a_record_whose_id_does_not_match_its_key(self) -> None:
self.base.mkdir(parents=True)
for version in (0, 1, 2):
for version in (0, 1, 2, 3):
with self.subTest(version=version):
raw = {
"schema_version": version,
@@ -302,11 +305,23 @@ class DomainStateTests(unittest.TestCase):
self.assertIs(vm.spawn_pending, operation == "spawning")
self.assertEqual(VmRecord.from_dict(vm.to_dict()), vm)
def test_new_state_uses_schema_two_and_rejects_future_versions(self) -> None:
self.assertEqual(State().schema_version, 2)
def test_new_state_uses_schema_three_and_rejects_future_versions(self) -> None:
self.assertEqual(State().schema_version, 3)
with self.assertRaisesRegex(StateError, "newer than"):
State.from_dict({"schema_version": STATE_SCHEMA_VERSION + 1, "vms": {}})
def test_storage_and_pending_resize_reject_malformed_persisted_values(self) -> None:
raw = make_vm().to_dict()
for value in (0, 1024, True, "30720", 30720.0, -1, {}, [], 2**63):
with self.subTest(storage=value), self.assertRaises(StateError):
VmRecord.from_dict({**raw, "storage_mib": value})
valid = {"cpu": 2, "ram_mib": 4096, "storage_mib": 50 * 1024}
for value in ({}, [], "resize", {**valid, "cpu": True}, {**valid, "cpu": float("nan")},
{**valid, "ram_mib": 127}, {**valid, "storage_mib": 29 * 1024},
{**valid, "password": "not-a-resource"}):
with self.subTest(pending=value), self.assertRaises(StateError):
VmRecord.from_dict({**raw, "pending_resize": value})
def test_generated_ids_have_the_reserved_uuid_format(self) -> None:
first, second = new_vm_id(), new_vm_id()
self.assertNotEqual(first, second)
tests/test_storage.py
+155
View File
@@ -0,0 +1,155 @@
from __future__ import annotations
import hashlib
import shutil
import tempfile
import unittest
from pathlib import Path
from unittest.mock import Mock, patch
from uvm.config import MIN_STORAGE_MIB, Settings
from uvm.errors import UvmError, ValidationError
from uvm.images import ImageStore
from uvm.system import CommandResult, CommandRunner
MIB = 1024 * 1024
class DiskResizeTests(unittest.TestCase):
def test_grow_extends_the_file_first_and_shrink_truncates_only_after_verified_resize(self) -> None:
for original_mib, target_mib in ((8, 16), (32, 16)):
with self.subTest(original=original_mib), tempfile.TemporaryDirectory() as temporary:
disk = Path(temporary) / "rootfs.ext4"
with disk.open("wb") as image:
image.truncate(original_mib * MIB)
sizes = []
def run(command, **_options):
sizes.append((command[0], disk.stat().st_size))
return CommandResult(args=tuple(map(str, command)), returncode=1 if command[0] == "e2fsck" else 0)
runner = Mock(spec=CommandRunner)
runner.run.side_effect = run
store = ImageStore(Settings(), runner)
with patch.object(store, "_filesystem_size", return_value=target_mib * MIB):
store._resize_filesystem(disk, target_mib)
self.assertEqual(sizes, [("e2fsck", original_mib * MIB), ("resize2fs", max(original_mib, target_mib) * MIB)])
self.assertEqual(disk.stat().st_size, target_mib * MIB)
def test_failed_or_interrupted_staging_never_changes_the_original_disk(self) -> None:
for failure in (UvmError("filesystem cannot fit"), KeyboardInterrupt()):
with self.subTest(failure=type(failure)), tempfile.TemporaryDirectory() as temporary:
disk = Path(temporary) / "rootfs.ext4"
disk.write_bytes(b"data, password hash, and SSH host keys")
disk.chmod(0o600)
before = (disk.stat().st_ino, disk.read_bytes())
store = ImageStore(Settings(base=Path(temporary)), Mock(spec=CommandRunner))
def fail(candidate, _capacity):
self.assertNotEqual(candidate, disk)
candidate.write_bytes(b"partly resized staging image")
raise failure
with (
patch.object(store, "_copy_sparse", side_effect=shutil.copyfile),
patch.object(store, "_resize_filesystem", side_effect=fail),
self.assertRaises(type(failure)),
):
store.resize_vm_disk(disk, MIN_STORAGE_MIB)
self.assertEqual((disk.stat().st_ino, disk.read_bytes()), before)
self.assertEqual(list(disk.parent.iterdir()), [disk])
def test_filesystem_check_errors_prevent_resize_and_truncation(self) -> None:
with tempfile.TemporaryDirectory() as temporary:
disk = Path(temporary) / "rootfs.ext4"
disk.write_bytes(b"original")
runner = Mock(spec=CommandRunner)
runner.run.return_value = CommandResult(("e2fsck",), 4, stderr="unrepaired errors")
store = ImageStore(Settings(), runner)
with self.assertRaisesRegex(UvmError, "filesystem check failed"):
store._resize_filesystem(disk, 16)
self.assertEqual(disk.read_bytes(), b"original")
self.assertEqual(runner.run.call_count, 1)
def test_geometry_mismatch_prevents_truncating_a_larger_filesystem(self) -> None:
with tempfile.TemporaryDirectory() as temporary:
disk = Path(temporary) / "rootfs.ext4"
with disk.open("wb") as image:
image.truncate(32 * MIB)
runner = Mock(spec=CommandRunner)
runner.run.return_value = CommandResult(("tool",), 0)
store = ImageStore(Settings(), runner)
with (
patch.object(store, "_filesystem_size", return_value=32 * MIB),
self.assertRaisesRegex(UvmError, "does not match"),
):
store._resize_filesystem(disk, 16)
self.assertEqual(disk.stat().st_size, 32 * MIB)
def test_retry_after_disk_publication_recognizes_the_completed_capacity(self) -> None:
with tempfile.TemporaryDirectory() as temporary:
disk = Path(temporary) / "rootfs.ext4"
with disk.open("wb") as image:
image.truncate(MIN_STORAGE_MIB * MIB)
runner = Mock(spec=CommandRunner)
store = ImageStore(Settings(), runner)
with patch.object(store, "_filesystem_size", return_value=MIN_STORAGE_MIB * MIB):
store.resize_vm_disk(disk, MIN_STORAGE_MIB)
runner.run.assert_not_called()
def test_resize_rejects_subminimum_capacity_and_shared_or_linked_disks(self) -> None:
with tempfile.TemporaryDirectory() as temporary:
settings = Settings(base=Path(temporary))
settings.images_dir.mkdir()
settings.rootfs_image.write_bytes(b"template")
disk = Path(temporary) / "rootfs.ext4"
disk.write_bytes(b"private disk")
linked = Path(temporary) / "linked.ext4"
linked.hardlink_to(disk)
symlink = Path(temporary) / "symlink.ext4"
symlink.symlink_to(disk)
runner = Mock(spec=CommandRunner)
store = ImageStore(settings, runner)
with self.assertRaises(ValidationError):
store.resize_vm_disk(disk, MIN_STORAGE_MIB - 1)
for target in (settings.rootfs_image, linked, symlink):
with self.subTest(target=target), self.assertRaisesRegex(UvmError, "unsafe"):
store.resize_vm_disk(target, MIN_STORAGE_MIB)
runner.run.assert_not_called()
@unittest.skipUnless(all(shutil.which(tool) for tool in ("mkfs.ext4", "e2fsck", "resize2fs", "dumpe2fs", "debugfs", "cp")),
"e2fsprogs and coreutils are required for offline disk checks")
class OfflineExt4ResizeTests(unittest.TestCase):
def test_real_sparse_disk_grows_and_shrinks_with_guest_data_and_keys_intact(self) -> None:
with tempfile.TemporaryDirectory() as temporary:
base = Path(temporary)
source = base / "template.ext4"
root = base / "guest"
root.mkdir()
contents = {
"data": "retained application data\n",
"shadow": "root:$6$unchanged-password-hash:1:0:99999:7:::\n",
"host-key": "unchanged-private-host-key\n",
}
for name, value in contents.items():
(root / name).write_text(value, encoding="utf-8")
with source.open("wb") as image:
image.truncate(32 * MIB)
runner = CommandRunner(emit=None)
runner.run(("mkfs.ext4", "-q", "-F", "-b", "4096", "-N", "128", "-d", root, source), capture=True)
template_digest = hashlib.sha256(source.read_bytes()).hexdigest()
store = ImageStore(Settings(base=base), runner)
disk = store.create_vm_disk(source, base / "rootfs.ext4")
for capacity in (MIN_STORAGE_MIB, 31 * 1024, MIN_STORAGE_MIB):
store.resize_vm_disk(disk, capacity)
self.assertEqual(disk.stat().st_size, capacity * MIB)
self.assertEqual(store._filesystem_size(disk), capacity * MIB)
self.assertEqual(disk.stat().st_mode & 0o777, 0o600)
self.assertLess(disk.stat().st_blocks * 512, disk.stat().st_size // 2)
runner.run(("e2fsck", "-f", "-n", disk), capture=True)
for name, value in contents.items():
result = runner.run(("debugfs", "-R", f"cat /{name}", disk), capture=True)
self.assertEqual(result.stdout, value)
self.assertEqual(hashlib.sha256(source.read_bytes()).hexdigest(), template_digest)
tests/test_validation.py
+16 -1
View File
@@ -5,11 +5,26 @@ import unittest
from uvm.domain import PortRule
from uvm.errors import ValidationError
from uvm.validation import (
parse_cpu, parse_name, parse_password, parse_ports, parse_ram, parse_timeout, parse_username,
parse_cpu, parse_name, parse_password, parse_ports, parse_ram, parse_storage, parse_timeout, parse_username,
)
class ValidationTests(unittest.TestCase):
def test_storage_defaults_to_gib_and_supports_larger_and_fractional_units(self) -> None:
for value, expected in (
("30", 30720), ("30G", 30720), ("30GB", 30720), ("30GiB", 30720),
(" 50 g ", 51200), ("30720M", 30720), ("30.5G", 31232),
("1T", 1048576), ("0.5TiB", 524288), ("30.0001G", 30721),
):
with self.subTest(value=value):
self.assertEqual(parse_storage(value), expected)
def test_storage_rejects_subminimum_invalid_and_unrepresentable_capacity(self) -> None:
for value in ("0", "29G", "29.999999G", "30719M", "30M", "", "-30G", "nan",
"inf", "True", "30G garbage", "100000000000000000000000T"):
with self.subTest(value=value), self.assertRaises(ValidationError):
parse_storage(value)
def test_parse_ram_uses_mib_by_default(self) -> None:
self.assertEqual(parse_ram("512"), 512)
self.assertEqual(parse_ram("1G"), 1024)