Enhance UVM installation and configuration for Ubuntu 22.04 root filesystem

- Update README and USAGE documentation to reflect changes in rootfs handling, including the default local build from Ubuntu 22.04 packages.
- Modify the installer to support building the rootfs locally with authenticated packages and to require SHA-256 checksums for prebuilt images.
- Implement checks for setuid permissions on sudo and update the provisioning logic for guest users to ensure proper sudo access.
- Adjust tests to validate the new rootfs building process and ensure that existing functionality is preserved.
- Refactor code to improve clarity and maintainability, including the introduction of constants for Ubuntu release and guest packages.

tests/test_config.py

@@ -32,7 +32,7 @@ class ArchitectureSettingsTests(unittest.TestCase):
                     )
                     self.assertEqual(settings.architecture, architecture)
                     self.assertEqual(settings.kernel_url, f"{base}/kernels/vmlinux.bin")
-                    self.assertEqual(settings.rootfs_url, f"{base}/rootfs/bionic.rootfs.ext4")
+                    self.assertEqual(settings.rootfs_url, "")
 
     def test_preserves_individual_environment_url_overrides_on_arm64(self) -> None:
         kernel = "https://example.com/custom-arm64-kernel"
@@ -56,7 +56,7 @@ class ArchitectureSettingsTests(unittest.TestCase):
                 )
                 self.assertEqual(
                     settings.rootfs_url,
-                    overrides.get("UVM_ROOTFS_URL", f"{base}/rootfs/bionic.rootfs.ext4"),
+                    overrides.get("UVM_ROOTFS_URL", ""),
                 )
 
     def test_rejects_unsupported_hosts_even_with_custom_guest_assets(self) -> None:

tests/test_images.py

@@ -130,6 +130,19 @@ class ImageStoreTests(unittest.TestCase):
         self.assertIn("Match User service\n    PasswordAuthentication yes", updated)
         self.assertNotIn("ChallengeResponseAuthentication yes", updated)
 
+    def test_full_ubuntu_sshd_global_directives_stay_outside_match_blocks(self) -> None:
+        updated = _enable_ssh_password_authentication(
+            "Include /etc/ssh/sshd_config.d/*.conf\n"
+            "UsePAM yes\nSubsystem sftp /usr/lib/openssh/sftp-server\n"
+            "Match Group restricted\n    X11Forwarding no\n    PasswordAuthentication no\n",
+            "codemate-user",
+        )
+
+        self.assertLess(updated.index("UsePAM yes"), updated.index("Match User codemate-user"))
+        self.assertLess(updated.index("Subsystem sftp"), updated.index("Match User codemate-user"))
+        self.assertLess(updated.index("Match User codemate-user"), updated.index("Match Group restricted"))
+        self.assertIn("Match Group restricted\n    X11Forwarding no", updated)
+
     def test_removes_only_the_public_firecracker_demo_key(self) -> None:
         own_key = "ssh-ed25519 AAAA-own-key developer@example"
 
@@ -241,6 +254,8 @@ class OfflineGuestCredentialsTests(unittest.TestCase):
             ),
             "/etc/group": "root:x:0:\nshadow:x:42:\nubuntu:x:1000:\nreserved:x:1001:\nnogroup:x:65534:\n",
             "/etc/gshadow": "root:*::\nshadow:*::\nubuntu:!::\nreserved:!::\nnogroup:*::\n",
+            "/etc/sudoers": "Defaults env_reset\nroot ALL=(ALL:ALL) ALL\n%sudo ALL=(ALL:ALL) ALL\n",
+            "/usr/bin/sudo": "sudo binary fixture\n",
             "/etc/ssh/sshd_config": "PasswordAuthentication no\nPermitRootLogin no\n",
             "/root/.ssh/authorized_keys": f"{_FIRECRACKER_DEMO_PUBLIC_KEY} demo\nssh-ed25519 AAAA-own-key\n",
         }
@@ -251,7 +266,8 @@ class OfflineGuestCredentialsTests(unittest.TestCase):
             path = root / guest_path.lstrip("/")
             path.parent.mkdir(parents=True, exist_ok=True)
             path.write_text(contents, encoding="utf-8")
-            path.chmod(0o640 if guest_path in ("/etc/shadow", "/etc/gshadow") else 0o644)
+            mode = {"/etc/shadow": 0o640, "/etc/gshadow": 0o640, "/etc/sudoers": 0o440, "/usr/bin/sudo": 0o4755}
+            path.chmod(mode.get(guest_path, 0o644))
         with self.template.open("wb") as image:
             image.truncate(16 * 1024 * 1024)
         self.runner.run(("mkfs.ext4", "-q", "-F", "-d", root, self.template), capture=True)
@@ -305,6 +321,10 @@ class OfflineGuestCredentialsTests(unittest.TestCase):
         self.assertLessEqual(int(shadow[2]), int(time.time() // 86400))
         self.assertEqual(shadow[3:], ["0", "99999", "7", "", "", ""])
         self.assert_password("codemate-user", "custom-password")
+        self.assertIn("codemate-user ALL=(ALL:ALL) PASSWD: ALL\n", self.contents("/etc/sudoers"))
+        self.assertIn("Defaults:codemate-user !rootpw, !runaspw, !targetpw\n", self.contents("/etc/sudoers"))
+        self.assert_metadata("/etc/sudoers", 0o440, 0, 0)
+        self.assert_metadata("/usr/bin/sudo", 0o4755, 0, 0)
         for guest_path in ("/etc/passwd", "/etc/group"):
             self.assert_metadata(guest_path, 0o644, 0, 0)
         for guest_path in ("/etc/shadow", "/etc/gshadow"):
@@ -332,6 +352,10 @@ class OfflineGuestCredentialsTests(unittest.TestCase):
                     self.assertEqual(self.contents(guest_path), self.files[guest_path])
                 self.assertEqual(self.entry("/etc/shadow", username)[2:], original_shadow[2:])
                 self.assert_password(username, "changed-password")
+                if username != "root":
+                    self.assertIn(f"{username} ALL=(ALL:ALL) PASSWD: ALL\n", self.contents("/etc/sudoers"))
+                else:
+                    self.assertEqual(self.contents("/etc/sudoers"), self.files["/etc/sudoers"])
                 for line in self.files["/etc/shadow"].splitlines():
                     if line.split(":", 1)[0] != username:
                         self.assertIn(line + "\n", self.contents("/etc/shadow"))
@@ -375,3 +399,20 @@ class OfflineGuestCredentialsTests(unittest.TestCase):
         self.assertEqual(self.contents("/home/codemate-user"), self.files["/home/codemate-user"])
         self.assertEqual(self.contents("/etc/passwd"), self.files["/etc/passwd"])
         self.assertEqual(self.contents("/etc/shadow"), self.files["/etc/shadow"])
+
+    def test_custom_users_fail_with_an_actionable_error_on_stripped_images(self) -> None:
+        del self.files["/usr/bin/sudo"]
+        del self.files["/etc/sudoers"]
+        self.make_disk()
+
+        with self.assertRaisesRegex(UvmError, "guest image needs setuid-root sudo.*uvm install --force"):
+            self.store.provision_credentials(self.disk, "codemate-user", "custom-password")
+
+        self.assertEqual(self.contents("/etc/shadow"), self.files["/etc/shadow"])
+
+    def test_sudo_without_setuid_permission_does_not_produce_a_broken_administrator(self) -> None:
+        self.make_disk()
+        self.runner.run(("debugfs", "-w", "-R", "set_inode_field /usr/bin/sudo mode 0100755", self.disk), capture=True)
+
+        with self.assertRaisesRegex(UvmError, "guest image needs setuid-root sudo"):
+            self.store.provision_credentials(self.disk, "codemate-user", "custom-password")

tests/test_install.py

@@ -3,6 +3,7 @@ from __future__ import annotations
 import hashlib
 import io
 import os
+import shutil
 import tarfile
 import tempfile
 import unittest
@@ -12,11 +13,12 @@ from types import SimpleNamespace
 from unittest.mock import Mock, patch
 
 from uvm.config import Settings
-from uvm.errors import UvmError
-from uvm.install import Installer
-from uvm.integrity import load_manifest
+from uvm.errors import CommandError, UvmError
+from uvm.images import ImageStore
+from uvm.install import Installer, _ROOTFS_RECIPE
+from uvm.integrity import load_manifest, write_manifest
 from uvm.state import StateStore
-from uvm.system import CommandRunner
+from uvm.system import CommandResult, CommandRunner
 
 
 class InstallerArchitectureTests(unittest.TestCase):
@@ -56,6 +58,10 @@ class InstallerArchitectureTests(unittest.TestCase):
                     firecracker_sha256=hashlib.sha256(archive_bytes).hexdigest(),
                     kernel_sha256=hashlib.sha256(kernel_bytes).hexdigest(),
                     rootfs_sha256=hashlib.sha256(rootfs_bytes).hexdigest(),
+                    rootfs_url=(
+                        "https://s3.amazonaws.com/spec.ccfc.min/img/quickstart_guide/"
+                        f"{architecture}/rootfs/bionic.rootfs.ext4"
+                    ),
                 )
                 guest_base = (
                     "https://s3.amazonaws.com/spec.ccfc.min/img/quickstart_guide/"
@@ -111,3 +117,151 @@ class InstallerArchitectureTests(unittest.TestCase):
             check_kvm.assert_not_called()
             download.assert_not_called()
             self.assertFalse(settings.integrity_manifest_path.exists())
+
+    def test_default_build_records_the_authenticated_local_image_digest(self) -> None:
+        with tempfile.TemporaryDirectory() as temporary:
+            settings = Settings(
+                base=Path(temporary), firecracker_sha256="1" * 64,
+                kernel_sha256=hashlib.sha256(b"kernel").hexdigest(),
+            )
+            settings.bin_dir.mkdir()
+            settings.firecracker_binary.write_bytes(b"firecracker")
+            settings.jailer_binary.write_bytes(b"jailer")
+            installer = Installer(settings, Mock(spec=CommandRunner), StateStore(settings), emit=None)
+            with (
+                patch("uvm.install.require_root"), patch("uvm.install.check_kvm"),
+                patch.object(installer, "install_apt_packages"),
+                patch.object(installer, "install_firecracker_binary", return_value=settings.firecracker_binary),
+                patch.object(installer, "download", side_effect=lambda _url, path, **_kwargs: path.write_bytes(b"kernel")),
+                patch.object(installer, "_build_ubuntu_rootfs", side_effect=lambda path: path.write_bytes(b"full Ubuntu")) as build,
+            ):
+                installer.install()
+
+            build.assert_called_once_with(settings.rootfs_image)
+            manifest = load_manifest(settings.integrity_manifest_path)
+            self.assertTrue(manifest.verified)
+            self.assertEqual(manifest.checksums["rootfs"], hashlib.sha256(b"full Ubuntu").hexdigest())
+            self.assertEqual(ImageStore(settings).installed_assets().rootfs, settings.rootfs_image)
+
+    def test_old_rootfs_checksum_requires_an_explicit_prebuilt_url_before_mutations(self) -> None:
+        settings = Settings(firecracker_sha256="1" * 64, kernel_sha256="2" * 64, rootfs_sha256="3" * 64)
+        runner = Mock(spec=CommandRunner)
+        installer = Installer(settings, runner, Mock(spec=StateStore), emit=None)
+        with self.assertRaisesRegex(UvmError, "unset UVM_ROOTFS_SHA256"):
+            installer._validate_integrity_policy()
+        runner.run.assert_not_called()
+
+    def test_only_verified_current_recipe_images_are_reused(self) -> None:
+        for verified, marker, force in (
+            (True, _ROOTFS_RECIPE, False), (False, _ROOTFS_RECIPE, False),
+            (True, "", False), (True, _ROOTFS_RECIPE, True),
+        ):
+            with self.subTest(verified=verified, marker=marker, force=force), tempfile.TemporaryDirectory() as temporary:
+                settings = Settings(base=Path(temporary))
+                settings.images_dir.mkdir()
+                settings.rootfs_image.write_bytes(b"image")
+                write_manifest(settings.integrity_manifest_path,
+                               {"rootfs": hashlib.sha256(b"image").hexdigest()}, verified=verified)
+                runner = Mock(spec=CommandRunner)
+                runner.run.return_value = CommandResult(("debugfs",), 0, stdout=marker)
+                installer = Installer(settings, runner, StateStore(settings), emit=None)
+                with patch.object(installer, "_build_ubuntu_rootfs") as build:
+                    installer._install_ubuntu_rootfs(settings.rootfs_image, force=force)
+                self.assertEqual(build.called, not (verified and marker == _ROOTFS_RECIPE and not force))
+
+
+class UbuntuBuildRunner:
+    """Simulate package installation; exercise real ext4 formatting and readback."""
+
+    def __init__(self, *, failure: str | None = None) -> None:
+        self.calls: list[tuple[str, ...]] = []
+        self.failure = failure
+        self.runner = CommandRunner(emit=None)
+
+    def run(self, command, **options) -> CommandResult:
+        args = tuple(map(str, command))
+        self.calls.append(args)
+        if "debootstrap" in args:
+            root = Path(args[-2])
+            if self.failure == "bootstrap":
+                raise CommandError("bootstrap interrupted")
+            status = "\n\n".join(
+                f"Package: {package}\nStatus: install ok installed\nVersion: 1.0"
+                for package in ("apt", "sudo", "openssh-server", "systemd-sysv")
+            ) + "\n"
+            files = {
+                "etc/passwd": "root:x:0:0::/root:/bin/bash\n",
+                "etc/shadow": "root:*:1:0:99999:7:::\n", "etc/group": "root:x:0:\n",
+                "etc/sudoers": "root ALL=(ALL:ALL) ALL\n",
+                "etc/ssh/sshd_config": "UsePAM yes\n",
+                "etc/ssh/ssh_host_rsa_key": "template host key",
+                "usr/bin/sudo": "sudo binary", "usr/bin/apt-get": "apt binary",
+                "usr/sbin/sshd": "sshd binary", "usr/lib/systemd/systemd": "init binary",
+                "var/lib/dpkg/status": status,
+                "var/lib/dpkg/info/apt.list": "/usr/bin/apt-get\n",
+                "var/lib/dpkg/info/sudo.list": "/usr/bin/sudo\n",
+            }
+            if self.failure == "metadata":
+                files["var/lib/dpkg/status"] = ""
+            for relative, contents in files.items():
+                path = root / relative
+                path.parent.mkdir(parents=True, exist_ok=True)
+                path.write_text(contents, encoding="utf-8")
+            (root / "usr/bin/sudo").chmod(0o4755)
+            (root / "etc/resolv.conf").symlink_to("../run/systemd/resolve/stub-resolv.conf")
+            return CommandResult(args, 0)
+        if args[0] == "systemctl":
+            return CommandResult(args, 0)
+        return self.runner.run(command, **options)
+
+
+@unittest.skipUnless(all(shutil.which(tool) for tool in ("mkfs.ext4", "e2fsck", "debugfs")),
+                     "e2fsprogs is required for offline Ubuntu image checks")
+class UbuntuRootfsBuildTests(unittest.TestCase):
+    def test_build_keeps_package_metadata_and_sudo_permissions_with_native_signed_bootstrap(self) -> None:
+        for architecture, deb_arch, mirror in (
+            ("aarch64", "arm64", "https://ports.ubuntu.com/ubuntu-ports"),
+            ("x86_64", "amd64", "https://archive.ubuntu.com/ubuntu"),
+        ):
+            with self.subTest(architecture=architecture), tempfile.TemporaryDirectory() as temporary:
+                settings = Settings(base=Path(temporary), architecture=architecture)
+                runner = UbuntuBuildRunner()
+                installer = Installer(settings, runner, StateStore(settings), emit=None)  # type: ignore[arg-type]
+                installer._build_ubuntu_rootfs(settings.rootfs_image)
+
+                bootstrap = next(call for call in runner.calls if "debootstrap" in call)
+                self.assertIn(f"--arch={deb_arch}", bootstrap)
+                self.assertEqual(bootstrap[-1], mirror)
+                self.assertIn("--force-check-gpg", bootstrap)
+                self.assertIn("--extra-suites=jammy-updates,jammy-security", bootstrap)
+                self.assertIn("--mount-proc", bootstrap)
+                self.assertIn("--kill-child", bootstrap)
+                self.assertEqual(settings.rootfs_image.stat().st_mode & 0o777, 0o600)
+                self.assertEqual(list(settings.images_dir.iterdir()), [settings.rootfs_image])
+
+                def guest(command):
+                    return runner.runner.run(("debugfs", "-R", command, settings.rootfs_image), capture=True)
+
+                self.assertIn("Package: apt\nStatus: install ok installed", guest("cat /var/lib/dpkg/status").stdout)
+                self.assertEqual(guest("cat /var/lib/dpkg/info/sudo.list").stdout, "/usr/bin/sudo\n")
+                self.assertRegex(guest("stat /usr/bin/sudo").stdout, r"Mode:\s+0?4755")
+                self.assertEqual(guest("cat /etc/uvm-image").stdout, _ROOTFS_RECIPE)
+                self.assertIn("nameserver 1.1.1.1", guest("cat /etc/resolv.conf").stdout)
+                self.assertIn("KeepConfiguration=static", guest("cat /etc/systemd/network/10-uvm.network").stdout)
+                self.assertIn("jammy-security main universe", guest("cat /etc/apt/sources.list").stdout)
+                for path in ("/usr/sbin/policy-rc.d", "/etc/ssh/ssh_host_rsa_key"):
+                    self.assertIn("File not found", guest(f"stat {path}").stderr)
+
+    def test_failed_bootstrap_or_missing_package_metadata_preserves_the_old_template(self) -> None:
+        for failure in ("bootstrap", "metadata"):
+            with self.subTest(failure=failure), tempfile.TemporaryDirectory() as temporary:
+                settings = Settings(base=Path(temporary))
+                settings.images_dir.mkdir()
+                settings.rootfs_image.write_bytes(b"old template")
+                runner = UbuntuBuildRunner(failure=failure)
+                installer = Installer(settings, runner, StateStore(settings), emit=None)  # type: ignore[arg-type]
+                with self.assertRaises(UvmError):
+                    installer._build_ubuntu_rootfs(settings.rootfs_image)
+                self.assertEqual(settings.rootfs_image.read_bytes(), b"old template")
+                self.assertEqual(list(settings.images_dir.iterdir()), [settings.rootfs_image])
+                self.assertFalse(any(call[0] == "mkfs.ext4" for call in runner.calls))