From d870c98bfa857cf4c05b9cf6c864ad78517c02c8 Mon Sep 17 00:00:00 2001 From: kstyagi Date: Fri, 25 Sep 2026 20:31:20 +0530 Subject: [PATCH] feat: add terminal session management with HTTP/WebSocket routes and session history --- DEVELOPER.md | 99 +++++++++++++++++++++++++++++++++--- README.md | 47 ++++++++++++++++- USAGE.md | 42 ++++++++++++++- tests/test_guest_terminal.py | 10 ++-- tests/test_sessions.py | 59 ++++++++++++++++++++- 5 files changed, 240 insertions(+), 17 deletions(-) diff --git a/DEVELOPER.md b/DEVELOPER.md index 1b4dfa0..d902ccc 100644 --- a/DEVELOPER.md +++ b/DEVELOPER.md @@ -71,6 +71,9 @@ Packet integration tests must remain inside disposable isolated namespaces. │ ├── __main__.py # `python -m nest` │ ├── cli.py # Nest workspace serve command │ ├── server.py # Nest composition of the dashboard and UVM API +│ ├── sessions.py # Live guest terminals and browser-independent session lifecycle +│ ├── session_store.py # Private SQLite session history and bounded transcripts +│ ├── session_routes.py # VM-scoped session HTTP and WebSocket endpoints │ └── dashboard/ │ ├── router.py # Nest dashboard document route │ └── static/ # Packaged HTML, CSS, JavaScript, and fonts @@ -91,6 +94,7 @@ Packet integration tests must remain inside disposable isolated namespaces. │ ├── known_hosts.py # Deletion-only cleanup of local SSH host-key caches │ ├── guest_agent.py # Standalone guest daemon, job scopes, and bounded wire protocol │ ├── guest_exec.py # Host-side guest sessions and execution results +│ ├── guest_terminal.py # Async PTY transport over a process-pinned guest connection │ ├── network.py # IP/MAC/TAP/bridge/NAT/forward rules │ ├── firewall.py # Owned nft bridge tables and offline conntrack cleanup │ ├── lifecycle.py # Lifecycle, bulk operations, recovery, port intent @@ -877,8 +881,8 @@ or secret manager when that matters. ### API Contract -The root redirect and dashboard document are Nest routes. Both server entry -points expose the remaining management routes below. +The root redirect, dashboard document, and terminal-session routes belong to +Nest. Both server entry points expose the UVM management routes below. | Method | Route | Router | Notes | |---|---|---|---| @@ -902,6 +906,11 @@ points expose the remaining management routes below. | `POST` | `/vms/{vm_id}/exec` | `vms.py` | Exact ID; `command`, strict `sudo`, bounded `timeout_seconds` | | `POST` | `/vms/{vm_id}/agent/install` | `vms.py` | Exact ID; install/upgrade the agent on a stopped disk | | `GET` | `/vms/{identifier}/logs` | `vms.py` | Bounded non-following log tail | +| `GET` | `/vms/{vm_id}/sessions` | `nest/session_routes.py` | Saved session list (Nest only) | +| `POST` | `/vms/{vm_id}/sessions` | `nest/session_routes.py` | Launch a guest PTY; optional rows/cols in a JSON object (Nest only) | +| `GET` | `/vms/{vm_id}/sessions/{session_id}` | `nest/session_routes.py` | VM-scoped session metadata (Nest only) | +| `POST` | `/vms/{vm_id}/sessions/{session_id}/close` | `nest/session_routes.py` | End a shell; retain history (Nest only) | +| `WS` | `/vms/{vm_id}/sessions/{session_id}/terminal` | `nest/session_routes.py` | Authenticated replay/input/resize (Nest only) | All management routes use `get_authorized_application()` and require: @@ -969,7 +978,8 @@ The synchronous HTTP response exposes only exit code, stdout/stderr, timeout, and truncation. Captured text uses UTF-8 replacement. Nonzero process exits are HTTP 200 results; agent failures are 502, and failures before submission are 503. Lost/malformed responses after submission carry `outcome_unknown: true`. There -is no command retry, durable job registry, streaming response, or reconnection. +is no command retry, durable job registry, streaming response, or reconnection +for these one-shot exec requests. The root systemd agent accepts only host-CID connections and limits concurrent connections/jobs to eight. Each request creates a delegated cgroup v2 job scope. @@ -977,7 +987,8 @@ A fresh child interpreter joins the scope before initializing the chosen user's supplementary groups, GID, UID, home, and environment, then execs `/bin/sh -c`. `sudo` selects guest root directly; otherwise the recorded username is used. Password authentication is not involved, and a recorded `root` login remains -root for both flag values. Input is closed and there is no PTY. +root for both flag values. One-shot commands have closed stdin; Nest terminal +sessions use the separate PTY channel described below. Both output pipes are drained concurrently. The guest enforces the deadline, signals the process group, escalates via `cgroup.kill`, and confirms cleanup. @@ -986,9 +997,12 @@ PGID reuse before reap. Normal exits clean up remaining background descendants too. An unconfirmed cleanup is an unknown outcome. Stopping the agent service also kills its job scopes; dropping an HTTP connection does not cancel a job. -The v4 Ubuntu builder installs Python and the service. Creation provisions the -VM ID/username in root-only `/etc/uvm-agent.json` before marking the disk ready. -Older templates are left for an explicit stopped-disk `agent/install` request. +The v5 Ubuntu builder installs Python, the terminal-capable service, and an +ownership manifest. Creation provisions the VM ID/username in root-only +`/etc/uvm-agent.json` before marking the disk ready. Cached v4 templates are also +configured; their original manifest-less agent is recognized by its exact +published source digest. Earlier templates require an explicit stopped-disk +`agent/install` request. Provisioning and upgrades stage a sparse copy, replay/check ext4, validate guest path ownership and recorded agent-file hashes, enable the service, verify writes, and publish atomically. A root-only guest manifest records managed file hashes @@ -1003,6 +1017,59 @@ device and uses CID 3. The process adapter retains both sockets while identity is uncertain and removes them only after confirmed VMM exit. IPv4 port policies do not govern this host-only management channel. +### Nest Terminal Sessions + +Nest owns session HTTP/WebSocket routes and history. UVM owns the process-pinned +connection and guest PTY. `LifecycleService.open_terminal()` validates an exact +running VM ID and connects under the operation lock, then opens the PTY outside +the lock. It never reconnects that socket to a replacement VMM. + +The existing version-1 hello and exec request stay compatible. A terminal request +uses `{"version":1,"terminal":{"rows":24,"cols":80}}`. The guest replies with +`{"type":"ready"}` and exchanges length-prefixed JSON output/input/resize/close +messages. Bytes are base64 encoded; raw chunks are at most 8192 bytes, frames +16384 bytes, and dimensions 2–200 rows / 2–400 columns. Partial frames have finite +deadlines while idle shells can remain connected indefinitely. An old agent +rejects the new request; the host reports the stopped-disk upgrade procedure. + +The guest opens a real PTY and spawns a fresh interpreter in a new OS session. +The child joins its delegated job cgroup, establishes the controlling terminal, +assumes the recorded account, and execs that account's interactive shell with +`TERM=xterm-256color`. Input, terminal echo, Ctrl+C, foreground job control, and +window-size changes are handled by the guest terminal driver. Normal exit, +explicit close, or host connection loss cleans up the job cgroup and processes. + +`SessionManager` owns background output readers independently of WebSocket +attachments. Creating a shell is shielded from an HTTP disconnect; its connection +is either tracked or closed. Browser navigation, reload, and logout detach only +the browser. Nest shutdown closes guest connections and marks live sessions +interrupted. In-flight transcript writes finish before releasing store ownership. + +`SessionStore` keeps metadata and raw output in +`$UVM_BASE/nest/sessions.sqlite3`, separate from the VM registry. A private +directory, mode-0600 files, an exclusive owner lock, and SQLite transactions +protect history. On reopening after a server restart, unfinished records become +interrupted. Session lookups always include the exact VM ID. Output is capped at +the newest 8 MiB per session with byte offsets and an explicit truncation marker; +session metadata is retained. Only PTY output is recorded, not a separate input +or password log. Nest permits 32 active sessions; guest terminals share the +agent's eight connection slots with one-shot jobs. + +HTTP routes use the existing `X-UVM-Token` dependency. Browser WebSockets check +the Origin and require `{"type":"auth","token":"...","offset":0}` as their +first frame within ten seconds. Tokens never enter query strings. Each output +message has its byte offset and base64 data; the browser acknowledges the next +offset after xterm renders the chunk. Only one chunk is outstanding, providing +backpressure. Reattachment resumes from an acknowledged offset; a truncated +prefix resets the viewer. Closed sessions use the same channel for read-only +replay. WebSocket transport support comes from the `websockets` dependency. + +The dashboard's Sessions tab and nested hash routes belong to the selected VM. +`sessions.js` owns attachment teardown, replay, reconnects, and the terminal +widget. Vendored xterm 5.5.0 and fit addon 0.10.0 (with MIT licenses) are served +locally under `static/vendor/`. Terminal keyboard input bypasses dashboard search +shortcuts. The widget has a 10,000-line scrollback and a 64 KiB per-paste limit. + ## 14. Configuration Reference The environment-variable table is in [README.md](README.md#configuration); @@ -1088,6 +1155,24 @@ FastAPI route tests are conditionally enabled when both `fastapi` and `httpx` are installed. They should be enabled in CI using the `test` extra. `test_nest_server.py` checks the workspace launcher, relocated frontend assets, authenticated machine controls, API documentation, and graceful shutdown. +`test_sessions.py` covers scoped persistence, interrupted-session recovery, +authentication, reconnects, output acknowledgements, and cancellation cleanup. +`test_guest_terminal.py` exercises actual PTYs, persistent shell state, resizing, +Ctrl+C, and child cleanup using temporary account homes and fake cgroup scopes. + +Optional browser regression checks use Playwright against an isolated fixture +server with fake VMs and guest channels. With the project's test extra installed: + +```sh +npm install --prefix /tmp/nest-browser playwright +/tmp/nest-browser/node_modules/.bin/playwright install chromium +NODE_PATH=/tmp/nest-browser/node_modules \ + NEST_TEST_PYTHON="$(command -v python)" node tests/browser_sessions.cjs +``` + +This checks launch/input, reload/reconnect, session switching, per-VM isolation, +closed-session replay, and the mobile layout. It does not boot a real microVM. + When `nft` and `conntrack` are installed, command-parser tests append an invalid final command/option so parsing is checked without executing any mutation. These checks supplement fixture-based readback and cleanup tests; they do not diff --git a/README.md b/README.md index ce6e31c..31f7cf8 100644 --- a/README.md +++ b/README.md @@ -310,8 +310,9 @@ and `/dashboard` routes belong to the Nest server. The Nest dashboard includes a fleet overview, searchable machine collection, private network and port-policy view, host diagnostics, and machine provisioning. -Open a machine to manage its lifecycle, edit or sync its port rules, copy its SSH -command, or read recent logs. Select machines in the collection for bulk operations. +Open a machine to manage its lifecycle, launch terminal sessions, edit or sync +its port rules, copy its SSH command, or read recent logs. Select machines in the +collection for bulk operations. Use `Ctrl+K` / `⌘K` (or `/`) to find a machine or page. Fleet data refreshes every five seconds while the tab is visible; host checks refresh every thirty seconds and can also be run manually. Charts show observed machine states and allocated @@ -342,6 +343,11 @@ The server exposes these routes: | `POST` | `/vms/{vm_id}/exec` | Execute a command/script inside a running VM over vsock, with optional root privileges | | `POST` | `/vms/{vm_id}/agent/install` | Install/upgrade the execution agent on a stopped VM's private disk | | `GET` | `/vms/{identifier}/logs` | Read a bounded, non-following tail of a VM log | +| `GET` | `/vms/{vm_id}/sessions` | List this VM's saved sessions (Nest) | +| `POST` | `/vms/{vm_id}/sessions` | Launch a guest terminal; body: `{}` or `{"rows":24,"cols":80}` (Nest) | +| `GET` | `/vms/{vm_id}/sessions/{session_id}` | Session metadata and status (Nest) | +| `POST` | `/vms/{vm_id}/sessions/{session_id}/close` | End the shell and retain its history (Nest) | +| `WS` | `/vms/{vm_id}/sessions/{session_id}/terminal` | Authenticated terminal input, output, resize, and replay (Nest) | API port rules are objects such as `{"port": 80, "protocol": "HTTP"}`; `no_default_ports` is a boolean, default `false`. Create also accepts `storage`, @@ -359,6 +365,43 @@ Bulk requests use an action (`start`, `stop`, `restart`, `delete`, or for each fixed target and do not prompt for confirmation. Interactive SSH remains CLI-only. The API and dashboard are not background reconcilers. +### MicroVM Terminal Sessions + +Open **Machines → select a microVM → Sessions**, then click **Launch session**. +Each session opens an interactive guest shell in the dashboard, with terminal +colors, keyboard input, Ctrl+C, resizing, and persistent shell state such as the +working directory. Sessions run as the VM's configured guest user; normal guest +`sudo` commands can request that user's password in the terminal. + +The history list belongs to the selected VM. Reopen an active session to +continue using the same shell, or select an ended session to view its saved +output. Navigation, browser reloads, and signing out leave the shell running. +**End session** closes the shell and its processes. Stopping/restarting the VM +or the Nest server ends live sessions; their saved history remains available. + +Nest stores session metadata and the latest **8 MiB of output per session** in +`$UVM_BASE/nest/sessions.sqlite3` (default `/var/lib/uvm/nest/sessions.sqlite3`). +The viewer indicates truncated output. Raw keyboard input is not logged +separately; the transcript contains what the guest terminal outputs. History +files are private to the host account running Nest. One Nest server owns +sessions for a given VM registry at a time, with up to 32 active sessions; the +guest agent shares its eight connection slots between terminals and exec calls. + +For existing VMs, use **stop → Sessions → Guest terminal setup → Install / update +terminal support → start** once. Newly built v5 Ubuntu images include terminal +support; cached v4 templates are upgraded during new-VM provisioning. The same +guest-agent/kernel requirements described below apply to terminals. + +Terminal support adds the Python `websockets` dependency. In your micromamba +environment, install the current project and launch with its interpreter: + +```sh +micromamba activate nest +python -m pip install -e '.[test]' +sudo env UVM_API_TOKEN="$UVM_API_TOKEN" \ + "$CONDA_PREFIX/bin/python" -m nest --serve +``` + ### Execute Commands Without SSH Use an exact VM ID from create/list: diff --git a/USAGE.md b/USAGE.md index cefc9d5..e4bb502 100644 --- a/USAGE.md +++ b/USAGE.md @@ -801,7 +801,8 @@ The installed command is `nest --serve`; `python3 -B -m nest --serve` also works from the source checkout. Nest uses `UVM_API_TOKEN` and the `UVM_*` host settings for its machine-management backend. To launch the standalone UVM API, use `./uvm.py --serve` with the same host/port options and configuration. That -server exposes the management endpoints and `/docs`; Nest serves the dashboard. +server exposes the UVM management endpoints and `/docs`; Nest serves the +dashboard and terminal-session endpoints. The API token is required even for a loopback server. All VM, diagnostics, and installation routes require it. The dashboard shell, liveness route, and FastAPI's @@ -812,6 +813,40 @@ with no whitespace. Use a protected environment file or secret manager rather than recording a real token in shell history; do not share tokens or guest passwords in reports. +### Use Sessions Inside a MicroVM + +1. Open **Machines** and select the microVM. +2. Start it if it is stopped. +3. Click **Launch session**, or open its **Sessions** tab and launch there. +4. Type directly in the embedded terminal. Each session keeps its own shell, + current directory, environment, and foreground program. +5. Use **Session history** to switch between that VM's sessions. Active sessions + reconnect to their existing shell; ended sessions replay saved output. +6. Use **End session** when finished, or run `exit` inside the shell. + +Sessions continue while you navigate elsewhere, reload, or sign out of the +dashboard. VM shutdown/restart or Nest shutdown ends the shell. Session metadata, +start/end times, exit status, and output survive Nest restarts in +`$UVM_BASE/nest/sessions.sqlite3`; files are mode `0600` in a mode-`0700` directory. +Only sessions belonging to the selected exact VM ID are shown. Reusing a VM name +does not attach another VM's history. + +The retained transcript is capped at its latest 8 MiB, and the terminal viewer +has a 10,000-line scrollback. A truncation note appears when older output is +discarded. Terminal input supports pastes up to 64 KiB at a time. Guest output +is saved even while no browser is attached. + +For an older VM, stop it, open **Sessions → Guest terminal setup**, click +**Install / update terminal support**, and start it again. This uses the existing +stopped-disk agent upgrade endpoint. If the agent is still starting or installing +Python, wait for it to become ready before launching a session. + +The terminal transport uses the guest agent over vsock. It logs in as the VM's +configured username; `sudo` inside the terminal uses normal guest authentication. +Nest supports 32 active sessions per server; each guest agent shares eight +connection slots with its command-execution endpoint. Run one Nest session +manager per `UVM_BASE` registry. Reverse proxies must forward WebSocket upgrades. + ### Start a TLS Server Non-loopback binds require a valid TLS certificate and key. The files must be @@ -876,6 +911,11 @@ curl --cacert /path/to/ca.pem \ | `POST` | `/vms/{vm_id}/exec` | Execute a guest command/script over vsock, with optional root privileges | | `POST` | `/vms/{vm_id}/agent/install` | Install/upgrade the agent on a stopped VM's disk | | `GET` | `/vms/{identifier}/logs` | Read a bounded, non-following log tail | +| `GET` | `/vms/{vm_id}/sessions` | List saved sessions for an exact VM ID (Nest) | +| `POST` | `/vms/{vm_id}/sessions` | Launch a terminal, with `{}` or optional `rows`/`cols` (Nest) | +| `GET` | `/vms/{vm_id}/sessions/{session_id}` | Inspect a session (Nest) | +| `POST` | `/vms/{vm_id}/sessions/{session_id}/close` | End a session and retain its output (Nest) | +| `WS` | `/vms/{vm_id}/sessions/{session_id}/terminal` | Authenticated live terminal/replay (Nest) | Individual lifecycle requests do not expose a CLI dry-run flag and do not prompt for confirmation. `timeout` is an optional finite, non-negative query parameter diff --git a/tests/test_guest_terminal.py b/tests/test_guest_terminal.py index 64b82df..f7bf0fc 100644 --- a/tests/test_guest_terminal.py +++ b/tests/test_guest_terminal.py @@ -78,7 +78,7 @@ agent['child_main'](Path({str(self.scope.path)!r}), {username!r}, None) self.assertFalse(self.thread.is_alive()) async def test_terminal_has_tty_persistent_shell_state_and_exit_code(self): - await self.terminal.write(b"test -t 0 && printf 'REAL-TTY\\n'\r") + await self.terminal.write(b"test -t 0 && printf '\\nREAL-TTY\\n'\r") await self.until(b"\r\nREAL-TTY\r\n") await self.terminal.write(b"export NEST_TEST=hello; cd /\r") await self.terminal.write(b"printf 'STATE:%s:%s\\n' \"$NEST_TEST\" \"$PWD\"\r") @@ -95,15 +95,15 @@ agent['child_main'](Path({str(self.scope.path)!r}), {username!r}, None) await self.terminal.resize(40, 120) await self.terminal.write(b"stty size\r") await self.until(b"40 120\r\n") - await self.terminal.write(b"sleep 20\r") - await asyncio.sleep(0.1) + await self.terminal.write(b"sh -c 'printf \"\\nJOB-STARTED\\n\"; exec sleep 20'\r") + await self.until(b"\r\nJOB-STARTED\r\n") await self.terminal.write(b"\x03") await self.until(self.prompt) - await self.terminal.write(b"printf 'INTERRUPTED\\n'\r") + await self.terminal.write(b"printf '\\nINTERRUPTED\\n'\r") await self.until(b"\r\nINTERRUPTED\r\n") async def test_explicit_end_terminates_the_shell(self): - await self.terminal.write(b"printf 'READY\\n'\r") + await self.terminal.write(b"printf '\\nREADY\\n'\r") await self.until(b"\r\nREADY\r\n") await self.terminal.end() async with asyncio.timeout(5): diff --git a/tests/test_sessions.py b/tests/test_sessions.py index afe24da..d7ff58f 100644 --- a/tests/test_sessions.py +++ b/tests/test_sessions.py @@ -7,12 +7,13 @@ import tempfile import unittest from pathlib import Path from types import SimpleNamespace +from unittest.mock import patch from nest.server import create_app from nest.session_store import SessionStore from nest.sessions import SessionManager from tests.test_server import FASTAPI_AVAILABLE, fake_application -from uvm.errors import ConflictError, GuestAgentUnavailableError, NotFoundError +from uvm.errors import ConflictError, GuestAgentUnavailableError, GuestExecutionError, NotFoundError if FASTAPI_AVAILABLE: @@ -29,7 +30,10 @@ class FakeTerminal: self.closed = False async def read(self): - return await self.events.get() + event = await self.events.get() + if isinstance(event, Exception): + raise event + return event async def write(self, data): self.input.append(data) @@ -176,6 +180,45 @@ class SessionManagerTests(unittest.IsolatedAsyncioTestCase): await self.manager.shutdown() self.assertTrue(self.application.terminals[0].closed) + async def test_guest_loss_ends_the_session_and_retains_output(self): + record = await self.manager.create("vm-test", 24, 80) + terminal = self.application.terminals[0] + await terminal.events.put(GuestExecutionError("VM connection lost", outcome_unknown=True)) + async with asyncio.timeout(2): + while (await self.manager.get("vm-test", record["id"]))["status"] == "active": + await asyncio.sleep(0.01) + saved, _start, output = await self.manager.read("vm-test", record["id"], 0) + self.assertEqual(saved["status"], "interrupted") + self.assertEqual(output, b"guest$ ") + self.assertTrue(terminal.closed) + + async def test_shutdown_finishes_inflight_output_before_releasing_history_ownership(self): + import threading + entered, release = threading.Event(), threading.Event() + history = self.manager._history() + append = history.append + + def slow_append(*args): + entered.set() + release.wait(2) + append(*args) + + with patch.object(history, "append", side_effect=slow_append): + record = await self.manager.create("vm-test", 24, 80) + self.assertTrue(await asyncio.to_thread(entered.wait, 2)) + closing = asyncio.create_task(self.manager.shutdown()) + try: + await asyncio.sleep(0.03) + self.assertFalse(closing.done()) + with self.assertRaises(ConflictError): + SessionStore(history.path) + finally: + release.set() + await closing + reopened = SessionStore(history.path) + self.addCleanup(reopened.close) + self.assertEqual(reopened.read("vm-test", record["id"], 0)[2], b"guest$ ") + @unittest.skipUnless(FASTAPI_AVAILABLE, "FastAPI and httpx are required") class SessionApiTests(unittest.TestCase): @@ -265,3 +308,15 @@ class SessionApiTests(unittest.TestCase): self.assertEqual(transcript, b"guest$ pwd\r\r\nguest$ ") self.assertEqual(self.client.get(self.base, headers=self.headers).json()[0]["status"], "closed") self.assertTrue(self.application.terminals[0].closed) + + def test_malformed_input_ends_only_the_browser_attachment(self): + record = self.create() + ws = self.connect(record) + self.output(ws) + ws.send_json({"type": "input", "data": "not base64!"}) + while (message := ws.receive_json())["type"] == "session": + pass + self.assertEqual(message["type"], "error") + self.assertEqual(self.application.terminals[0].input, []) + self.assertFalse(self.application.terminals[0].closed) + self.assertEqual(self.client.get(self.base + "/" + record["id"], headers=self.headers).json()["status"], "active")