From b5bc525f0ce99c31ced50452c8a0f480488aef5e Mon Sep 17 00:00:00 2001 From: minatoaquaMK2 Date: Mon, 19 Oct 2026 08:49:40 +0000 Subject: [PATCH 01/75] feat(sandboxd): add tenant managed persistent volumes Keep tenant data disks across claim release and template replacement. Add node-local management, SDK methods, and durable writer ownership. --- docs/index.md | 2 + docs/managed-volumes.md | 152 +++++ docs/sandboxd-api.md | 6 + docs/sdk-python.md | 3 + docs/sdk.md | 3 + e2e/managed_volume_http_test.go | 293 +++++++++ e2e/managed_volume_privacy_test.go | 126 ++++ e2e/wire_parity_test.go | 20 + sandboxd/config/config.go | 81 ++- sandboxd/config/managed_volume_test.go | 137 ++++ sandboxd/pool/claim.go | 50 +- sandboxd/pool/failover.go | 7 + sandboxd/pool/managed_attachment.go | 383 ++++++++++++ sandboxd/pool/managed_integration_test.go | 284 +++++++++ sandboxd/pool/managed_live.go | 32 + sandboxd/pool/managed_recovery.go | 92 +++ .../pool/managed_recovery_regression_test.go | 225 +++++++ .../pool/managed_startup_boundary_test.go | 102 +++ sandboxd/pool/managed_volume.go | 405 ++++++++++++ sandboxd/pool/managed_volume_test.go | 584 ++++++++++++++++++ sandboxd/pool/pool.go | 25 + sandboxd/pool/reconcile.go | 28 +- sandboxd/pool/remove.go | 24 + sandboxd/pool/volume.go | 105 +++- sandboxd/pool/volume_mutation.go | 52 +- sandboxd/pool/volume_registry.go | 402 ++++++++++++ sandboxd/server/api.go | 4 + sandboxd/server/managed_volumes.go | 129 ++++ sandboxd/server/server.go | 11 +- sandboxd/server/server_http.go | 11 + sandboxd/types/api.go | 32 +- sdk/go/client.go | 13 +- sdk/go/managed_volume.go | 129 ++++ sdk/go/managed_volume_test.go | 144 +++++ sdk/go/options.go | 24 +- sdk/python/cocoonsandbox/client.py | 77 ++- sdk/python/cocoonsandbox/errors.py | 8 + sdk/python/tests/test_managed_volumes.py | 185 ++++++ 38 files changed, 4346 insertions(+), 44 deletions(-) create mode 100644 docs/managed-volumes.md create mode 100644 e2e/managed_volume_http_test.go create mode 100644 e2e/managed_volume_privacy_test.go create mode 100644 sandboxd/config/managed_volume_test.go create mode 100644 sandboxd/pool/managed_attachment.go create mode 100644 sandboxd/pool/managed_integration_test.go create mode 100644 sandboxd/pool/managed_live.go create mode 100644 sandboxd/pool/managed_recovery.go create mode 100644 sandboxd/pool/managed_recovery_regression_test.go create mode 100644 sandboxd/pool/managed_startup_boundary_test.go create mode 100644 sandboxd/pool/managed_volume.go create mode 100644 sandboxd/pool/managed_volume_test.go create mode 100644 sandboxd/pool/volume_registry.go create mode 100644 sandboxd/server/managed_volumes.go create mode 100644 sdk/go/managed_volume.go create mode 100644 sdk/go/managed_volume_test.go create mode 100644 sdk/python/tests/test_managed_volumes.py diff --git a/docs/index.md b/docs/index.md index 0c0e15e5..7cccc2a2 100644 --- a/docs/index.md +++ b/docs/index.md @@ -29,6 +29,8 @@ vsock-only I/O (hardened default); `net=egress` attaches a bridge/CNI NIC. online pool reconfigure, metrics, the usage journal - [Replace a dataset in a running sandbox](live-volumes.md) — swapping a read-only catalog volume on a running claim with the volume API +- [Tenant managed persistent volumes](managed-volumes.md) — runtime resource + ownership, warm attachment, recovery, feature map and verification evidence - [Go SDK](sdk.md) — connecting (single node and clusters), every option, the full sandbox surface, error handling - [Python SDK](sdk-python.md) — the guest and data-plane surface for the diff --git a/docs/managed-volumes.md b/docs/managed-volumes.md new file mode 100644 index 00000000..87fea148 --- /dev/null +++ b/docs/managed-volumes.md @@ -0,0 +1,152 @@ +# Tenant managed persistent volumes + +Managed volumes keep a tenant's data disk when a sandbox claim is released or +replaced with a different template. This phase adds node-local create, inspect, +list and conditional delete, claim-time attachment to warm VMs, and durable +single-writer exclusion. It builds on the existing +[read-only live volume API](live-volumes.md). + +This is a subset of [issue #300](https://github.com/cocoonstack/sandbox/issues/300). +Hibernate/wake, online growth, placement by volume and archive export remain +follow-up work. Attached volumes still block capture. Do not use this branch to +claim that all of #300 is complete. + +## Configure one node + +Set a nonempty `api_token` and an absolute, clean `managed_volume_dir` in the +sandboxd configuration. The managed directory must be private to the daemon, +on persistent storage, and outside disposable VM working directories. Every +existing ancestor must be owned by root or the daemon user. The daemon creates +private `disks/`, `registry.json` and `lock` entries; it exclusively locks the +registry directory. + +Startup rejects managed stores inside `data_dir/readers`, `data_dir/goldens` +and top-level `data_dir/fork-*` trees before creating files. The check includes +existing symbolic links at `data_dir`, `readers` and `goldens`. A store such as +`data_dir/managed-volumes` remains valid. Keep these directories under trusted +host control; validation does not detect bind-mount aliases or prevent a +concurrent pathname swap. + +```json +{ + "api_token": "replace-with-your-root-token", + "managed_volume_dir": "/var/lib/sandboxd/managed-volumes" +} +``` + +Keep the registry and backing files together. Do not resize, rename, replace or +hardlink a backing file by hand. Each managed resource records its immutable +ID, owner, node, capacity and backing device/inode/size. An identity mismatch +requires operator recovery and fails closed. + +The existing static catalog remains operator-owned. Its files and every path +component must be controlled by trusted operator or daemon principals. Static +catalog paths must not alias private managed storage, and a held name must +continue to identify the same backing. Stable symlink aliases and replacements +before attachment are checked. These checks precede the runtime's later path +open; they cannot defeat a concurrent host pathname swap. An untrusted actor +must not be able to change a catalog path component. + +## Manage a resource + +Use an explicit root or tenant management token. A sandbox token alone cannot +manage resources or call the existing live volume endpoints. Tenant tokens can +create and attach only their own managed resources; root must name an existing +tenant owner when creating one. + +| Request | Success | Refusal or retry behavior | +| --- | --- | --- | +| `PUT /v1/volumes/work` with `{"size_gib":20}` | `201`, `volume`, identity `ETag` | Identical retry returns `200` and the same identity; changed owner or capacity returns `409`. | +| `GET /v1/volumes` | Catalog and managed summaries visible to caller | Managed summaries include identity and owner; use individual GET for holders. | +| `GET /v1/volumes/work` | Managed detail with public holder IDs, modes and phases | Foreign and unknown names share `404`; host paths and process IDs stay private. | +| `DELETE /v1/volumes/work` with `If-Match: "vol_…"` | `204` | Missing precondition `428`; wrong identity `412`; held resource `409`. | + +Capacity is an integer from 1 through 1024 GiB. The backing is a raw sparse file; +logical capacity is not reserved physical space. Storage exhaustion can still +occur while guests write. Creation does not format or mount a filesystem. +Deletion tombstones preserve the outcome of an earlier deletion. Replaying that +old identity returns `204` without deleting a newly created same-name resource. + +The Go SDK exposes `CreateVolume`, `GetVolume` and `DeleteVolume`. Managed detail +uses `ManagedVolumeInfo`; existing `VolumeInfo` remains comparable. Structured +`ManagedVolumeError` unwraps to `APIError`. Python exposes `create_volume`, +`get_volume` and `delete_volume`; structured fields remain on `APIError`. + +## Format once, reuse across templates + +The initial raw disk requires explicit filesystem setup. This Python example +uses an attach-only claim, waits for the disk serial, formats it once and +unmounts it before release. Rerunning the example against an already formatted +resource deliberately fails the blank-disk assertion. + +```python +from cocoonsandbox import Client + +client = Client("127.0.0.1:7777", api_token="tenant-management-token") +volume = client.create_volume("work", 20) +with client.new("rt:24.04", volumes=[{"name": "work", "mode": "rw"}], mount=False) as sb: + device = sb.exec( + "sh", "-c", + 'for i in $(seq 1 200); do for p in /sys/block/*/serial; do ' + '[ "$(cat "$p")" = work ] && { echo "/dev/$(basename "$(dirname "$p")")"; exit 0; }; ' + 'done; sleep 0.01; done; exit 1', + ).strip() + sb.exec("sh", "-c", 'blkid "$1" >/dev/null 2>&1; [ "$?" = 2 ]', "probe", device) + sb.exec("mkfs.ext4", "-q", "-F", device) + sb.exec("mkdir", "-p", "/workspace") + sb.exec("mount", device, "/workspace") + sb.write_file("/workspace/marker", b"persistent") + sb.exec("sync", "-f", "/workspace/marker") + sb.exec("umount", "/workspace") + +with client.new("new-template:local", volumes=[{"name": "work", "mode": "rw", "mount": "/workspace"}]) as sb: + assert sb.read_file("/workspace/marker") == b"persistent" + +client.delete_volume("work", volume["id"]) +``` + +A reader excludes a writer, and a writer excludes both readers and other +writers. Multiple readers may share a ready disk. Live additions and removal +of managed entries are deferred, including mounted read-only managed entries. +A running managed claim can still change mounted read-only catalog entries +through the existing `GET/PUT/PATCH /v1/sandboxes/{id}/volumes` interface. +Attach-only managed entries retain their empty mount across those changes and +across daemon restart. + +## Ownership and recovery + +The managed registry supplements the existing flat `claims.json` journal. +Admission durably reserves the resource, then binds its VM process generation +before any disk effect. A live operation records the union of applied, desired +and possibly attached entries before changing a device. A confirmed retry can +trim that union only after device cleanup and claim-journal commit succeed. + +Release durably marks a hold retiring before cancelling the existing +`volumeRun` and dropping the claim. A successful runtime removal command alone +does not free the hold. Fresh runtime inspection must report absence and the +recorded boot ID/PID/start generation must be gone. Failed writes, uncertain +attachments and delayed removal retain ownership. These rules also apply when +`sync_claims` is disabled. + +A failed egress recovery fences the claim and attempts VM removal even if +managed retirement persistence fails. A stubborn VM or unavailable registry +can keep the hold for operator recovery. This phase does not promise that an +uncooperative VMM can always be killed. + +After restart, the persisted egress-quarantine failure marker resumes retirement +and removal. Ordinary failed claims keep their existing ownership until release. +Unclaimed stopped VM records with no live process are removed before startup +continues. Unreadable process metadata and unconfirmed removal still block +startup. The stale-create collector and recorded process-generation checks +remain in force. + +## Follow-up order + +1. Add quiesce/unmount/detach/capture and restore/reattach/remount with failure + rollback. Keep single-writer ownership while a claim is hibernated. +2. Add online growth with explicit block-device and guest-filesystem outcomes. +3. Add owner-node placement and a clear unavailable-owner response. +4. Add archive/export and restore of external data disks. +5. Add Kubernetes lifecycle/status ownership once the daemon contracts are stable. + +These are proposed next steps. None is implemented or qualified by this phase. diff --git a/docs/sandboxd-api.md b/docs/sandboxd-api.md index 31a255cd..ebf4c96a 100644 --- a/docs/sandboxd-api.md +++ b/docs/sandboxd-api.md @@ -255,6 +255,12 @@ list; the field is emitted (as `true`) only for a writable entry this node declares and omitted otherwise (peer-only rows never carry it), so a read-only entry's response is byte-identical to v1. +When `managed_volume_dir` is enabled, this list also includes caller-visible +managed summaries and requires an explicit root or tenant token. Individual +managed GET responses carry holder details. The node-local create, inspect +and conditional-delete routes are documented in +[Tenant managed persistent volumes](managed-volumes.md#manage-a-resource). + ## GET /v1/sandboxes/{id}/volumes Returns the persisted desired set (`volumes_want`), confirmed applied set diff --git a/docs/sdk-python.md b/docs/sdk-python.md index d726d366..e168fc4f 100644 --- a/docs/sdk-python.md +++ b/docs/sdk-python.md @@ -21,6 +21,9 @@ wire is chunked, the caller's copy is not). Wire fidelity is pinned by the shared protocol fixture corpus that the Rust guest, Go, and Python all round-trip in CI. +For node-local `create_volume`, `get_volume`, `delete_volume` and the format-once +workflow, see [managed volumes](managed-volumes.md). + ## A complete example Claim a sandbox, push a project in, run a build, then freeze the built state diff --git a/docs/sdk.md b/docs/sdk.md index 2d66f2aa..c7da4017 100644 --- a/docs/sdk.md +++ b/docs/sdk.md @@ -9,6 +9,9 @@ module). One `Client` talks to one entry node; sandbox handles dial their owning node directly, so a client works unchanged against a single node or a cluster. +For node-local `CreateVolume`, `GetVolume`, `DeleteVolume`, managed detail +types and structured errors, see [managed volumes](managed-volumes.md). + ## A complete example Claim a sandbox, push a project into it, run a build, then freeze the built diff --git a/e2e/managed_volume_http_test.go b/e2e/managed_volume_http_test.go new file mode 100644 index 00000000..68857812 --- /dev/null +++ b/e2e/managed_volume_http_test.go @@ -0,0 +1,293 @@ +package e2e + +import ( + "encoding/json/v2" + "fmt" + "io" + "net/http" + "path/filepath" + "strings" + "sync" + "testing" + + "github.com/cocoonstack/sandbox/sandboxd/config" + "github.com/cocoonstack/sandbox/sandboxd/tenants" +) + +func TestManagedVolumeHTTPIdentityAndConditionalDelete(t *testing.T) { + st := startManagedHTTPStack(t, "root-token") + created := requestManagedVolume(t, st, http.MethodPut, "/v1/volumes/work", "t1-token", `{"size_gib":20}`, "") + if created.status != http.StatusCreated { + t.Fatalf("PV-M01 create status=%d body=%s, want 201", created.status, created.body) + } + volume := decodeManagedVolume(t, created.body) + if volume.ID == "" || volume.Name != "work" || volume.Tenant != "t1" || volume.Kind != "managed" || + volume.SizeBytes != 20<<30 || volume.State != "ready" { + t.Fatalf("PV-M01 unexpected resource projection: %+v", volume) + } + if created.etag != `"`+volume.ID+`"` { + t.Fatalf("PV-M01 etag=%q, want resource identity", created.etag) + } + for _, body := range []string{`{"size_gib":20}`, `{"tenant":"t1","size_gib":20}`} { + repeated := requestManagedVolume(t, st, http.MethodPut, "/v1/volumes/work", "t1-token", body, "") + if repeated.status != http.StatusOK || decodeManagedVolume(t, repeated.body).ID != volume.ID { + t.Fatalf("PV-M02 create retry status=%d body=%s", repeated.status, repeated.body) + } + } + changed := requestManagedVolume(t, st, http.MethodPut, "/v1/volumes/work", "t1-token", `{"size_gib":21}`, "") + if changed.status != http.StatusConflict { + t.Fatalf("PV-M03 changed capacity status=%d, want 409", changed.status) + } + got := requestManagedVolume(t, st, http.MethodGet, "/v1/volumes/work", "t1-token", "", "") + if got.status != http.StatusOK || got.etag != created.etag || decodeManagedVolume(t, got.body).ID != volume.ID { + t.Fatalf("PV-M02 inspect status=%d body=%s", got.status, got.body) + } + for _, tc := range []struct { + etag string + want int + }{ + {want: http.StatusPreconditionRequired}, + {etag: `"vol_00000000000000000000000000000000"`, want: http.StatusPreconditionFailed}, + } { + refused := requestManagedVolume(t, st, http.MethodDelete, "/v1/volumes/work", "t1-token", "", tc.etag) + if refused.status != tc.want { + t.Fatalf("PV-M10 delete precondition status=%d, want %d", refused.status, tc.want) + } + } + for range 2 { + deleted := requestManagedVolume(t, st, http.MethodDelete, "/v1/volumes/work", "t1-token", "", created.etag) + if deleted.status != http.StatusNoContent { + t.Fatalf("PV-M09/PV-M10 delete/retry status=%d body=%s", deleted.status, deleted.body) + } + } + replacement := requestManagedVolume(t, st, http.MethodPut, "/v1/volumes/work", "t1-token", `{"size_gib":20}`, "") + if replacement.status != http.StatusCreated || decodeManagedVolume(t, replacement.body).ID == volume.ID { + t.Fatalf("PV-M10 replacement lacks a fresh identity: status=%d body=%s", replacement.status, replacement.body) + } + stale := requestManagedVolume(t, st, http.MethodDelete, "/v1/volumes/work", "t1-token", "", created.etag) + if stale.status != http.StatusNoContent { + t.Fatalf("PV-M10 authorized old deletion replay status=%d, want 204", stale.status) + } + survivor := requestManagedVolume(t, st, http.MethodGet, "/v1/volumes/work", "t1-token", "", "") + if survivor.status != http.StatusOK || decodeManagedVolume(t, survivor.body).ID != decodeManagedVolume(t, replacement.body).ID { + t.Fatalf("PV-M10 stale delete changed replacement: status=%d body=%s", survivor.status, survivor.body) + } +} + +func TestManagedVolumeHTTPAuthorizationAndPrivacy(t *testing.T) { + st := startManagedHTTPStack(t, "root-token") + created := requestManagedVolume(t, st, http.MethodPut, "/v1/volumes/private", "t1-token", `{"size_gib":1}`, "") + if created.status != http.StatusCreated { + t.Fatalf("PV-M04 fixture create status=%d body=%s", created.status, created.body) + } + for _, method := range []string{http.MethodGet, http.MethodPut, http.MethodDelete} { + foreign := requestManagedVolume(t, st, method, "/v1/volumes/private", "t2-token", `{"size_gib":1}`, created.etag) + if foreign.status != http.StatusNotFound { + t.Errorf("PV-M04/PV-M08 foreign %s status=%d, want 404", method, foreign.status) + } + if method == http.MethodGet { + missing := requestManagedVolume(t, st, method, "/v1/volumes/unknown", "t2-token", "", "") + if foreign.status != missing.status || string(foreign.body) != string(missing.body) { + t.Errorf("PV-M04 foreign and unknown inspection differ: %d/%d", foreign.status, missing.status) + } + } + assertManagedReplyPrivate(t, foreign.body) + } + listing := requestManagedVolume(t, st, http.MethodGet, "/v1/volumes", "t2-token", "", "") + if listing.status != http.StatusOK || strings.Contains(string(listing.body), `"private"`) { + t.Fatalf("PV-M04 foreign listing status=%d body=%s", listing.status, listing.body) + } + root := requestManagedVolume(t, st, http.MethodGet, "/v1/volumes/private", "root-token", "", "") + if root.status != http.StatusOK || decodeManagedVolume(t, root.body).Tenant != "t1" { + t.Fatalf("PV-M04 root inspection status=%d body=%s", root.status, root.body) + } + for _, token := range []string{"", "invalid-token"} { + for _, method := range []string{http.MethodGet, http.MethodPut, http.MethodDelete} { + reply := requestManagedVolume(t, st, method, "/v1/volumes/private", token, `{"size_gib":1}`, created.etag) + if reply.status != http.StatusUnauthorized { + t.Errorf("PV-M05 invalid credential %s status=%d, want 401", method, reply.status) + } + assertManagedReplyPrivate(t, reply.body) + } + } + claimStatus, claim := rawClaim(t, st, "") + if claimStatus != http.StatusOK { + t.Fatalf("PV-M05 sandbox credential fixture status=%d", claimStatus) + } + sandboxAttempt := requestManagedVolume(t, st, http.MethodGet, "/v1/volumes/private", claim.Token, "", "") + if sandboxAttempt.status != http.StatusUnauthorized { + t.Errorf("PV-M05 sandbox credential grants volume management: status=%d", sandboxAttempt.status) + } + released := requestManagedVolume(t, st, http.MethodPost, "/v1/sandboxes/"+claim.ID+"/release", claim.Token, "", "") + if released.status != http.StatusNoContent { + t.Fatalf("PV-M05 release credential fixture status=%d", released.status) + } + override := requestManagedVolume(t, st, http.MethodPut, "/v1/volumes/override", "t1-token", `{"size_gib":1,"tenant":"t2"}`, "") + if override.status != http.StatusForbidden { + t.Errorf("PV-M06 tenant override status=%d, want 403", override.status) + } + invalidRoot := requestManagedVolume(t, st, http.MethodPut, "/v1/volumes/root-owned", "root-token", `{"size_gib":1}`, "") + if invalidRoot.status != http.StatusBadRequest { + t.Errorf("PV-M06 root missing tenant status=%d, want 400", invalidRoot.status) + } + rootCreate := requestManagedVolume(t, st, http.MethodPut, "/v1/volumes/two", "root-token", `{"size_gib":1,"tenant":"t2"}`, "") + if rootCreate.status != http.StatusCreated || decodeManagedVolume(t, rootCreate.body).Tenant != "t2" { + t.Fatalf("PV-M06 root explicit tenant status=%d body=%s", rootCreate.status, rootCreate.body) + } + deleted := requestManagedVolume(t, st, http.MethodDelete, "/v1/volumes/private", "t1-token", "", created.etag) + if deleted.status != http.StatusNoContent { + t.Fatalf("PV-M09 fixture deletion status=%d body=%s", deleted.status, deleted.body) + } + foreignTombstone := requestManagedVolume(t, st, http.MethodDelete, "/v1/volumes/private", "t2-token", "", created.etag) + if foreignTombstone.status != http.StatusNotFound { + t.Fatalf("PV-M05/PV-M10 deletion ID incorrectly grants authority: status=%d", foreignTombstone.status) + } +} + +func TestManagedVolumeHTTPRejectsInputAndSerializesCreation(t *testing.T) { + st := startManagedHTTPStack(t, "root-token") + for _, body := range []string{ + `{}`, `{"size_gib":0}`, `{"size_gib":-1}`, `{"size_gib":1025}`, + `{"size_gib":1.5}`, `{"size_gib":9223372036854775808}`, + `{"size_gib":1,"unknown":true}`, `{"size_gib":1} {}`, `{"size_gib":`, + } { + reply := requestManagedVolume(t, st, http.MethodPut, "/v1/volumes/invalid", "t1-token", body, "") + if reply.status != http.StatusBadRequest { + t.Errorf("PV-M07 input %q status=%d, want 400", body, reply.status) + } + } + oversized := `{"size_gib":1,"padding":"` + strings.Repeat("x", 1<<20) + `"}` + tooLarge := requestManagedVolume(t, st, http.MethodPut, "/v1/volumes/oversized", "t1-token", oversized, "") + if tooLarge.status != http.StatusBadRequest { + t.Errorf("PV-M07 oversized body status=%d, want 400", tooLarge.status) + } + for _, name := range []string{".bad", "cocoon-root"} { + reply := requestManagedVolume(t, st, http.MethodPut, "/v1/volumes/"+name, "t1-token", `{"size_gib":1}`, "") + if reply.status != http.StatusBadRequest { + t.Errorf("PV-M07/PV-M08 name %q status=%d, want 400", name, reply.status) + } + } + const callers = 8 + replies := make([]managedHTTPReply, callers) + var group sync.WaitGroup + for i := range replies { + group.Go(func() { + replies[i] = requestManagedVolume(t, st, http.MethodPut, "/v1/volumes/race", "t1-token", `{"size_gib":1}`, "") + }) + } + group.Wait() + var id string + var created int + for _, reply := range replies { + if reply.status != http.StatusOK && reply.status != http.StatusCreated { + t.Fatalf("PV-M12 concurrent create status=%d body=%s", reply.status, reply.body) + } + volume := decodeManagedVolume(t, reply.body) + if id != "" && volume.ID != id { + t.Fatalf("PV-M12 competing creates returned different identities") + } + id = volume.ID + if reply.status == http.StatusCreated { + created++ + } + } + if created != 1 { + t.Fatalf("PV-M12 got %d creations, want exactly one", created) + } +} + +func TestManagedVolumeHTTPRequiresExplicitCredential(t *testing.T) { + st := startManagedHTTPStack(t, "root-token") + missing := requestManagedVolume(t, st, http.MethodPut, "/v1/volumes/implicit", "", `{"size_gib":1,"tenant":"t1"}`, "") + if missing.status != http.StatusUnauthorized { + t.Fatalf("PV-M05 legacy empty root configuration grants management without credentials: status=%d", missing.status) + } + for _, route := range []string{"/v1/volumes", "/v1/volumes/implicit"} { + for _, token := range []string{"", "invalid-token"} { + got := requestManagedVolume(t, st, http.MethodGet, route, token, "", "") + if got.status != http.StatusUnauthorized { + t.Fatalf("PV-M05 open legacy root configuration grants discovery: status=%d", got.status) + } + } + } +} + +type managedHTTPReply struct { + status int + body []byte + etag string +} + +type managedHTTPVolume struct { + ID string `json:"id"` + Name string `json:"name"` + Tenant string `json:"tenant"` + Kind string `json:"kind"` + State string `json:"state"` + SizeBytes int64 `json:"size_bytes"` +} + +func startManagedHTTPStack(t *testing.T, rootToken string) *stack { + t.Helper() + var cfg config.Config + body := fmt.Sprintf(`{"managed_volume_dir":%q}`, filepath.Join(t.TempDir(), "managed")) + if err := json.Unmarshal([]byte(body), &cfg); err != nil { + t.Fatal(err) + } + st := startConfigStack(t, rootToken, &cfg) + if rootToken == "" { + return st + } + if err := st.mgr.SetTenants(t.Context(), []tenants.TenantSpec{ + {Name: "t1", Token: "t1-token"}, {Name: "t2", Token: "t2-token"}, + }); err != nil { + t.Fatalf("configure tenants: %v", err) + } + return st +} + +func requestManagedVolume(t *testing.T, st *stack, method, route, token, body, etag string) managedHTTPReply { + t.Helper() + req, err := http.NewRequestWithContext(t.Context(), method, "http://"+st.addr+route, strings.NewReader(body)) + if err != nil { + t.Fatal(err) + } + if token != "" { + req.Header.Set("Authorization", "Bearer "+token) + } + req.Header.Set("Content-Type", "application/json") + if etag != "" { + req.Header.Set("If-Match", etag) + } + resp, err := http.DefaultClient.Do(req) + if err != nil { + t.Fatalf("%s %s: %v", method, route, err) + } + defer resp.Body.Close() + data, err := io.ReadAll(resp.Body) + if err != nil { + t.Fatal(err) + } + assertManagedReplyPrivate(t, data) + return managedHTTPReply{status: resp.StatusCode, body: data, etag: resp.Header.Get("ETag")} +} + +func decodeManagedVolume(t *testing.T, body []byte) managedHTTPVolume { + t.Helper() + var reply struct { + Volume managedHTTPVolume `json:"volume"` + } + if err := json.Unmarshal(body, &reply); err != nil { + t.Fatalf("decode managed volume: %v", err) + } + return reply.Volume +} + +func assertManagedReplyPrivate(t *testing.T, body []byte) { + t.Helper() + for _, secret := range []string{"root-token", "t1-token", "t2-token", "/tmp/", "disk.raw", "registry.json"} { + if strings.Contains(string(body), secret) { + t.Errorf("managed API exposed private host or credential detail") + } + } +} diff --git a/e2e/managed_volume_privacy_test.go b/e2e/managed_volume_privacy_test.go new file mode 100644 index 00000000..530ed292 --- /dev/null +++ b/e2e/managed_volume_privacy_test.go @@ -0,0 +1,126 @@ +package e2e + +import ( + "encoding/json/v2" + "fmt" + "net/http" + "net/http/httptest" + "os" + "path/filepath" + "slices" + "strings" + "testing" + + "github.com/cocoonstack/sandbox/sandboxd/config" + "github.com/cocoonstack/sandbox/sandboxd/server" + "github.com/cocoonstack/sandbox/sandboxd/tenants" +) + +func TestLiveAttachManagedVolumePrivacy(t *testing.T) { + st := startManagedHTTPStack(t, "root-token") + for name, token := range map[string]string{"private": "t1-token", "owned": "t2-token"} { + created := requestManagedVolume(t, st, http.MethodPut, "/v1/volumes/"+name, token, `{"size_gib":1}`, "") + if created.status != http.StatusCreated { + t.Fatalf("create %s: %d %s", name, created.status, created.body) + } + } + claim := requestManagedVolume(t, st, http.MethodPost, "/v1/claim", "t2-token", `{"template":"rt:24.04","net":"none","size":"small"}`, "") + if claim.status != http.StatusOK { + t.Fatalf("claim: %d %s", claim.status, claim.body) + } + var sb server.ClaimResponse + if err := json.Unmarshal(claim.body, &sb); err != nil { + t.Fatal(err) + } + route := "/v1/sandboxes/" + sb.ID + "/volumes" + foreign := requestManagedVolume(t, st, http.MethodPatch, route, "t2-token", `{"add":[{"name":"private","mount":"/volumes/probe"}]}`, "") + unknown := requestManagedVolume(t, st, http.MethodPatch, route, "t2-token", `{"add":[{"name":"unknown","mount":"/volumes/probe"}]}`, "") + if foreign.status != http.StatusNotFound || unknown.status != http.StatusNotFound || string(foreign.body) != string(unknown.body) { + t.Errorf("foreign=%d %s; unknown=%d %s; want identical 404 replies", foreign.status, foreign.body, unknown.status, unknown.body) + } + owned := requestManagedVolume(t, st, http.MethodPatch, route, "t2-token", `{"add":[{"name":"owned","mount":"/volumes/probe"}]}`, "") + if owned.status != http.StatusBadRequest || !strings.Contains(string(owned.body), "claim-time attachment only") { + t.Errorf("owned=%d %s; want unsupported live attach", owned.status, owned.body) + } +} + +type catalogPeerPlacer struct{ edgePlacer } + +func (*catalogPeerPlacer) VolumeCandidates(string, []string) []string { return []string{"peer:7777"} } +func (*catalogPeerPlacer) VolumeOwners([]string) []string { return []string{"peer:7777"} } +func (*catalogPeerPlacer) VolumeHolders() map[string]int { return map[string]int{"peer-only": 1} } + +func TestRootPeerCatalogRedirectWithManagedStore(t *testing.T) { + for _, enabled := range []bool{false, true} { + t.Run(fmt.Sprint(enabled), func(t *testing.T) { + var st *stack + if enabled { + st = startManagedHTTPStack(t, "root-token") + } else { + st = startConfigStack(t, "root-token", &config.Config{}) + } + srv := server.New(st.token, "", st.mgr, st.eng.real, &catalogPeerPlacer{}, nil, nil, nil) + ts := httptest.NewServer(srv.Handler()) + defer ts.Close() + defer srv.CloseRelays() + probe := *st + probe.addr = strings.TrimPrefix(ts.URL, "http://") + response := requestManagedVolume(t, &probe, http.MethodPost, "/v1/claim", "root-token", `{"template":"rt:24.04","net":"none","size":"small","volumes":[{"name":"peer-only"}]}`, "") + if response.status != http.StatusOK { + t.Fatalf("claim: %d %s", response.status, response.body) + } + var reply server.ClaimResponse + if err := json.Unmarshal(response.body, &reply); err != nil { + t.Fatal(err) + } + if !slices.Equal(reply.Redirect, []string{"peer:7777"}) || reply.ID != "" { + t.Fatalf("redirect reply = %+v", reply) + } + }) + } +} + +func TestStaticVolumePrivacyWithManagedStore(t *testing.T) { + for _, enabled := range []bool{false, true} { + t.Run(fmt.Sprint(enabled), func(t *testing.T) { + dir := t.TempDir() + path := filepath.Join(dir, "catalog.raw") + if err := os.WriteFile(path, []byte("catalog"), 0600); err != nil { + t.Fatal(err) + } + cfg := &config.Config{Volumes: []config.VolumeSpec{{Name: "private-static", Path: path, Tenants: []string{"t1"}}}} + expected := http.StatusBadRequest + if enabled { + cfg.ManagedVolumeDir = filepath.Join(dir, "managed") + expected = http.StatusNotFound + } + st := startConfigStack(t, "root-token", cfg) + if err := st.mgr.SetTenants(t.Context(), []tenants.TenantSpec{{Name: "t1", Token: "t1-token"}, {Name: "t2", Token: "t2-token"}}); err != nil { + t.Fatal(err) + } + t.Run("claim", func(t *testing.T) { + foreign := requestManagedVolume(t, st, http.MethodPost, "/v1/claim", "t2-token", `{"template":"rt:24.04","net":"none","size":"small","volumes":[{"name":"private-static"}]}`, "") + unknown := requestManagedVolume(t, st, http.MethodPost, "/v1/claim", "t2-token", `{"template":"rt:24.04","net":"none","size":"small","volumes":[{"name":"unknown"}]}`, "") + if foreign.status != expected || unknown.status != expected || string(foreign.body) != string(unknown.body) { + t.Fatalf("foreign=%d %s; unknown=%d %s; want identical %d replies", foreign.status, foreign.body, unknown.status, unknown.body, expected) + } + }) + t.Run("live-attach", func(t *testing.T) { + claim := requestManagedVolume(t, st, http.MethodPost, "/v1/claim", "t2-token", `{"template":"rt:24.04","net":"none","size":"small"}`, "") + if claim.status != http.StatusOK { + t.Fatalf("claim: %d %s", claim.status, claim.body) + } + var sb server.ClaimResponse + if err := json.Unmarshal(claim.body, &sb); err != nil { + t.Fatal(err) + } + route := "/v1/sandboxes/" + sb.ID + "/volumes" + foreign := requestManagedVolume(t, st, http.MethodPatch, route, "t2-token", `{"add":[{"name":"private-static","mount":"/volumes/probe"}]}`, "") + unknown := requestManagedVolume(t, st, http.MethodPatch, route, "t2-token", `{"add":[{"name":"unknown","mount":"/volumes/probe"}]}`, "") + if foreign.status != expected || unknown.status != expected || string(foreign.body) != string(unknown.body) { + t.Fatalf("foreign=%d %s; unknown=%d %s; want identical %d replies", foreign.status, foreign.body, unknown.status, unknown.body, expected) + } + }) + }) + } +} diff --git a/e2e/wire_parity_test.go b/e2e/wire_parity_test.go index 012f172e..7ab3937b 100644 --- a/e2e/wire_parity_test.go +++ b/e2e/wire_parity_test.go @@ -44,6 +44,10 @@ var wirePairs = []struct { {"checkpointRecord", false, types.Checkpoint{}, []string{"node", "key", "tenant", "policy_source", "no_egress", "guest_env", "archive"}}, {"volumeListResponse", false, server.VolumeListResponse{}, nil}, {"VolumeInfo", false, types.VolumeInfo{}, nil}, + {"ManagedVolumeInfo", false, types.ManagedVolumeInfo{}, nil}, + {"VolumeHolder", false, types.VolumeHolder{}, nil}, + {"VolumeCreate", true, types.VolumeCreate{}, nil}, + {"volumeResponse", false, server.VolumeResponse{}, nil}, {"VolumeState", false, types.VolumeState{}, nil}, {"SandboxVolumesResponse", false, types.SandboxVolumesResponse{}, nil}, {"VolumePatch", true, types.VolumePatch{}, nil}, @@ -117,6 +121,7 @@ func parseSDKFields(t *testing.T) map[string]map[string]string { t.Fatal(err) } out := map[string]map[string]string{} + embedded := map[string][]string{} for _, path := range paths { if strings.HasSuffix(path, "_test.go") { continue @@ -133,6 +138,11 @@ func parseSDKFields(t *testing.T) map[string]map[string]string { if st, ok := spec.Type.(*ast.StructType); ok { out[spec.Name.Name] = map[string]string{} for _, f := range st.Fields.List { + if len(f.Names) == 0 { + if id, ok := f.Type.(*ast.Ident); ok { + embedded[spec.Name.Name] = append(embedded[spec.Name.Name], id.Name) + } + } if f.Tag == nil { continue } @@ -145,6 +155,16 @@ func parseSDKFields(t *testing.T) map[string]map[string]string { return true }) } + var flatten func(string) map[string]string + flatten = func(name string) map[string]string { + for _, base := range embedded[name] { + maps.Copy(out[name], flatten(base)) + } + return out[name] + } + for name := range out { + flatten(name) + } return out } diff --git a/sandboxd/config/config.go b/sandboxd/config/config.go index fba3efba..fee0e129 100644 --- a/sandboxd/config/config.go +++ b/sandboxd/config/config.go @@ -274,9 +274,10 @@ type Config struct { DiskReserve *DiskReserve `json:"disk_reserve,omitempty"` - Volumes []VolumeSpec `json:"volumes,omitempty"` - Mesh *MeshConfig `json:"mesh,omitempty"` - Pools []PoolSpec `json:"pools"` + ManagedVolumeDir string `json:"managed_volume_dir,omitempty"` + Volumes []VolumeSpec `json:"volumes,omitempty"` + Mesh *MeshConfig `json:"mesh,omitempty"` + Pools []PoolSpec `json:"pools"` } // HasEgress reports whether the node can attach egress-lane VMs. @@ -398,6 +399,9 @@ func (c *Config) validateReplication() error { } func (c *Config) validateVolumes() error { + if err := c.ValidateManagedVolumeStore(); err != nil { + return err + } names := make(map[string]struct{}, len(c.Volumes)) paths := make(map[string]string, len(c.Volumes)) for _, volume := range c.Volumes { @@ -422,6 +426,58 @@ func (c *Config) validateVolumes() error { return nil } +func (c *Config) ValidateManagedVolumeStore() error { + if c.ManagedVolumeDir == "" { + return nil + } + if !filepath.IsAbs(c.ManagedVolumeDir) || filepath.Clean(c.ManagedVolumeDir) != c.ManagedVolumeDir || c.ManagedVolumeDir == "/" { + return fmt.Errorf("managed_volume_dir must be a clean absolute non-root path") + } + if c.APIToken == "" { + return fmt.Errorf("managed_volume_dir requires a nonempty api_token") + } + dataDir, err := filepath.Abs(c.DataDir) + if err != nil { + return fmt.Errorf("resolve data_dir: %w", err) + } + resolved, err := resolveExistingAncestors(dataDir) + if err != nil { + return fmt.Errorf("resolve data_dir: %w", err) + } + for _, root := range []string{dataDir, resolved} { + relative, err := filepath.Rel(root, c.ManagedVolumeDir) + if err != nil { + return fmt.Errorf("compare managed_volume_dir with data_dir: %w", err) + } + first, _, _ := strings.Cut(relative, string(filepath.Separator)) + if first == "readers" || first == "goldens" || strings.HasPrefix(first, "fork-") { + return fmt.Errorf("managed_volume_dir must be outside data_dir startup cleanup paths (readers, goldens, fork-*)") + } + } + for _, namespace := range []string{"readers", "goldens"} { + swept, err := resolveExistingAncestors(filepath.Join(dataDir, namespace)) + if err != nil { + return fmt.Errorf("resolve data_dir cleanup path: %w", err) + } + relative, err := filepath.Rel(swept, c.ManagedVolumeDir) + if err != nil { + return fmt.Errorf("compare managed_volume_dir with cleanup path: %w", err) + } + if relative != ".." && !strings.HasPrefix(relative, ".."+string(filepath.Separator)) { + return fmt.Errorf("managed_volume_dir must be outside data_dir %s cleanup path", namespace) + } + relative, err = filepath.Rel(c.ManagedVolumeDir, swept) + if err != nil { + return fmt.Errorf("compare cleanup path with managed_volume_dir: %w", err) + } + first, _, _ := strings.Cut(relative, string(filepath.Separator)) + if first == "disks" { + return fmt.Errorf("data_dir %s cleanup path must be outside managed volume backing storage", namespace) + } + } + return nil +} + func (c *Config) validatePreview() error { if c.PreviewListen == "" { return nil @@ -663,6 +719,25 @@ func StorageBytes(s string) (int64, error) { return units.RAMInBytes(s) } +func resolveExistingAncestors(path string) (string, error) { + suffix := "" + for { + _, err := os.Lstat(path) + if err == nil { + resolved, err := filepath.EvalSymlinks(path) + if err != nil { + return "", err + } + return filepath.Join(resolved, suffix), nil + } + if !os.IsNotExist(err) || filepath.Dir(path) == path { + return "", err + } + suffix = filepath.Join(filepath.Base(path), suffix) + path = filepath.Dir(path) + } +} + func validatePolicy(p *egress.Policy, secrets map[string]struct{}) error { if p == nil { return nil diff --git a/sandboxd/config/managed_volume_test.go b/sandboxd/config/managed_volume_test.go new file mode 100644 index 00000000..aec306e1 --- /dev/null +++ b/sandboxd/config/managed_volume_test.go @@ -0,0 +1,137 @@ +package config + +import ( + "os" + "path/filepath" + "testing" +) + +func TestManagedVolumeStoreExcludesStartupCleanupPaths(t *testing.T) { + root := t.TempDir() + data := filepath.Join(root, "data") + for _, tc := range []struct { + path string + allow bool + }{ + {filepath.Join(data, "managed"), true}, + {filepath.Join(root, "outside"), true}, + {filepath.Join(data, "readers-backup", "managed"), true}, + {filepath.Join(data, "forks", "managed"), true}, + {filepath.Join(data, "readers"), false}, + {filepath.Join(data, "readers", "managed"), false}, + {filepath.Join(data, "fork-state"), false}, + {filepath.Join(data, "fork-state", "managed"), false}, + {filepath.Join(data, "goldens"), false}, + {filepath.Join(data, "goldens", "managed"), false}, + } { + t.Run(tc.path, func(t *testing.T) { + cfg := Config{DataDir: data, ManagedVolumeDir: tc.path, APIToken: "root"} + err := cfg.ValidateManagedVolumeStore() + if (err == nil) != tc.allow { + t.Fatalf("path %q accepted=%v, want %v: %v", tc.path, err == nil, tc.allow, err) + } + }) + } +} + +func TestManagedVolumeStoreResolvesDataDirectoryAliases(t *testing.T) { + root := t.TempDir() + real := filepath.Join(root, "real") + alias := filepath.Join(root, "alias") + if err := os.Mkdir(real, 0700); err != nil { + t.Fatal(err) + } + if err := os.Symlink(real, alias); err != nil { + t.Fatal(err) + } + t.Chdir(root) + for _, tc := range []struct { + name, data, managed string + allow bool + }{ + {"alias-readers", alias, filepath.Join(real, "readers", "managed"), false}, + {"alias-goldens", alias, filepath.Join(real, "goldens"), false}, + {"alias-fork", alias, filepath.Join(real, "fork-persistent"), false}, + {"alias-new-suffix", filepath.Join(alias, "new", "data"), filepath.Join(real, "new", "data", "readers", "managed"), false}, + {"alias-safe", alias, filepath.Join(real, "managed"), true}, + {"relative", "real", filepath.Join(real, "readers", "managed"), false}, + {"relative-safe", "real", filepath.Join(real, "readers-safe"), true}, + {"empty", "", filepath.Join(root, "fork-store"), false}, + {"empty-safe", "", filepath.Join(root, "managed"), true}, + {"ancestor", filepath.Join(real, "data"), real, true}, + {"same-root", real, real, true}, + } { + t.Run(tc.name, func(t *testing.T) { + cfg := Config{DataDir: tc.data, ManagedVolumeDir: tc.managed, APIToken: "root"} + err := cfg.ValidateManagedVolumeStore() + if (err == nil) != tc.allow { + t.Fatalf("data=%q managed=%q accepted=%v, want %v: %v", tc.data, tc.managed, err == nil, tc.allow, err) + } + }) + } + if _, err := os.Stat(filepath.Join(real, "new")); !os.IsNotExist(err) { + t.Fatalf("validation created data directories: %v", err) + } +} + +func TestManagedVolumeStoreRejectsCleanupDirectoryAliases(t *testing.T) { + for _, namespace := range []string{"readers", "goldens"} { + t.Run(namespace, func(t *testing.T) { + root := t.TempDir() + data := filepath.Join(root, "data") + external := filepath.Join(root, "external") + for _, dir := range []string{data, external} { + if err := os.Mkdir(dir, 0700); err != nil { + t.Fatal(err) + } + } + if err := os.Symlink(external, filepath.Join(data, namespace)); err != nil { + t.Fatal(err) + } + for _, tc := range []struct { + path string + allow bool + }{ + {external, false}, + {filepath.Join(external, "managed"), false}, + {filepath.Join(data, "managed"), true}, + {root, true}, + } { + cfg := Config{DataDir: data, ManagedVolumeDir: tc.path, APIToken: "root"} + err := cfg.ValidateManagedVolumeStore() + if (err == nil) != tc.allow { + t.Errorf("%s alias target %q accepted=%v, want %v: %v", namespace, tc.path, err == nil, tc.allow, err) + } + } + if _, err := os.Stat(filepath.Join(external, "managed")); !os.IsNotExist(err) { + t.Fatalf("validation created managed directory: %v", err) + } + }) + } +} + +func TestManagedVolumeStoreProtectsBackingDirectoryFromCleanupAliases(t *testing.T) { + for _, namespace := range []string{"readers", "goldens"} { + for _, target := range []string{"disks", "disks/nested", "disks-backup", "work/readers"} { + t.Run(namespace+"/"+target, func(t *testing.T) { + root := t.TempDir() + data := filepath.Join(root, "data") + managed := filepath.Join(root, "managed") + aliasTarget := filepath.Join(managed, target) + for _, dir := range []string{data, aliasTarget} { + if err := os.MkdirAll(dir, 0700); err != nil { + t.Fatal(err) + } + } + if err := os.Symlink(aliasTarget, filepath.Join(data, namespace)); err != nil { + t.Fatal(err) + } + cfg := Config{DataDir: data, ManagedVolumeDir: managed, APIToken: "root"} + allow := target == "disks-backup" || target == "work/readers" + if err := cfg.ValidateManagedVolumeStore(); (err == nil) != allow { + t.Fatalf("cleanup alias %q accepted=%v, want %v: %v", target, err == nil, allow, err) + } + }) + } + } +} diff --git a/sandboxd/pool/claim.go b/sandboxd/pool/claim.go index 683074ee..d7f64d8d 100644 --- a/sandboxd/pool/claim.go +++ b/sandboxd/pool/claim.go @@ -55,7 +55,7 @@ func (m *Manager) ClaimWarm(ctx context.Context, key types.PoolKey, o ClaimOptio } // holds belong to this path until finalize; past it the sandbox carries them. reserved := applied - defer func() { m.unreserveVolumes(reserved) }() + defer func() { m.releaseUnbound(volumeSpecs, reserved) }() m.mu.Lock() var sb *types.Sandbox if p := m.pools[key]; p != nil { @@ -68,7 +68,7 @@ func (m *Manager) ClaimWarm(ctx context.Context, key types.PoolKey, o ClaimOptio return nil, ErrNoWarm } if volumeErr := m.applyVolumes(ctx, sb, volumeSpecs, applied); volumeErr != nil { - m.abortVolumeClaim(ctx, sb.VMName, &reserved) + m.abortVolumeClaim(ctx, sb.VMName, &reserved, volumeSpecs) return nil, volumeErr } o.apply(sb) @@ -89,7 +89,7 @@ func (m *Manager) ClaimProvision(ctx context.Context, key types.PoolKey, o Claim return nil, err } reserved := applied - defer func() { m.unreserveVolumes(reserved) }() + defer func() { m.releaseUnbound(volumeSpecs, reserved) }() golden, err := m.resolveGolden(ctx, key, o.Tenant) if err != nil { return nil, fmt.Errorf("resolve template: %w", err) @@ -104,7 +104,7 @@ func (m *Manager) ClaimProvision(ctx context.Context, key types.PoolKey, o Claim return nil, err } if volumeErr := m.applyVolumes(ctx, sb, volumeSpecs, applied); volumeErr != nil { - m.abortVolumeClaim(ctx, sb.VMName, &reserved) + m.abortVolumeClaim(ctx, sb.VMName, &reserved, volumeSpecs) return nil, volumeErr } sb.TemplateDigest = golden.templateDigest @@ -309,6 +309,10 @@ func (m *Manager) releaseResolved(ctx context.Context, id string, sb *types.Sand marked = ck m.mu.Lock() } + if err := m.retireManaged(id); err != nil { + m.mu.Unlock() + return err + } volumeHolds := m.cancelVolumeRun(sb) snap, ck, vmName := sb.HibernateSnap, sb.ArchiveCk, sb.VMName if ck != "" { @@ -454,13 +458,26 @@ func (m *Manager) finalize(ctx context.Context, sb *types.Sandbox, ttl time.Dura func (m *Manager) finalizeBatch(ctx context.Context, sbs []*types.Sandbox, ttl time.Duration, claimRefPrefix string, inherited bool) error { now := time.Now() for _, sb := range sbs { + id := sb.ID + _, managed := m.managedHold(id) stampIdentity(sb, clampTTL(ttl)) + if managed { + sb.ID = id + } if claimRefPrefix != "" { sb.ClaimRef = claimRefPrefix + sb.ID } sb.TouchAt(now) } - err := m.checkInjects(m.view.Load(), sbs) + var err error + for _, sb := range sbs { + if err = m.activateManaged(sb.ID); err != nil { + break + } + } + if err == nil { + err = m.checkInjects(m.view.Load(), sbs) + } if err == nil { err = m.deliverEnv(ctx, sbs, inherited) } @@ -510,6 +527,9 @@ func (m *Manager) finalizeBatch(ctx context.Context, sbs []*types.Sandbox, ttl t // rollbackClaim unwinds a claim batch: a NIC that cannot be locked is never handed out. func (m *Manager) rollbackClaim(ctx context.Context, sbs []*types.Sandbox) { + for _, sb := range sbs { + _ = m.retireManaged(sb.ID) + } m.mu.Lock() ids := make([]string, len(sbs)) holds := make([][]types.Volume, len(sbs)) @@ -529,7 +549,13 @@ func (m *Manager) rollbackClaim(ctx context.Context, sbs []*types.Sandbox) { } // abortVolumeClaim removes a failed claim's VM and clears reserved against a double release. -func (m *Manager) abortVolumeClaim(ctx context.Context, vmName string, reserved *[]types.Volume) { +func (m *Manager) abortVolumeClaim(ctx context.Context, vmName string, reserved *[]types.Volume, specs []resolvedVolume) { + if len(specs) > 0 && specs[0].holdID != "" { + if h, ok := m.managedHold(specs[0].holdID); ok && h.VMName == "" { + m.releaseUnbound(specs, *reserved) + *reserved = nil + } + } m.removeOrRetry(ctx, vmName, "", volumeTeardown{holds: *reserved}) *reserved = nil } @@ -572,6 +598,9 @@ func (m *Manager) reapOnce(ctx context.Context) { } expired = append(expired, victim{action: action, id: id, sb: sb}) default: + if err := m.retireManaged(id); err != nil { + continue + } expired = append(expired, victim{action: reapDestroy, id: id, vmName: sb.VMName, snap: sb.HibernateSnap, tenant: sb.Tenant, sb: sb, holds: m.cancelVolumeRun(sb)}) m.dropClaimLocked(sb) dropped = append(dropped, id) @@ -754,6 +783,15 @@ func (m *Manager) admitOptions(ctx context.Context, key types.PoolKey, o ClaimOp if err != nil { return nil, nil, err } + for _, v := range volumeSpecs { + if v.managedID != "" { + if err := m.tenantRemoved(ctx, o.Tenant); err != nil { + return nil, nil, err + } + applied, err := m.admitManaged(o.Tenant, volumeSpecs) + return volumeSpecs, applied, err + } + } applied, err := m.admitVolumes(o.Tenant, volumeSpecs) return volumeSpecs, applied, err } diff --git a/sandboxd/pool/failover.go b/sandboxd/pool/failover.go index dd0896f4..926a8404 100644 --- a/sandboxd/pool/failover.go +++ b/sandboxd/pool/failover.go @@ -411,6 +411,13 @@ func (m *Manager) forgetClaim(ctx context.Context, sb *types.Sandbox) bool { m.mu.Unlock() return false } + if _, managed := m.managedHold(sb.ID); managed { + if err := m.retireManaged(sb.ID); err != nil { + m.mu.Unlock() + return false + } + m.cancelVolumeRun(sb) + } m.dropClaimLocked(sb) js := m.store.del(sb.ID) m.mu.Unlock() diff --git a/sandboxd/pool/managed_attachment.go b/sandboxd/pool/managed_attachment.go new file mode 100644 index 00000000..b75f4f84 --- /dev/null +++ b/sandboxd/pool/managed_attachment.go @@ -0,0 +1,383 @@ +package pool + +import ( + "context" + "errors" + "fmt" + "os" + "strconv" + "strings" + "syscall" + + "github.com/cocoonstack/sandbox/sandboxd/types" +) + +func readProcessIdentity(pid int) (processIdentity, error) { + return readProcessIdentityAt("/proc", pid) +} + +func readProcessIdentityAt(procDir string, pid int) (processIdentity, error) { + if pid <= 0 { + return processIdentity{}, errors.New("VMM PID unavailable") + } + boot, err := os.ReadFile(procDir + "/sys/kernel/random/boot_id") + if err != nil { + // Missing boot metadata cannot prove that the PID is gone. + return processIdentity{}, fmt.Errorf("read VMM boot identity: %v", err) + } + bootID := strings.TrimSpace(string(boot)) + if bootID == "" { + return processIdentity{}, errors.New("empty VMM boot identity") + } + stat, err := os.ReadFile(procDir + "/" + strconv.Itoa(pid) + "/stat") + if err != nil { + return processIdentity{}, err + } + end := strings.LastIndexByte(string(stat), ')') + if end < 0 { + return processIdentity{}, errors.New("invalid process identity") + } + fields := strings.Fields(string(stat[end+1:])) + if len(fields) < 20 { + return processIdentity{}, errors.New("invalid process identity") + } + ticks, err := strconv.ParseUint(fields[19], 10, 64) + if err != nil { + return processIdentity{}, err + } + return processIdentity{BootID: bootID, PID: pid, Start: ticks}, nil +} + +func (r *volumeRegistry) processGone(p processIdentity) bool { + if p.PID <= 0 || p.Start == 0 || p.BootID == "" { + return false + } + current, err := r.process(p.PID) + if errors.Is(err, os.ErrNotExist) { + return true + } + return err == nil && current != p +} + +func (m *Manager) admitManaged(tenant string, volumes []resolvedVolume) ([]types.Volume, error) { + applied := appliedVolumes(volumes) + m.mu.Lock() + defer m.mu.Unlock() + if err := m.quotaErr(1, tenant); err != nil { + return nil, err + } + r := m.managed + r.mu.Lock() + defer r.mu.Unlock() + if r.fault != nil { + return nil, managedStorageError() + } + h := volumeHold{SandboxID: "sb_" + randHex(8), Tenant: tenant, State: holdAttaching} + for _, volume := range volumes { + if volume.managedID != "" { + v, ok := r.data.Volumes[volume.applied.Name] + if !ok || v.ID != volume.managedID || (tenant != "" && v.Tenant != tenant) { + return nil, managedNotFound() + } + if v.State != volumeReady || r.verify(v) != nil { + return nil, managedStorageError() + } + for _, other := range r.data.Holds { + for _, ref := range other.Refs { + if ref.ID == v.ID && (ref.Volume.RW() || volume.applied.RW()) { + return nil, &VolumeError{Status: 409, Code: "volume_busy", Message: "volume is held by a sandbox", Volume: v.Name, Holders: r.holders(v.ID)} + } + } + } + h.Refs = append(h.Refs, volumeRef{Volume: volume.applied, ID: v.ID, Path: r.path(v.ID), Backing: v.Backing}) + } else { + identity, err := catalogBacking(volume.disk.Path) + if err != nil { + return nil, ErrVolumeUnavailable + } + h.Refs = append(h.Refs, volumeRef{Volume: volume.applied, Path: volume.disk.Path, Backing: identity}) + } + } + if err := m.reserveVolumes(applied); err != nil { + return nil, err + } + if err := m.confirmVolumesClean(volumes); err != nil { + m.releaseVolumes(applied) + return nil, err + } + r.data.Holds[h.SandboxID] = h + for i := range volumes { + volumes[i].holdID = h.SandboxID + } + if err := r.commit(); err != nil { + return nil, err + } + return applied, nil +} + +func (m *Manager) releaseUnbound(volumes []resolvedVolume, reserved []types.Volume) { + if len(reserved) == 0 { + return + } + if len(volumes) == 0 || volumes[0].holdID == "" { + m.unreserveVolumes(reserved) + return + } + r := m.managed + m.mu.Lock() + defer m.mu.Unlock() + r.mu.Lock() + defer r.mu.Unlock() + id := volumes[0].holdID + h, ok := r.data.Holds[id] + if !ok || h.VMName != "" || r.fault != nil { + return + } + delete(r.data.Holds, id) + if r.commit() == nil { + m.releaseVolumes(reserved) + } +} + +func (m *Manager) bindManaged(ctx context.Context, sb *types.Sandbox, volumes []resolvedVolume) error { + if len(volumes) == 0 || volumes[0].holdID == "" { + return nil + } + id := volumes[0].holdID + sb.ID = id + rec, present, err := m.eng.Inspect(ctx, sb.VMName) + if err != nil || !present { + return managedStorageError() + } + p, err := m.managed.process(rec.PID) + if err != nil { + return managedStorageError() + } + r := m.managed + r.mu.Lock() + defer r.mu.Unlock() + if r.fault != nil { + return managedStorageError() + } + h, ok := r.data.Holds[id] + if !ok || h.State != holdAttaching || h.VMName != "" { + return managedStorageError() + } + h.VMName = sb.VMName + h.Process = p + r.data.Holds[id] = h + return r.commit() +} + +func (m *Manager) activateManaged(id string) error { + r := m.managed + if r == nil { + return nil + } + r.mu.Lock() + defer r.mu.Unlock() + h, ok := r.data.Holds[id] + if !ok { + return nil + } + if r.fault != nil || h.State == holdRetiring { + return managedStorageError() + } + h.State = holdClaimed + r.data.Holds[id] = h + return r.commit() +} + +func (m *Manager) finishManaged(id string, clears []string) bool { + r := m.managed + m.mu.Lock() + defer m.mu.Unlock() + r.mu.Lock() + defer r.mu.Unlock() + h, ok := r.data.Holds[id] + if !ok { + return true + } + if r.fault != nil { + return false + } + if err := m.store.commit(m.store.del(id)); err != nil { + return false + } + for _, path := range clears { + for _, ref := range h.Refs { + if ref.Path == path { + if identity, err := catalogBacking(path); err == nil && identity == ref.Backing { + _ = clearVolumeDirty(path) + } + break + } + } + } + delete(r.data.Holds, id) + if r.commit() != nil { + r.data.Holds[id] = h + return false + } + refs := make([]types.Volume, len(h.Refs)) + for i, ref := range h.Refs { + refs[i] = ref.Volume + } + m.releaseVolumes(refs) + return true +} + +func (m *Manager) recoverManaged(ctx context.Context, claims map[string]*types.Sandbox, live map[string]types.VMRecord) (map[string]bool, error) { + owned := map[string]bool{} + r := m.managed + if r == nil { + return owned, nil + } + r.mu.Lock() + if r.fault != nil { + r.mu.Unlock() + return owned, managedStorageError() + } + holds := make([]volumeHold, 0, len(r.data.Holds)) + for _, h := range r.data.Holds { + holds = append(holds, h) + } + r.mu.Unlock() + for _, h := range holds { + m.mu.Lock() + for _, ref := range h.Refs { + m.adoptVolumes([]types.Volume{ref.Volume}) + } + m.mu.Unlock() + sb, exists := claims[h.SandboxID] + rec, present := live[h.VMName] + valid := exists && sb.VMName == h.VMName && sb.Tenant == h.Tenant && sb.Failed != egressQuarantineFailure && h.State != holdRetiring && present + if valid { + identity, err := r.process(rec.PID) + if err != nil && !errors.Is(err, os.ErrNotExist) { + return owned, managedStorageError() + } + valid = err == nil && identity == h.Process + } + if valid { + for _, volume := range heldVolumes(sb) { + matched := false + for _, ref := range h.Refs { + if ref.Volume == volume { + matched = true + break + } + } + if !matched { + return owned, managedStorageError() + } + } + owned[h.VMName] = true + if err := m.activateManaged(h.SandboxID); err != nil { + return owned, err + } + continue + } + delete(claims, h.SandboxID) + if err := m.retireManaged(h.SandboxID); err != nil { + return owned, err + } + if h.VMName == "" { + if !m.finishManaged(h.SandboxID, nil) { + return owned, managedStorageError() + } + continue + } + owned[h.VMName] = true + m.removeOrRetry(ctx, h.VMName, h.SandboxID, volumeTeardown{managedID: h.SandboxID}) + } + for _, sb := range claims { + for _, v := range heldVolumes(sb) { + r.mu.Lock() + _, managed := r.data.Volumes[v.Name] + _, held := r.data.Holds[sb.ID] + r.mu.Unlock() + if managed && !held { + return owned, managedStorageError() + } + } + } + return owned, nil +} + +func (m *Manager) managedLiveAdd(id string, volumes []resolvedVolume) error { + if len(volumes) == 0 { + return nil + } + r := m.managed + if r == nil { + return nil + } + r.mu.Lock() + defer r.mu.Unlock() + h, ok := r.data.Holds[id] + if !ok { + return nil + } + if r.fault != nil || h.State != holdClaimed { + return managedStorageError() + } + for _, v := range volumes { + for _, ref := range h.Refs { + if ref.Volume.Name == v.applied.Name { + return volumeError(409, "volume_attachment_uncertain", "previous attachment requires sandbox release") + } + } + identity, err := catalogBacking(v.disk.Path) + if err != nil { + return ErrVolumeUnavailable + } + h.Refs = append(h.Refs, volumeRef{Volume: v.applied, Path: v.disk.Path, Backing: identity}) + } + r.data.Holds[id] = h + return r.commit() +} + +func (m *Manager) managedLiveRemove(id string, volumes []types.Volume) error { + if len(volumes) == 0 { + return nil + } + r := m.managed + if r == nil { + return nil + } + r.mu.Lock() + defer r.mu.Unlock() + h, ok := r.data.Holds[id] + if !ok { + return nil + } + if r.fault != nil || h.State != holdClaimed { + return managedStorageError() + } + remove := map[string]bool{} + for _, v := range volumes { + remove[v.Name] = true + } + refs := make([]volumeRef, 0, len(h.Refs)) + for _, ref := range h.Refs { + if !remove[ref.Volume.Name] { + refs = append(refs, ref) + } + } + h.Refs = refs + r.data.Holds[id] = h + return r.commit() +} + +func catalogBacking(path string) (backingIdentity, error) { + st, err := os.Stat(path) + if err != nil { + return backingIdentity{}, err + } + if !st.Mode().IsRegular() { + return backingIdentity{}, errors.New("catalog backing is not a regular file") + } + raw := st.Sys().(*syscall.Stat_t) + return backingIdentity{Device: uint64(raw.Dev), Inode: raw.Ino, Size: st.Size()}, nil +} diff --git a/sandboxd/pool/managed_integration_test.go b/sandboxd/pool/managed_integration_test.go new file mode 100644 index 00000000..f5c8c0a2 --- /dev/null +++ b/sandboxd/pool/managed_integration_test.go @@ -0,0 +1,284 @@ +package pool + +import ( + "context" + "errors" + "os" + "path/filepath" + "strings" + "testing" + "time" + + "github.com/cocoonstack/sandbox/sandboxd/config" + "github.com/cocoonstack/sandbox/sandboxd/engine" + "github.com/cocoonstack/sandbox/sandboxd/tenants" + "github.com/cocoonstack/sandbox/sandboxd/types" +) + +func TestManagedStoreRequiresAuthenticatedRootConfiguration(t *testing.T) { + cfg := managedConfig(t) + cfg.APIToken = "" + _, err := NewManager(t.Context(), cfg, newManagedEngine(), testSecrets(t), Backends{}) + if err == nil || !strings.Contains(err.Error(), "api_token") { + t.Fatalf("anonymous managed store opened: %v", err) + } +} + +func TestManagedNamesAndCatalogSymlinksSurviveRestart(t *testing.T) { + cfg := managedConfig(t) + image := writeVolumeImage(t, "catalog.img", "catalog") + link := filepath.Join(t.TempDir(), "catalog-link") + if err := os.Symlink(image, link); err != nil { + t.Fatal(err) + } + cfg.Volumes = []config.VolumeSpec{{Name: "catalog", Path: link, Writable: true}} + e := newManagedEngine() + m := openManagedTest(t, cfg, e) + v := createManagedTest(t, m, "kb-data") + sb, err := m.ClaimProvision(t.Context(), testKey, managedOptions("kb-data", "catalog")) + if err != nil { + t.Fatal(err) + } + m.Close() + m = openManagedTest(t, cfg, e) + if err := m.Reconcile(t.Context()); err != nil { + t.Fatal(err) + } + if got, err := m.GetVolume("kb-data", "t1"); err != nil || got.ID != v.ID { + t.Fatalf("reopened name: %+v %v", got, err) + } + if err := m.Release(t.Context(), sb.ID, Cred{Operator: true}); err != nil { + t.Fatal(err) + } +} + +func TestManagedRejectsCatalogSymlinkToManagedBacking(t *testing.T) { + cfg := managedConfig(t) + e := newManagedEngine() + m := openManagedTest(t, cfg, e) + v := createManagedTest(t, m, "data") + link := filepath.Join(t.TempDir(), "catalog-link") + if err := os.Symlink(m.managed.path(volumeID(v.ID)), link); err != nil { + t.Fatal(err) + } + m.Close() + cfg.Volumes = []config.VolumeSpec{{Name: "alias", Path: link, Writable: true}} + if next, err := NewManager(t.Context(), cfg, e, testSecrets(t), testBackends(t, cfg)); err == nil { + next.Close() + t.Fatal("managed backing alias accepted as an independent catalog writer") + } +} + +func TestManagedRejectsCatalogReplacementBeforeAttach(t *testing.T) { + for _, alias := range []bool{false, true} { + t.Run(map[bool]string{false: "different inode", true: "managed alias"}[alias], func(t *testing.T) { + cfg := managedConfig(t) + path := writeVolumeImage(t, "catalog.img", "original") + cfg.Volumes = []config.VolumeSpec{{Name: "catalog", Path: path, Writable: true}} + e := newManagedEngine() + m := openManagedTest(t, cfg, e) + v := createManagedTest(t, m, "data") + specs, _, err := m.admitOptions(t.Context(), testKey, managedOptions("data", "catalog")) + if err != nil { + t.Fatal(err) + } + rec, err := e.RunCold(t.Context(), "sbx-replaced", testKey, "") + if err != nil { + t.Fatal(err) + } + sb := &types.Sandbox{VMName: rec.Config.Name, VsockSocket: rec.VsockSocket, Key: testKey} + if err := m.bindManaged(t.Context(), sb, specs); err != nil { + t.Fatal(err) + } + if err := os.Rename(path, path+".old"); err != nil { + t.Fatal(err) + } + if alias { + if err := os.Symlink(m.managed.path(volumeID(v.ID)), path); err != nil { + t.Fatal(err) + } + } else if err := os.WriteFile(path, []byte("replacement"), 0600); err != nil { + t.Fatal(err) + } + for _, spec := range specs { + if spec.applied.Name != "catalog" { + continue + } + if err := m.applyVolume(t.Context(), sb, spec, false); err == nil { + t.Fatal("replacement attached after admission pinned another backing") + } + } + e.mu.Lock() + defer e.mu.Unlock() + if len(e.volumeSpecs) != 0 { + t.Fatalf("device effects before replacement rejection: %v", e.volumeOps) + } + }) + } +} + +func TestManagedReleaseDoesNotClearReplacementCatalogDirtyMarker(t *testing.T) { + cfg := managedConfig(t) + path := writeVolumeImage(t, "catalog.img", "old") + cfg.Volumes = []config.VolumeSpec{{Name: "catalog", Path: path, Writable: true}} + e := newManagedEngine() + m := openManagedTest(t, cfg, e) + createManagedTest(t, m, "data") + sb, err := m.ClaimProvision(t.Context(), testKey, ClaimOptions{Tenant: "t1", Volumes: []types.Volume{ + {Name: "data", Mode: "rw"}, {Name: "catalog", Mode: "rw"}, + }}) + if err != nil { + t.Fatal(err) + } + if err := os.Rename(path, path+".old"); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(path, []byte("replacement"), 0600); err != nil { + t.Fatal(err) + } + if err := m.Release(t.Context(), sb.ID, Cred{Operator: true}); err != nil { + t.Fatal(err) + } + if !volumeDirty(path) { + t.Fatal("release cleared a replacement backing's dirty marker") + } + _, err = m.ClaimProvision(t.Context(), testKey, ClaimOptions{Tenant: "t1", Volumes: []types.Volume{{Name: "catalog"}}}) + if !errors.Is(err, ErrVolumeNeedsRecovery) { + t.Fatalf("replacement reader admitted: %v", err) + } +} + +func TestManagedSettledRetryDrainsPreviouslyUncertainCatalogRef(t *testing.T) { + cfg := managedConfig(t) + cfg.Volumes = []config.VolumeSpec{{Name: "catalog", Path: writeVolumeImage(t, "catalog.img", "catalog"), Writable: true}} + e := newManagedEngine() + m := openManagedTest(t, cfg, e) + createManagedTest(t, m, "data") + sb, err := m.ClaimProvision(t.Context(), testKey, managedOptions("data")) + if err != nil { + t.Fatal(err) + } + e.diskAttachErrFor, e.diskAttachErr = "catalog", errors.New("uncertain attach") + _, err = m.PatchVolumes(t.Context(), sb.ID, "", types.VolumePatch{Add: []types.Volume{{Name: "catalog"}}}) + if err == nil { + t.Fatal("injected attach succeeded") + } + e.diskAttachErr, e.diskAttachErrFor = nil, "" + if err := os.Mkdir(m.store.path+".tmp", 0700); err != nil { + t.Fatal(err) + } + _, err = m.PatchVolumes(t.Context(), sb.ID, "", types.VolumePatch{Remove: []string{"catalog"}}) + if err == nil { + t.Fatal("injected claim commit succeeded") + } + if err := os.Remove(m.store.path + ".tmp"); err != nil { + t.Fatal(err) + } + waitFor(t, m.store.synced) + if _, err := m.PatchVolumes(t.Context(), sb.ID, "", types.VolumePatch{}); err != nil { + t.Fatal(err) + } + h, ok := m.managedHold(sb.ID) + if !ok || len(h.Refs) != 1 { + t.Fatalf("settled retry leaked uncertain reference: %+v", h) + } + writer, err := m.ClaimProvision(t.Context(), testKey, managedOptions("catalog")) + if err != nil { + t.Fatalf("confirmed detached catalog still busy: %v", err) + } + if err := m.Release(t.Context(), writer.ID, Cred{Operator: true}); err != nil { + t.Fatal(err) + } +} + +type managedCancelEngine struct { + *managedTestEngine + entered chan struct{} +} + +func (e *managedCancelEngine) DiskAttach(ctx context.Context, name string, spec engine.VolumeSpec) error { + if spec.Name == "catalog" { + close(e.entered) + <-ctx.Done() + return ctx.Err() + } + return e.managedTestEngine.DiskAttach(ctx, name, spec) +} + +func TestManagedReleaseCancelsExistingVolumeRunAndRetainsFullUnion(t *testing.T) { + cfg := managedConfig(t) + cfg.Volumes = []config.VolumeSpec{{Name: "catalog", Path: writeVolumeImage(t, "catalog.img", "catalog"), Writable: true}} + e := &managedCancelEngine{managedTestEngine: newManagedEngine(), entered: make(chan struct{})} + m, err := NewManager(t.Context(), cfg, e, testSecrets(t), testBackends(t, cfg)) + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { m.Close() }) + m.managed.process = e.identity + if err := m.PutTenant(t.Context(), tenants.TenantSpec{Name: "t1", Token: "t1-token"}); err != nil { + t.Fatal(err) + } + createManagedTest(t, m, "data") + sb, err := m.ClaimProvision(t.Context(), testKey, managedOptions("data")) + if err != nil { + t.Fatal(err) + } + ctx, cancel := context.WithCancel(t.Context()) + defer cancel() + done := make(chan error, 1) + go func() { + _, err := m.PatchVolumes(ctx, sb.ID, "", types.VolumePatch{Add: []types.Volume{{Name: "catalog"}}}) + done <- err + }() + <-e.entered + e.falseRemove.Store(true) + if err := m.Release(t.Context(), sb.ID, Cred{Operator: true}); err == nil { + t.Fatal("surviving VMM reported removed") + } + select { + case err := <-done: + if !errors.Is(err, context.Canceled) { + t.Fatalf("mutation cancellation: %v", err) + } + case <-time.After(time.Second): + t.Fatal("release did not cancel the live volume run") + } + h, ok := m.managedHold(sb.ID) + if !ok || h.State != holdRetiring || len(h.Refs) != 2 { + t.Fatalf("retirement lost possible device references: %+v", h) + } + _, err = m.ClaimProvision(t.Context(), testKey, managedOptions("catalog")) + if !errors.Is(err, ErrVolumeBusy) { + t.Fatalf("uncertain catalog writer admitted: %v", err) + } + e.falseRemove.Store(false) + m.retryRemovals(t.Context()).Wait() + if _, held := m.managedHold(sb.ID); held { + t.Fatal("hold retained after confirmed VMM removal") + } +} + +func TestManagedQuarantineStillRemovesVMWhenRetirementCannotCommit(t *testing.T) { + cfg := managedConfig(t) + e := newManagedEngine() + m := openManagedTest(t, cfg, e) + createManagedTest(t, m, "data") + sb, err := m.ClaimProvision(t.Context(), testKey, managedOptions("data")) + if err != nil { + t.Fatal(err) + } + m.managed.commitFile = func(string, []byte) error { return errors.New("retirement unavailable") } + if m.quarantineClaim(t.Context(), sb) { + t.Fatal("uncommitted retirement reported fully cleaned") + } + if _, present, err := e.Inspect(t.Context(), sb.VMName); err != nil || present { + t.Fatalf("quarantined VM remains alive: %v %v", present, err) + } + current, ok := m.Sandbox(sb.ID) + if !ok || current.Failed == "" { + t.Fatalf("quarantined row remained serviceable: %+v", current) + } + if _, held := m.managedHold(sb.ID); !held { + t.Fatal("failed retirement discarded disk ownership") + } +} diff --git a/sandboxd/pool/managed_live.go b/sandboxd/pool/managed_live.go new file mode 100644 index 00000000..7e5396c5 --- /dev/null +++ b/sandboxd/pool/managed_live.go @@ -0,0 +1,32 @@ +package pool + +import ( + "fmt" + + "github.com/cocoonstack/sandbox/sandboxd/types" +) + +func (m *Manager) managedVolumesHeld(id string) ([]types.Volume, bool) { + h, ok := m.managedHold(id) + if !ok { + return nil, false + } + volumes := make([]types.Volume, len(h.Refs)) + for i, ref := range h.Refs { + volumes[i] = ref.Volume + } + return volumes, true +} + +func (m *Manager) validateManagedTarget(id string, target []types.Volume) error { + h, ok := m.managedHold(id) + if !ok { + return nil + } + for _, ref := range h.Refs { + if ref.ID != "" && volumeIndex(target, ref.Volume.Name) < 0 { + return fmt.Errorf("%w: managed volumes cannot be removed live", ErrBadVolume) + } + } + return nil +} diff --git a/sandboxd/pool/managed_recovery.go b/sandboxd/pool/managed_recovery.go new file mode 100644 index 00000000..00f3e6d6 --- /dev/null +++ b/sandboxd/pool/managed_recovery.go @@ -0,0 +1,92 @@ +package pool + +import ( + "context" + "errors" + "maps" + "os" + "strings" + + "github.com/cocoonstack/sandbox/sandboxd/engine" + "github.com/cocoonstack/sandbox/sandboxd/types" +) + +func (m *Manager) recoverManagedOrphans(ctx context.Context, claims map[string]*types.Sandbox, live map[string]types.VMRecord, owned map[string]bool) error { + r := m.managed + if r == nil { + return nil + } + r.mu.Lock() + if len(r.data.Volumes) == 0 && len(r.data.Orphans) == 0 { + r.mu.Unlock() + return nil + } + orphans := maps.Clone(r.data.Orphans) + if orphans == nil { + orphans = map[string]processIdentity{} + } + r.mu.Unlock() + known := maps.Clone(owned) + for _, sb := range claims { + known[sb.VMName] = true + } + for name, rec := range live { + if !strings.HasPrefix(name, vmPrefix) || known[name] { + continue + } + if rec.State == vmStateCreating { + outcome, err := m.eng.ReconcileStaleCreate(context.WithoutCancel(ctx), name) + if err != nil { + return managedStorageError() + } + switch outcome { + case engine.StaleCreateCollected, engine.StaleCreateNotFound: + delete(live, name) + continue + default: + return managedStorageError() + } + } + if _, tracked := orphans[name]; tracked { + continue + } + identity, err := r.process(rec.PID) + if (rec.PID > 0 && errors.Is(err, os.ErrNotExist)) || (rec.PID == 0 && rec.State == "stopped") { + _ = m.eng.Remove(context.WithoutCancel(ctx), name) + if !m.confirmGone(ctx, name) { + return managedStorageError() + } + delete(live, name) + continue + } + if err != nil { + return managedStorageError() + } + orphans[name] = identity + } + if len(orphans) == 0 { + return nil + } + r.mu.Lock() + r.data.Orphans = maps.Clone(orphans) + err := r.commit() + r.mu.Unlock() + if err != nil { + return err + } + for name, p := range orphans { + _ = m.eng.Remove(context.WithoutCancel(ctx), name) + if !m.confirmGone(ctx, name) || !r.processGone(p) { + return managedStorageError() + } + r.mu.Lock() + delete(r.data.Orphans, name) + err := r.commit() + r.mu.Unlock() + if err != nil { + return err + } + delete(live, name) + } + return nil +} diff --git a/sandboxd/pool/managed_recovery_regression_test.go b/sandboxd/pool/managed_recovery_regression_test.go new file mode 100644 index 00000000..b44a2617 --- /dev/null +++ b/sandboxd/pool/managed_recovery_regression_test.go @@ -0,0 +1,225 @@ +package pool + +import ( + "errors" + "os" + "path/filepath" + "testing" + + "github.com/cocoonstack/sandbox/sandboxd/types" +) + +func TestManagedQuarantineRemovalSurvivesFailedRetirementRestart(t *testing.T) { + for _, syncClaims := range []bool{false, true} { + for _, published := range []bool{false, true} { + name := "async" + if syncClaims { + name = "sync" + } + if published { + name += "/published" + } else { + name += "/unpublished" + } + t.Run(name, func(t *testing.T) { + cfg := managedConfig(t) + cfg.SyncClaims = syncClaims + e := newManagedEngine() + m := openManagedTest(t, cfg, e) + createManagedTest(t, m, "data") + sb, err := m.ClaimProvision(t.Context(), testKey, managedOptions("data")) + if err != nil { + t.Fatal(err) + } + sb.Key.Net = types.NetEgress + if err := m.store.commit(m.store.set(sb)); err != nil { + t.Fatal(err) + } + e.falseRemove.Store(true) + m.managed.commitFile = func(path string, raw []byte) error { + if published { + if err := writeVolumeSnapshot(path, raw); err != nil { + return err + } + } + return errors.New("retirement persistence fault") + } + if m.quarantineClaim(t.Context(), sb) { + t.Fatal("ineffective removal reported success") + } + waitFor(t, m.store.synced) + if err := m.Close(); err != nil { + t.Fatal(err) + } + m = openManagedTest(t, cfg, e) + if err := m.Reconcile(t.Context()); err != nil { + t.Fatal(err) + } + _, err = m.ClaimProvision(t.Context(), testKey, managedOptions("data")) + requireVolumeStatus(t, err, 409) + e.falseRemove.Store(false) + m.retryRemovals(t.Context()).Wait() + if _, present, err := e.Inspect(t.Context(), sb.VMName); err != nil || present { + t.Fatalf("quarantined VM survived removal retry after restart: present=%v err=%v", present, err) + } + replacement, err := m.ClaimProvision(t.Context(), testKey, managedOptions("data")) + if err != nil { + t.Fatalf("confirmed removal did not free managed disk: %v", err) + } + if err := m.Release(t.Context(), replacement.ID, Cred{Operator: true}); err != nil { + t.Fatal(err) + } + }) + } + } +} + +func TestManagedOrdinaryFailedClaimRetainsOwnershipOnRestart(t *testing.T) { + cfg := managedConfig(t) + e := newManagedEngine() + m := openManagedTest(t, cfg, e) + createManagedTest(t, m, "data") + sb, err := m.ClaimProvision(t.Context(), testKey, managedOptions("data")) + if err != nil { + t.Fatal(err) + } + sb.Failed = "ordinary failure requiring explicit release" + if err := m.store.commit(m.store.set(sb)); err != nil { + t.Fatal(err) + } + if err := m.Close(); err != nil { + t.Fatal(err) + } + m = openManagedTest(t, cfg, e) + if err := m.Reconcile(t.Context()); err != nil { + t.Fatal(err) + } + m.retryRemovals(t.Context()).Wait() + if _, present, err := e.Inspect(t.Context(), sb.VMName); err != nil || !present { + t.Fatalf("ordinary failed claim removed: %v %v", present, err) + } + _, err = m.ClaimProvision(t.Context(), testKey, managedOptions("data")) + requireVolumeStatus(t, err, 409) + if err := m.Release(t.Context(), sb.ID, Cred{Operator: true}); err != nil { + t.Fatal(err) + } +} + +func TestManagedStartupCleansDeadOrphansAndRefusesUnknownProcess(t *testing.T) { + for _, tc := range []struct { + name string + pid int + stopped bool + processError error + allow bool + ineffectiveRemove bool + }{ + {name: "stopped-absent-pid", pid: 12345, stopped: true, processError: os.ErrNotExist, allow: true}, + {name: "running-absent-pid", pid: 12345, processError: os.ErrNotExist, allow: true}, + {name: "stopped-zero-pid", stopped: true, allow: true}, + {name: "running-zero-pid"}, + {name: "unreadable-pid", pid: 12345, stopped: true, processError: os.ErrPermission}, + {name: "ineffective-remove", pid: 12345, stopped: true, processError: os.ErrNotExist, ineffectiveRemove: true}, + } { + t.Run(tc.name, func(t *testing.T) { + cfg := managedConfig(t) + e := newManagedEngine() + m := openManagedTest(t, cfg, e) + createManagedTest(t, m, "data") + const name = "sbx-stale-orphan" + e.mu.Lock() + e.vms[name] = "/fake/vsock" + e.stopped[name] = tc.stopped + e.pids[name] = tc.pid + e.mu.Unlock() + if tc.processError != nil { + m.managed.process = func(int) (processIdentity, error) { return processIdentity{}, tc.processError } + } else { + m.managed.process = readProcessIdentity + } + e.falseRemove.Store(tc.ineffectiveRemove) + err := m.Reconcile(t.Context()) + if tc.allow { + if err != nil { + t.Fatalf("dead orphan blocked startup: %v", err) + } + if _, present, err := e.Inspect(t.Context(), name); err != nil || present { + t.Fatalf("dead orphan survived cleanup: %v %v", present, err) + } + } else { + requireVolumeStatus(t, err, 503) + } + if tc.processError == os.ErrPermission || (!tc.stopped && tc.pid == 0) { + e.mu.Lock() + defer e.mu.Unlock() + if len(e.removes) != 0 { + t.Fatalf("unproven process removed: %v", e.removes) + } + } + }) + } +} + +func TestManagedStoreRejectsSweepOwnedPathsBeforeCreatingFiles(t *testing.T) { + for _, rel := range []string{"readers", "readers/managed", "fork-persistent", "fork-persistent/managed", "goldens", "goldens/managed"} { + t.Run(rel, func(t *testing.T) { + cfg := managedConfig(t) + cfg.ManagedVolumeDir = filepath.Join(cfg.DataDir, rel) + m, err := NewManager(t.Context(), cfg, newManagedEngine(), testSecrets(t), testBackends(t, cfg)) + if err == nil { + m.Close() + t.Fatal("accepted managed store inside startup cleanup namespace") + } + if _, err := os.Stat(cfg.ManagedVolumeDir); !errors.Is(err, os.ErrNotExist) { + t.Fatalf("unsafe store modified before rejection: %v", err) + } + }) + } +} + +func TestManagedStorePreservesBackingWhenCleanupAliasesDisks(t *testing.T) { + for _, namespace := range []string{"readers", "goldens"} { + t.Run(namespace, func(t *testing.T) { + cfg := managedConfig(t) + e := newManagedEngine() + m := openManagedTest(t, cfg, e) + if err := m.Reconcile(t.Context()); err != nil { + t.Fatal(err) + } + v := createManagedTest(t, m, "data") + backing := m.managed.path(volumeID(v.ID)) + f, err := os.OpenFile(backing, os.O_WRONLY, 0) + if err != nil { + t.Fatal(err) + } + _, writeErr := f.WriteAt([]byte("preserved"), 4096) + if err := errors.Join(writeErr, f.Close(), m.Close()); err != nil { + t.Fatal(err) + } + cleanup := filepath.Join(cfg.DataDir, namespace) + if err := os.Remove(cleanup); err != nil && !errors.Is(err, os.ErrNotExist) { + t.Fatal(err) + } + if err := os.Symlink(filepath.Join(cfg.ManagedVolumeDir, "disks"), cleanup); err != nil { + t.Fatal(err) + } + next, err := NewManager(t.Context(), cfg, e, testSecrets(t), testBackends(t, cfg)) + if err == nil { + t.Error("constructor accepted cleanup alias into managed backing directory") + if err := next.Reconcile(t.Context()); err != nil { + t.Errorf("reconcile: %v", err) + } + next.Close() + } + f, err = os.Open(backing) + if err != nil { + t.Fatalf("managed backing lost: %v", err) + } + defer f.Close() + marker := make([]byte, len("preserved")) + if _, err := f.ReadAt(marker, 4096); err != nil || string(marker) != "preserved" { + t.Fatalf("managed bytes changed: %q %v", marker, err) + } + }) + } +} diff --git a/sandboxd/pool/managed_startup_boundary_test.go b/sandboxd/pool/managed_startup_boundary_test.go new file mode 100644 index 00000000..c0491740 --- /dev/null +++ b/sandboxd/pool/managed_startup_boundary_test.go @@ -0,0 +1,102 @@ +package pool + +import ( + "errors" + "os" + "path/filepath" + "testing" + + "github.com/cocoonstack/sandbox/sandboxd/engine" +) + +func TestManagedStartupCollectsUnclaimedCreatingVMBeforeProcessProof(t *testing.T) { + for _, tc := range []struct { + name string + outcome engine.StaleCreateOutcome + collector error + available bool + }{ + {name: "collected", outcome: engine.StaleCreateCollected, available: true}, + {name: "not-found", outcome: engine.StaleCreateNotFound, available: true}, + {name: "busy", outcome: engine.StaleCreateBusy}, + {name: "not-creating", outcome: engine.StaleCreateNotCreating}, + {name: "collector-error", collector: errors.New("collector unavailable")}, + } { + t.Run(tc.name, func(t *testing.T) { + e := newManagedEngine() + m := openManagedTest(t, managedConfig(t), e) + createManagedTest(t, m, "data") + name := "sbx-unclaimed-create" + e.mu.Lock() + e.vms[name] = "/fake/vsock" + e.creating[name] = true + e.staleOutcome = tc.outcome + e.staleErr = tc.collector + e.mu.Unlock() + + err := m.Reconcile(t.Context()) + if tc.available { + if err != nil { + t.Fatalf("proven stale create blocked startup: %v", err) + } + } else { + requireVolumeStatus(t, err, 503) + } + e.mu.Lock() + defer e.mu.Unlock() + if len(e.staleReconciles) != 1 || e.staleReconciles[0] != name { + t.Fatalf("collector calls=%v, want %s", e.staleReconciles, name) + } + if len(e.removes) != 0 { + t.Fatalf("unsafe Remove fallback called: %v", e.removes) + } + _, present := e.vms[name] + if present == tc.available { + t.Fatalf("creating VM present=%v after outcome %q", present, tc.outcome) + } + }) + } +} + +func TestManagedDirectoryRequiresTrustedAncestorOwner(t *testing.T) { + if os.Geteuid() != 0 { + t.Skip("fixture needs root to create a root-owned child below a different owner") + } + for _, tc := range []struct { + name string + owner int + sticky bool + allow bool + }{ + {name: "untrusted-0755", owner: 65534}, + {name: "untrusted-sticky", owner: 65534, sticky: true}, + {name: "root-0755", owner: 0, allow: true}, + {name: "root-sticky", owner: 0, sticky: true, allow: true}, + } { + t.Run(tc.name, func(t *testing.T) { + parent := filepath.Join(t.TempDir(), "parent") + if err := os.Mkdir(parent, 0700); err != nil { + t.Fatal(err) + } + mode := os.FileMode(0755) + if tc.sticky { + mode = 0777 | os.ModeSticky + } + if err := os.Chmod(parent, mode); err != nil { + t.Fatal(err) + } + managed := filepath.Join(parent, "managed") + if err := os.Mkdir(managed, 0700); err != nil { + t.Fatal(err) + } + if err := os.Chown(parent, tc.owner, tc.owner); err != nil { + t.Fatal(err) + } + t.Cleanup(func() { _ = os.Chown(parent, 0, 0) }) + err := privateVolumeDir(managed) + if (err == nil) != tc.allow { + t.Fatalf("trusted=%v, validation error=%v", tc.allow, err) + } + }) + } +} diff --git a/sandboxd/pool/managed_volume.go b/sandboxd/pool/managed_volume.go new file mode 100644 index 00000000..e20ef258 --- /dev/null +++ b/sandboxd/pool/managed_volume.go @@ -0,0 +1,405 @@ +package pool + +import ( + "context" + "errors" + "os" + "path/filepath" + "slices" + "strings" + "syscall" + "time" + + "github.com/cocoonstack/sandbox/sandboxd/engine" + "github.com/cocoonstack/sandbox/sandboxd/types" +) + +// VolumeError carries only caller-safe details. Storage paths and process IDs +// remain in the registry and never become API diagnostics. +type VolumeError struct { + Status int `json:"-"` + Message string `json:"error"` + Code string `json:"code"` + Volume string `json:"volume,omitempty"` + Holders []types.VolumeHolder `json:"holders,omitempty"` +} + +func (e *VolumeError) Error() string { return e.Message } +func (e *VolumeError) Is(target error) bool { + return target == ErrVolumeBusy && e.Code == "volume_busy" +} +func volumeError(status int, code, message string) *VolumeError { + return &VolumeError{Status: status, Code: code, Message: message} +} +func managedStorageError() *VolumeError { + return volumeError(503, "volume_storage_unavailable", "volume storage requires recovery; retry after recovery") +} +func managedNotFound() *VolumeError { return volumeError(404, "volume_not_found", "unknown volume") } + +func (m *Manager) CreateVolume(ctx context.Context, name string, req types.VolumeCreate, tenant string) (types.ManagedVolumeInfo, bool, error) { + if !types.ValidVolumeName(name) || req.SizeGiB < 1 || req.SizeGiB > 1024 { + return types.ManagedVolumeInfo{}, false, volumeError(400, "invalid_volume", "invalid volume name or size_gib (1 through 1024)") + } + if tenant != "" && req.Tenant != "" && tenant != req.Tenant { + return types.ManagedVolumeInfo{}, false, volumeError(403, "volume_owner_forbidden", "tenant cannot select another volume owner") + } + owner := req.Tenant + if tenant != "" { + owner = tenant + } + if !types.NameRe.MatchString(owner) { + return types.ManagedVolumeInfo{}, false, volumeError(400, "invalid_volume_owner", "an existing tenant owner is required") + } + if _, ok, err := m.tenants.Lookup(ctx, owner); err != nil { + return types.ManagedVolumeInfo{}, false, managedStorageError() + } else if !ok { + return types.ManagedVolumeInfo{}, false, volumeError(400, "invalid_volume_owner", "an existing tenant owner is required") + } + r := m.managed + if r == nil { + return types.ManagedVolumeInfo{}, false, managedStorageError() + } + r.mu.Lock() + defer r.mu.Unlock() + if r.fault != nil { + return types.ManagedVolumeInfo{}, false, managedStorageError() + } + if _, ok := m.volumes[name]; ok { + return types.ManagedVolumeInfo{}, false, volumeError(409, "volume_name_conflict", "volume name is unavailable") + } + v, exists := r.data.Volumes[name] + if exists && tenant != "" && v.Tenant != tenant { + return types.ManagedVolumeInfo{}, false, managedNotFound() + } + size := req.SizeGiB << 30 + if exists && (v.Size != size || v.Tenant != owner) { + return types.ManagedVolumeInfo{}, false, volumeError(409, "volume_create_conflict", "volume already exists with different size or owner") + } + if !exists { + v = managedVolume{ID: volumeID("vol_" + randHex(16)), Name: name, Tenant: owner, Node: r.node, Size: size, State: volumeCreating, CreatedAt: time.Now().UTC()} + r.data.Volumes[name] = v + if err := r.commit(); err != nil { + return types.ManagedVolumeInfo{}, false, err + } + } + switch v.State { + case volumeCreating: + if err := r.completeCreate(v); err != nil { + return types.ManagedVolumeInfo{}, false, err + } + v = r.data.Volumes[name] + case volumeReady: + if err := r.verify(v); err != nil { + return types.ManagedVolumeInfo{}, false, r.markRecovery(v) + } + case volumeDeleting: + return types.ManagedVolumeInfo{}, false, volumeError(409, "volume_deleting", "volume deletion is pending") + default: + return types.ManagedVolumeInfo{}, false, managedStorageError() + } + return r.info(v), !exists, nil +} + +func (r *volumeRegistry) completeCreate(v managedVolume) error { + if err := r.verifyDirectories(); err != nil { + return managedStorageError() + } + path := r.path(v.ID) + f, err := openPrivateFile(path, syscall.O_CREAT|syscall.O_EXCL|syscall.O_RDWR) + if err != nil { + if errors.Is(err, os.ErrExist) { + return volumeError(503, "volume_create_incomplete", "incomplete volume allocation requires operator quarantine before retry") + } + return managedStorageError() + } + if err = f.Truncate(v.Size); err == nil { + err = f.Sync() + } + st, statErr := f.Stat() + closeErr := f.Close() + if err == nil { + err = errors.Join(statErr, closeErr, syncDir(r.dir+"/disks")) + } + if err != nil { + return managedStorageError() + } + raw := st.Sys().(*syscall.Stat_t) + v.Backing = backingIdentity{Device: uint64(raw.Dev), Inode: raw.Ino, Size: st.Size()} + v.State = volumeReady + r.data.Volumes[v.Name] = v + return r.commit() +} + +func (r *volumeRegistry) markRecovery(v managedVolume) error { + v.State = volumeRecovery + r.data.Volumes[v.Name] = v + if err := r.commit(); err != nil { + return err + } + return managedStorageError() +} + +func (m *Manager) GetVolume(name, tenant string) (types.ManagedVolumeInfo, error) { + r := m.managed + if r == nil { + return types.ManagedVolumeInfo{}, managedNotFound() + } + r.mu.Lock() + defer r.mu.Unlock() + if r.fault != nil { + return types.ManagedVolumeInfo{}, managedStorageError() + } + v, ok := r.data.Volumes[name] + if !ok || (tenant != "" && v.Tenant != tenant) { + return types.ManagedVolumeInfo{}, managedNotFound() + } + if v.State == volumeReady && r.verify(v) != nil { + if err := r.markRecovery(v); r.fault != nil { + return types.ManagedVolumeInfo{}, err + } + v = r.data.Volumes[name] + } + return r.info(v), nil +} + +func (m *Manager) DeleteVolume(ctx context.Context, name, expected, tenant string) error { + if err := ctx.Err(); err != nil { + return err + } + r := m.managed + if r == nil { + return managedNotFound() + } + r.mu.Lock() + defer r.mu.Unlock() + if r.fault != nil { + return managedStorageError() + } + if expected == "" { + return volumeError(428, "volume_precondition_required", "If-Match must identify the volume") + } + id := volumeID(expected) + if t, ok := r.data.Deleted[id]; ok && t.Name == name && (tenant == "" || tenant == t.Tenant) { + return nil + } + v, ok := r.data.Volumes[name] + if !ok || (tenant != "" && v.Tenant != tenant) { + return managedNotFound() + } + if v.ID != id { + return volumeError(412, "volume_precondition_failed", "volume identity does not match") + } + if holders := r.holders(v.ID); len(holders) > 0 { + return &VolumeError{Status: 409, Code: "volume_busy", Message: "volume is held by a sandbox", Volume: name, Holders: holders} + } + if err := r.verifyDirectories(); err != nil { + return managedStorageError() + } + path := r.path(id) + actual, err := inspectBacking(path) + missing := errors.Is(err, os.ErrNotExist) + if missing && v.State != volumeCreating && v.State != volumeDeleting { + return r.markRecovery(v) + } + if !missing && (err != nil || v.Backing.Inode == 0 || actual != v.Backing) { + return volumeError(503, "volume_backing_mismatch", "volume backing requires operator recovery") + } + if v.State != volumeDeleting { + v.State = volumeDeleting + r.data.Volumes[name] = v + if err := r.commit(); err != nil { + return err + } + } + if !missing { + if err := os.Remove(path); err != nil { + return managedStorageError() + } + } + if err := syncDir(r.dir + "/disks"); err != nil { + return managedStorageError() + } + if err := clearVolumeDirty(path); err != nil { + return managedStorageError() + } + r.data.Deleted[id] = volumeTombstone{ID: id, Name: name, Tenant: v.Tenant} + delete(r.data.Volumes, name) + return r.commit() +} + +func (r *volumeRegistry) holders(id volumeID) []types.VolumeHolder { + holders := []types.VolumeHolder{} + for _, h := range r.data.Holds { + for _, ref := range h.Refs { + if ref.ID == id { + mode := "ro" + if ref.Volume.RW() { + mode = "rw" + } + holders = append(holders, types.VolumeHolder{SandboxID: h.SandboxID, Mode: mode, State: string(h.State)}) + } + } + } + slices.SortFunc(holders, func(a, b types.VolumeHolder) int { return strings.Compare(a.SandboxID, b.SandboxID) }) + return holders +} +func (r *volumeRegistry) info(v managedVolume) types.ManagedVolumeInfo { + return types.ManagedVolumeInfo{VolumeInfo: types.VolumeInfo{Name: v.Name, DefaultMount: types.DefaultVolumeMount(v.Name), SizeBytes: v.Size, Available: v.State == volumeReady, Nodes: 1, Writable: true, Kind: "managed", ID: string(v.ID), Tenant: v.Tenant, Node: v.Node, State: string(v.State), CreatedAt: v.CreatedAt}, Holders: r.holders(v.ID)} +} + +// ManagedVolumeStatus lets listing fail clearly rather than presenting a corrupt +// registry as an empty managed catalog. A disabled store has no catalog cost. +func (m *Manager) ManagedVolumeStatus() error { + if m.managed == nil { + return nil + } + m.managed.mu.Lock() + defer m.managed.mu.Unlock() + if m.managed.fault != nil { + return managedStorageError() + } + return nil +} + +func (m *Manager) managedInfos(tenant string) []types.VolumeInfo { + r := m.managed + if r == nil { + return nil + } + r.mu.Lock() + defer r.mu.Unlock() + if r.fault != nil { + return nil + } + infos := []types.VolumeInfo{} + for _, v := range r.data.Volumes { + if tenant == "" || tenant == v.Tenant { + info := r.info(v) + if v.State == volumeReady && r.verify(v) != nil { + info.State = string(volumeRecovery) + info.Available = false + } + infos = append(infos, info.VolumeInfo) + } + } + return infos +} + +func (m *Manager) lookupManaged(name string) (catalogVolume, bool, error) { + r := m.managed + if r == nil { + return catalogVolume{}, false, nil + } + r.mu.Lock() + defer r.mu.Unlock() + if r.fault != nil { + return catalogVolume{}, false, managedStorageError() + } + v, ok := r.data.Volumes[name] + if !ok { + return catalogVolume{}, false, nil + } + entry := catalogVolume{disk: engine.VolumeSpec{Name: name, Path: r.path(v.ID), DirectIO: types.DirectIOOff}, tenants: []string{v.Tenant}, writable: true, managedID: v.ID} + if v.State != volumeReady { + return entry, true, managedStorageError() + } + if err := r.verify(v); err != nil { + return entry, true, r.markRecovery(v) + } + return entry, true, nil +} + +func (m *Manager) managedHold(id string) (volumeHold, bool) { + r := m.managed + if r == nil { + return volumeHold{}, false + } + r.mu.Lock() + defer r.mu.Unlock() + h, ok := r.data.Holds[id] + h.Refs = slices.Clone(h.Refs) + return h, ok +} + +func (m *Manager) managedHoldByVM(name string) (volumeHold, bool) { + r := m.managed + if r == nil { + return volumeHold{}, false + } + r.mu.Lock() + defer r.mu.Unlock() + for _, h := range r.data.Holds { + if h.VMName == name { + h.Refs = slices.Clone(h.Refs) + return h, true + } + } + return volumeHold{}, false +} + +func (m *Manager) retireManaged(id string) error { + r := m.managed + if r == nil { + return nil + } + r.mu.Lock() + defer r.mu.Unlock() + h, ok := r.data.Holds[id] + if !ok { + return nil + } + if r.fault != nil { + return managedStorageError() + } + if h.State == holdRetiring { + return nil + } + h.State = holdRetiring + r.data.Holds[id] = h + return r.commit() +} + +func (m *Manager) managedRefPath(id, name string) (string, bool) { + h, ok := m.managedHold(id) + if !ok { + return "", false + } + for _, ref := range h.Refs { + if ref.Volume.Name == name { + return ref.Path, true + } + } + return "", false +} + +func (m *Manager) ManagedVolumesEnabled() bool { return m.managed != nil } + +func (m *Manager) managedCatalogAlias(path string) bool { + r := m.managed + if r == nil { + return false + } + clean := filepath.Clean(path) + if canonical, err := filepath.EvalSymlinks(path); err == nil { + clean = canonical + } + if clean == r.dir || strings.HasPrefix(clean, r.dir+string(filepath.Separator)) { + return true + } + info, err := os.Stat(path) + if err != nil { + return false + } + identity := info.Sys().(*syscall.Stat_t) + r.mu.Lock() + defer r.mu.Unlock() + for _, v := range r.data.Volumes { + if v.Backing.Inode != 0 && identity.Ino == v.Backing.Inode && uint64(identity.Dev) == v.Backing.Device { + return true + } + if backing, err := os.Lstat(r.path(v.ID)); err == nil && os.SameFile(info, backing) { + return true + } + } + return false +} diff --git a/sandboxd/pool/managed_volume_test.go b/sandboxd/pool/managed_volume_test.go new file mode 100644 index 00000000..715d706a --- /dev/null +++ b/sandboxd/pool/managed_volume_test.go @@ -0,0 +1,584 @@ +package pool + +import ( + "context" + "encoding/json/v2" + "errors" + "fmt" + "os" + "path/filepath" + "strings" + "sync" + "sync/atomic" + "syscall" + "testing" + "time" + + "github.com/cocoonstack/sandbox/sandboxd/config" + "github.com/cocoonstack/sandbox/sandboxd/engine" + "github.com/cocoonstack/sandbox/sandboxd/tenants" + "github.com/cocoonstack/sandbox/sandboxd/types" +) + +type managedTestEngine struct { + liveEntered, liveResume chan struct{} + *fakeEngine + next atomic.Int64 + falseRemove atomic.Bool + missingInventory atomic.Bool + processLives atomic.Bool + processUnknown atomic.Bool + attachEntered chan struct{} + attachResume chan struct{} +} + +func newManagedEngine() *managedTestEngine { return &managedTestEngine{fakeEngine: newFakeEngine()} } +func (e *managedTestEngine) RunCold(ctx context.Context, name string, key types.PoolKey, storage string) (types.VMRecord, error) { + e.mu.Lock() + e.pids[name] = 1000 + int(e.next.Add(1)) + e.mu.Unlock() + return e.fakeEngine.RunCold(ctx, name, key, storage) +} +func (e *managedTestEngine) Remove(ctx context.Context, name string) error { + if e.falseRemove.Load() { + return nil + } + return e.fakeEngine.Remove(ctx, name) +} +func (e *managedTestEngine) Inspect(ctx context.Context, name string) (types.VMRecord, bool, error) { + if e.missingInventory.Load() { + return types.VMRecord{}, false, nil + } + return e.fakeEngine.Inspect(ctx, name) +} +func (e *managedTestEngine) DiskAttach(ctx context.Context, name string, spec engine.VolumeSpec) error { + if e.attachEntered != nil { + e.attachEntered <- struct{}{} + <-e.attachResume + } + return e.fakeEngine.DiskAttach(ctx, name, spec) +} +func (e *managedTestEngine) identity(pid int) (processIdentity, error) { + if e.processUnknown.Load() { + return processIdentity{}, os.ErrPermission + } + e.mu.Lock() + defer e.mu.Unlock() + for name, p := range e.pids { + if p == pid { + if _, ok := e.vms[name]; ok || e.processLives.Load() { + return processIdentity{PID: pid, Start: uint64(pid), BootID: "test-boot"}, nil + } + } + } + return processIdentity{}, os.ErrNotExist +} +func managedConfig(t *testing.T) *config.Config { + t.Helper() + dir := t.TempDir() + return &config.Config{DataDir: filepath.Join(dir, "data"), ManagedVolumeDir: filepath.Join(dir, "managed"), APIToken: "root", Pools: []config.PoolSpec{{PoolKey: testKey, Warm: 1}}} +} +func openManagedTest(t *testing.T, cfg *config.Config, e *managedTestEngine) *Manager { + t.Helper() + m, err := NewManager(t.Context(), cfg, e, testSecrets(t), testBackends(t, cfg)) + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { m.Close() }) + m.managed.process = e.identity + if err := m.PutTenant(t.Context(), tenants.TenantSpec{Name: "t1", Token: "t1-token"}); err != nil { + t.Fatal(err) + } + if err := m.PutTenant(t.Context(), tenants.TenantSpec{Name: "t2", Token: "t2-token"}); err != nil { + t.Fatal(err) + } + return m +} +func createManagedTest(t *testing.T, m *Manager, name string) types.ManagedVolumeInfo { + t.Helper() + v, created, err := m.CreateVolume(t.Context(), name, types.VolumeCreate{SizeGiB: 1}, "t1") + if err != nil || !created { + t.Fatalf("create: created=%v err=%v", created, err) + } + return v +} +func managedOptions(names ...string) ClaimOptions { + o := ClaimOptions{Tenant: "t1"} + for _, name := range names { + o.Volumes = append(o.Volumes, types.Volume{Name: name, Mode: types.VolumeModeRW, AttachOnly: true}) + } + return o +} +func requireVolumeStatus(t *testing.T, err error, status int) *VolumeError { + t.Helper() + var v *VolumeError + if !errors.As(err, &v) || v.Status != status { + t.Fatalf("error=%v, want volume status %d", err, status) + } + return v +} + +func TestManagedCreateReplaySparseOwnershipAndDeleteIdentity(t *testing.T) { + cfg := managedConfig(t) + e := newManagedEngine() + m := openManagedTest(t, cfg, e) + v := createManagedTest(t, m, "data") + path := m.managed.path(volumeID(v.ID)) + st, err := os.Stat(path) + if err != nil { + t.Fatal(err) + } + if st.Size() != 1<<30 || st.Sys().(*syscall.Stat_t).Blocks*512 >= st.Size() { + t.Fatal("backing is not a sparse GiB raw image") + } + f, err := os.OpenFile(path, os.O_WRONLY, 0600) + if err != nil { + t.Fatal(err) + } + _, err = f.WriteAt([]byte("preserved"), 4096) + if err != nil { + t.Fatal(err) + } + f.Close() + if v.Kind != "managed" || v.State != "ready" || v.Tenant != "t1" || v.Node == "" || v.CreatedAt.IsZero() { + t.Fatalf("projection=%+v", v) + } + _, _, err = m.CreateVolume(t.Context(), "data", types.VolumeCreate{SizeGiB: 2}, "t1") + requireVolumeStatus(t, err, 409) + _, _, err = m.CreateVolume(t.Context(), "data", types.VolumeCreate{SizeGiB: 1}, "t2") + requireVolumeStatus(t, err, 404) + _, err = m.GetVolume("data", "t2") + requireVolumeStatus(t, err, 404) + m.Close() + m = openManagedTest(t, cfg, e) + repeat, created, err := m.CreateVolume(t.Context(), "data", types.VolumeCreate{SizeGiB: 1, Tenant: "t1"}, "t1") + if err != nil || created || repeat.ID != v.ID { + t.Fatalf("replay=%+v created=%v err=%v", repeat, created, err) + } + f, err = os.Open(path) + if err != nil { + t.Fatal(err) + } + buf := make([]byte, 9) + _, err = f.ReadAt(buf, 4096) + f.Close() + if err != nil || string(buf) != "preserved" { + t.Fatalf("replay changed bytes: %q %v", buf, err) + } + requireVolumeStatus(t, m.DeleteVolume(t.Context(), "data", "", "t1"), 428) + requireVolumeStatus(t, m.DeleteVolume(t.Context(), "data", "wrong", "t1"), 412) + if err := m.DeleteVolume(t.Context(), "data", v.ID, "t1"); err != nil { + t.Fatal(err) + } + newer := createManagedTest(t, m, "data") + if newer.ID == v.ID { + t.Fatal("identity reused") + } + if err := m.DeleteVolume(t.Context(), "data", v.ID, "t1"); err != nil { + t.Fatal(err) + } + requireVolumeStatus(t, m.DeleteVolume(t.Context(), "data", v.ID, "t2"), 404) + if _, err := os.Stat(m.managed.path(volumeID(newer.ID))); err != nil { + t.Fatal("stale delete removed replacement", err) + } + m.Close() + m = openManagedTest(t, cfg, e) + if got, err := m.GetVolume("data", "t1"); err != nil || got.ID != newer.ID { + t.Fatalf("restart=%+v %v", got, err) + } +} + +func TestManagedConcurrentCreateAndWriterAdmission(t *testing.T) { + m := openManagedTest(t, managedConfig(t), newManagedEngine()) + var created atomic.Int32 + var wg sync.WaitGroup + ids := make(chan string, 12) + for range 12 { + wg.Go(func() { + v, new, err := m.CreateVolume(t.Context(), "race", types.VolumeCreate{SizeGiB: 1}, "t1") + if err != nil { + t.Error(err) + return + } + if new { + created.Add(1) + } + ids <- v.ID + }) + } + wg.Wait() + close(ids) + first := "" + for id := range ids { + if first == "" { + first = id + } + if id != first { + t.Error("concurrent identity changed") + } + } + if created.Load() != 1 { + t.Fatalf("created=%d", created.Load()) + } + successes := make(chan *types.Sandbox, 8) + for range 8 { + wg.Go(func() { + sb, err := m.ClaimProvision(t.Context(), testKey, managedOptions("race")) + if err == nil { + successes <- sb + return + } + var ve *VolumeError + if !errors.As(err, &ve) || ve.Status != 409 || len(ve.Holders) != 1 || ve.Holders[0].SandboxID == "" { + t.Errorf("conflict=%+v", err) + } + }) + } + wg.Wait() + close(successes) + var winner *types.Sandbox + count := 0 + for sb := range successes { + winner = sb + count++ + } + if count != 1 { + t.Fatalf("writers=%d", count) + } + if err := m.Release(t.Context(), winner.ID, Cred{Operator: true}); err != nil { + t.Fatal(err) + } + if err := m.DeleteVolume(t.Context(), "race", first, "t1"); err != nil { + t.Fatal(err) + } +} + +func TestManagedRetirementSurvivesRestartAndStaleClaim(t *testing.T) { + for _, syncClaims := range []bool{false, true} { + t.Run(fmt.Sprint(syncClaims), func(t *testing.T) { + cfg := managedConfig(t) + cfg.SyncClaims = syncClaims + e := newManagedEngine() + m := openManagedTest(t, cfg, e) + v := createManagedTest(t, m, "data") + sb, err := m.ClaimProvision(t.Context(), testKey, managedOptions("data")) + if err != nil { + t.Fatal(err) + } + e.falseRemove.Store(true) + if err := m.retireManaged(sb.ID); err != nil { + t.Fatal(err) + } + m.Close() + m = openManagedTest(t, cfg, e) + if err := m.Reconcile(t.Context()); err != nil { + t.Fatal(err) + } + if _, ok := m.Sandbox(sb.ID); ok { + t.Fatal("retiring stale claim was adopted") + } + _, err = m.ClaimProvision(t.Context(), testKey, managedOptions("data")) + busy := requireVolumeStatus(t, err, 409) + if busy.Holders[0].State != "retiring" || busy.Holders[0].SandboxID != sb.ID { + t.Fatalf("holder=%+v", busy) + } + requireVolumeStatus(t, m.DeleteVolume(t.Context(), "data", v.ID, "t1"), 409) + e.falseRemove.Store(false) + m.retryRemovals(t.Context()).Wait() + if err := m.DeleteVolume(t.Context(), "data", v.ID, "t1"); err != nil { + t.Fatal(err) + } + if len(m.managed.data.Holds) != 0 { + t.Fatal("retirement leaked a hold") + } + }) + } +} + +func TestManagedActiveClaimAdoptionAndProcessProof(t *testing.T) { + for _, syncClaims := range []bool{false, true} { + t.Run(fmt.Sprint(syncClaims), func(t *testing.T) { + cfg := managedConfig(t) + cfg.SyncClaims = syncClaims + e := newManagedEngine() + m := openManagedTest(t, cfg, e) + v := createManagedTest(t, m, "data") + sb, err := m.ClaimProvision(t.Context(), testKey, managedOptions("data")) + if err != nil { + t.Fatal(err) + } + m.Close() + m = openManagedTest(t, cfg, e) + if err := m.Reconcile(t.Context()); err != nil { + t.Fatal(err) + } + adopted, ok := m.Sandbox(sb.ID) + if !ok || adopted.Tenant != "t1" { + t.Fatal("writer not adopted") + } + e.processLives.Store(true) + if err := m.Release(t.Context(), sb.ID, Cred{Operator: true}); err == nil { + t.Fatal("absence alone released a live process") + } + requireVolumeStatus(t, m.DeleteVolume(t.Context(), "data", v.ID, "t1"), 409) + e.processUnknown.Store(true) + m.retryRemovals(t.Context()).Wait() + requireVolumeStatus(t, m.DeleteVolume(t.Context(), "data", v.ID, "t1"), 409) + e.processUnknown.Store(false) + e.processLives.Store(false) + m.retryRemovals(t.Context()).Wait() + if err := m.DeleteVolume(t.Context(), "data", v.ID, "t1"); err != nil { + t.Fatal(err) + } + }) + } +} + +func TestManagedPreclaimCrashRetainsOwnership(t *testing.T) { + for _, syncClaims := range []bool{false, true} { + for _, boundary := range []string{"reserved", "bound", "attached", "mounted"} { + t.Run(fmt.Sprintf("%s/sync=%v", boundary, syncClaims), func(t *testing.T) { + cfg := managedConfig(t) + cfg.SyncClaims = syncClaims + e := newManagedEngine() + m := openManagedTest(t, cfg, e) + v := createManagedTest(t, m, "data") + options := managedOptions("data") + if boundary == "mounted" { + options.Volumes[0].AttachOnly = false + } + specs, _, err := m.admitOptions(t.Context(), testKey, options) + if err != nil { + t.Fatal(err) + } + if boundary != "reserved" { + rec, err := e.RunCold(t.Context(), "sbx-interrupted", testKey, "") + if err != nil { + t.Fatal(err) + } + sb := &types.Sandbox{VMName: rec.Config.Name, VsockSocket: rec.VsockSocket, Key: testKey} + if err := m.bindManaged(t.Context(), sb, specs); err != nil { + t.Fatal(err) + } + if boundary == "attached" || boundary == "mounted" { + if err := m.applyVolume(t.Context(), sb, specs[0], false); err != nil { + t.Fatal(err) + } + } + e.falseRemove.Store(true) + } + m.Close() + m = openManagedTest(t, cfg, e) + if err := m.Reconcile(t.Context()); err != nil { + t.Fatal(err) + } + if boundary != "reserved" { + requireVolumeStatus(t, m.DeleteVolume(t.Context(), "data", v.ID, "t1"), 409) + } + e.falseRemove.Store(false) + m.retryRemovals(t.Context()).Wait() + if err := m.DeleteVolume(t.Context(), "data", v.ID, "t1"); err != nil { + t.Fatal(err) + } + }) + } + } +} + +func TestManagedAmbiguousCommitFailsClosedAndRecovers(t *testing.T) { + cfg := managedConfig(t) + e := newManagedEngine() + m := openManagedTest(t, cfg, e) + v := createManagedTest(t, m, "data") + writes := 0 + m.managed.commitFile = func(path string, raw []byte) error { + writes++ + if err := writeVolumeSnapshot(path, raw); err != nil { + return err + } + return errors.New("directory sync uncertain /private/path") + } + _, err := m.ClaimProvision(t.Context(), testKey, managedOptions("data")) + ve := requireVolumeStatus(t, err, 503) + if strings.Contains(ve.Error(), "private") { + t.Fatal("storage error leaked a path") + } + requireVolumeStatus(t, m.DeleteVolume(t.Context(), "data", v.ID, "t1"), 503) + if writes != 1 || len(e.volumeSpecs) != 0 { + t.Fatal("mutation or attach continued after uncertain commit") + } + m.Close() + m = openManagedTest(t, cfg, e) + if err := m.Reconcile(t.Context()); err != nil { + t.Fatal(err) + } + if err := m.DeleteVolume(t.Context(), "data", v.ID, "t1"); err != nil { + t.Fatal(err) + } +} + +func TestManagedCreateUnrecordedInodeRequiresQuarantine(t *testing.T) { + cfg := managedConfig(t) + e := newManagedEngine() + m := openManagedTest(t, cfg, e) + writes := 0 + m.managed.commitFile = func(path string, raw []byte) error { + writes++ + if writes == 2 { + return errors.New("ready metadata failed") + } + return writeVolumeSnapshot(path, raw) + } + _, _, err := m.CreateVolume(t.Context(), "data", types.VolumeCreate{SizeGiB: 1}, "t1") + requireVolumeStatus(t, err, 503) + m.Close() + m = openManagedTest(t, cfg, e) + v, err := m.GetVolume("data", "t1") + if err != nil || v.State != "creating" { + t.Fatalf("interrupted create=%+v %v", v, err) + } + path := m.managed.path(volumeID(v.ID)) + before, err := os.Stat(path) + if err != nil { + t.Fatal(err) + } + _, _, err = m.CreateVolume(t.Context(), "data", types.VolumeCreate{SizeGiB: 1}, "t1") + ve := requireVolumeStatus(t, err, 503) + if ve.Code != "volume_create_incomplete" { + t.Fatal(ve) + } + requireVolumeStatus(t, m.DeleteVolume(t.Context(), "data", v.ID, "t1"), 503) + after, err := os.Stat(path) + if err != nil || !os.SameFile(before, after) { + t.Fatal("unknown inode modified") + } + m.Close() + if err := os.Rename(path, filepath.Join(filepath.Dir(cfg.ManagedVolumeDir), "quarantine.raw")); err != nil { + t.Fatal(err) + } + m = openManagedTest(t, cfg, e) + repeat, created, err := m.CreateVolume(t.Context(), "data", types.VolumeCreate{SizeGiB: 1}, "t1") + if err != nil || created || repeat.ID != v.ID { + t.Fatalf("quarantine repair=%+v %v %v", repeat, created, err) + } +} + +func TestManagedCorruptAndUnsafeStorage(t *testing.T) { + for _, fault := range []string{"truncated", "version", "missing-registry", "replaced", "symlink", "missing", "permissions"} { + t.Run(fault, func(t *testing.T) { + cfg := managedConfig(t) + e := newManagedEngine() + m := openManagedTest(t, cfg, e) + v := createManagedTest(t, m, "data") + path := m.managed.path(volumeID(v.ID)) + m.Close() + switch fault { + case "truncated": + if err := os.WriteFile(filepath.Join(cfg.ManagedVolumeDir, "registry.json"), []byte("{"), 0600); err != nil { + t.Fatal(err) + } + case "version": + raw, err := os.ReadFile(filepath.Join(cfg.ManagedVolumeDir, "registry.json")) + if err != nil { + t.Fatal(err) + } + var data volumeRegistryData + if err := json.Unmarshal(raw, &data); err != nil { + t.Fatal(err) + } + data.Version = 99 + raw, _ = json.Marshal(data) + if err := os.WriteFile(filepath.Join(cfg.ManagedVolumeDir, "registry.json"), raw, 0600); err != nil { + t.Fatal(err) + } + case "missing-registry": + if err := os.Remove(filepath.Join(cfg.ManagedVolumeDir, "registry.json")); err != nil { + t.Fatal(err) + } + case "replaced": + if err := os.Rename(path, path+".original"); err != nil { + t.Fatal(err) + } + f, err := os.OpenFile(path, os.O_CREATE|os.O_EXCL|os.O_RDWR, 0600) + if err != nil { + t.Fatal(err) + } + if err := f.Truncate(1 << 30); err != nil { + t.Fatal(err) + } + f.Close() + case "symlink": + if err := os.Remove(path); err != nil { + t.Fatal(err) + } + if err := os.Symlink("/etc/passwd", path); err != nil { + t.Fatal(err) + } + case "missing": + if err := os.Remove(path); err != nil { + t.Fatal(err) + } + case "permissions": + if err := os.Chmod(path, 0000); err != nil { + t.Fatal(err) + } + } + m = openManagedTest(t, cfg, e) + _, err := m.ClaimProvision(t.Context(), testKey, managedOptions("data")) + requireVolumeStatus(t, err, 503) + if len(e.volumeSpecs) != 0 { + t.Fatal("unsafe backing attached") + } + if fault == "replaced" || fault == "symlink" || fault == "permissions" { + requireVolumeStatus(t, m.DeleteVolume(t.Context(), "data", v.ID, "t1"), 503) + } + }) + } +} + +func TestManagedStoreExclusiveAndDirectoryPermissions(t *testing.T) { + cfg := managedConfig(t) + e := newManagedEngine() + m := openManagedTest(t, cfg, e) + if _, err := NewManager(t.Context(), cfg, e, testSecrets(t), testBackends(t, cfg)); err == nil { + t.Fatal("second daemon accepted locked store") + } + m.Close() + if err := os.Chmod(cfg.ManagedVolumeDir, 0755); err != nil { + t.Fatal(err) + } + if _, err := NewManager(t.Context(), cfg, e, testSecrets(t), testBackends(t, cfg)); err == nil { + t.Fatal("public managed directory accepted") + } +} + +func TestManagedStalledAttachDoesNotBlockOtherClaims(t *testing.T) { + cfg := managedConfig(t) + e := newManagedEngine() + m := openManagedTest(t, cfg, e) + v := createManagedTest(t, m, "data") + unrelated, err := m.ClaimProvision(t.Context(), testKey, ClaimOptions{}) + if err != nil { + t.Fatal(err) + } + e.attachEntered = make(chan struct{}, 1) + e.attachResume = make(chan struct{}) + done := make(chan error, 1) + go func() { _, err := m.ClaimProvision(t.Context(), testKey, managedOptions("data")); done <- err }() + <-e.attachEntered + if _, err := m.Renew(t.Context(), unrelated.ID, Cred{Operator: true}, time.Minute, ""); err != nil { + t.Fatal(err) + } + if err := m.Release(t.Context(), unrelated.ID, Cred{Operator: true}); err != nil { + t.Fatal(err) + } + got, err := m.GetVolume("data", "t1") + if err != nil || len(got.Holders) != 1 || got.Holders[0].State != "attaching" { + t.Fatalf("attaching projection=%+v %v", got, err) + } + requireVolumeStatus(t, m.DeleteVolume(t.Context(), "data", v.ID, "t1"), 409) + close(e.attachResume) + if err := <-done; err != nil { + t.Fatal(err) + } +} diff --git a/sandboxd/pool/pool.go b/sandboxd/pool/pool.go index 9e0124cb..6f0ded95 100644 --- a/sandboxd/pool/pool.go +++ b/sandboxd/pool/pool.go @@ -329,6 +329,7 @@ type Manager struct { volumes map[string]catalogVolume // volumeAdmission counts the live holders of each volume name; guarded by m.mu. volumeAdmission map[string]volumeHolders + managed *volumeRegistry volumeRuns map[*types.Sandbox]*volumeRun volumeErrors map[*types.Sandbox]map[string]string @@ -438,6 +439,9 @@ type Manager struct { // NewManager builds a manager from the node config over backends the caller selected from it. func NewManager(ctx context.Context, cfg *config.Config, eng Engine, secrets *egress.SecretStore, b Backends) (*Manager, error) { + if err := cfg.ValidateManagedVolumeStore(); err != nil { + return nil, err + } refill := cmp.Or(cfg.RefillConcurrency, defaultRefill) m := &Manager{ eng: eng, @@ -550,6 +554,27 @@ func NewManager(ctx context.Context, cfg *config.Config, eng Engine, secrets *eg } m.warnVolumeTenants(ctx, cfg.Volumes) m.warnMissingClasses(ctx) + if cfg.ManagedVolumeDir != "" { + node, hostErr := os.Hostname() + if cfg.Mesh != nil { + node = cfg.Mesh.ID() + } + if node == "" { + return nil, fmt.Errorf("managed volume node identity unavailable: %v", hostErr) + } + var err error + m.managed, err = openVolumeRegistry(cfg.ManagedVolumeDir, node) + if err != nil { + return nil, err + } + for name, entry := range m.volumes { + _, collision := m.managed.data.Volumes[name] + if collision || m.managedCatalogAlias(entry.disk.Path) { + m.managed.close() + return nil, fmt.Errorf("catalog conflicts with existing managed resource") + } + } + } return m, nil } diff --git a/sandboxd/pool/reconcile.go b/sandboxd/pool/reconcile.go index 3bfa3cf1..69d8b615 100644 --- a/sandboxd/pool/reconcile.go +++ b/sandboxd/pool/reconcile.go @@ -36,7 +36,16 @@ func (m *Manager) Reconcile(ctx context.Context) error { snaps, snapsErr := m.eng.SnapshotList(ctx) imageIDs := m.imageIDs(ctx) - owned := map[string]bool{} + if m.managed != nil { + m.store.reset(claims) + } + owned, err := m.recoverManaged(ctx, claims, live) + if err != nil { + return err + } + if err := m.recoverManagedOrphans(ctx, claims, live, owned); err != nil { + return err + } referenced := map[string]bool{} var adopted []*types.Sandbox m.mu.Lock() @@ -66,7 +75,9 @@ func (m *Manager) Reconcile(ctx context.Context) error { continue } m.addClaimLocked(sb) - m.adoptVolumes(heldVolumes(sb)) + if _, managed := m.managedHold(sb.ID); !managed { + m.adoptVolumes(heldVolumes(sb)) + } owned[sb.VMName] = true referenced[sb.HibernateSnap] = true } @@ -257,8 +268,21 @@ func (m *Manager) resyncEgress(ctx context.Context, live map[string]types.VMReco } } +const egressQuarantineFailure = "sandbox quarantined after egress recovery failure" + // quarantineClaim parks a claim whose teardown failed: out of service and queued until a remove succeeds. func (m *Manager) quarantineClaim(ctx context.Context, sb *types.Sandbox) bool { + if m.retireManaged(sb.ID) != nil { + m.mu.Lock() + sb.Failed = egressQuarantineFailure + m.cancelVolumeRun(sb) + js := m.store.set(sb) + m.mu.Unlock() + m.store.recommit(ctx, js) + td := m.quiesceVolumes(ctx, sb) + m.removeOrRetry(ctx, sb.VMName, sb.ID, td) + return false + } td := m.quiesceVolumes(ctx, sb) gone := m.removeOrRetry(ctx, sb.VMName, sb.ID, td) m.forgetClaim(ctx, sb) diff --git a/sandboxd/pool/remove.go b/sandboxd/pool/remove.go index 75efec26..46ec818e 100644 --- a/sandboxd/pool/remove.go +++ b/sandboxd/pool/remove.go @@ -17,7 +17,17 @@ func (m *Manager) removeVM(ctx context.Context, name string) bool { m.out.ClosePrebound(name) // cancellation-immune: on a canceled ctx `cocoon vm rm` no-ops and orphans. ctx = context.WithoutCancel(ctx) + hold, managed := m.managedHoldByVM(name) + if managed { + if err := m.retireManaged(hold.SandboxID); err != nil { + _ = m.eng.Remove(ctx, name) + return false + } + } err := m.eng.Remove(ctx, name) + if managed { + return m.confirmGone(ctx, name) && m.managed.processGone(hold.Process) + } if err == nil { return true } @@ -44,8 +54,17 @@ func (m *Manager) confirmGone(ctx context.Context, name string) bool { // removeOrRetry reports whether the VM is confirmed gone; a survivor is queued for the reap tick. func (m *Manager) removeOrRetry(ctx context.Context, name, sandboxID string, td volumeTeardown) bool { + if h, ok := m.managedHoldByVM(name); ok { + td.managedID = h.SandboxID + } if m.removeVM(ctx, name) { m.finishVolumeTeardown(ctx, td) + if td.managedID != "" { + if _, held := m.managedHold(td.managedID); held { + m.queueRemoval(name, sandboxID, "", td, time.Time{}) + return false + } + } return true } m.queueRemoval(name, sandboxID, "", td, time.Time{}) @@ -102,6 +121,11 @@ func (m *Manager) retryRemoval(ctx context.Context, name string, pending pending return } m.finishRemoval(ctx, pending) + if pending.volumes.managedID != "" { + if _, held := m.managedHold(pending.volumes.managedID); held { + m.queueRemoval(name, pending.sandboxID, pending.tap, pending.volumes, time.Time{}) + } + } } func (m *Manager) finishRemoval(ctx context.Context, pending pendingRemoval) { diff --git a/sandboxd/pool/volume.go b/sandboxd/pool/volume.go index 5a1e8eed..1075ce04 100644 --- a/sandboxd/pool/volume.go +++ b/sandboxd/pool/volume.go @@ -31,20 +31,24 @@ type volumeHolders struct { } type resolvedVolume struct { - disk engine.VolumeSpec - applied types.Volume + disk engine.VolumeSpec + applied types.Volume + managedID volumeID + holdID string } // volumeTeardown captures marker paths at quiesce time; a later config can re-point the entry. type volumeTeardown struct { - holds []types.Volume - clears []string + managedID string + holds []types.Volume + clears []string } type catalogVolume struct { - disk engine.VolumeSpec - tenants []string - writable bool + managedID volumeID + disk engine.VolumeSpec + tenants []string + writable bool } func (v catalogVolume) allowed(tenant string) bool { @@ -71,10 +75,12 @@ func (m *Manager) Volumes(tenant string, holders map[string]int) []types.VolumeI } infos = append(infos, info) } + infos = append(infos, m.managedInfos(tenant)...) // root can see peer-only names; a tenant needs local catalog metadata to apply the access list if tenant == "" { for name, nodes := range holders { - if _, ok := m.volumes[name]; !ok { + _, managed, _ := m.lookupManaged(name) + if _, ok := m.volumes[name]; !ok && !managed { infos = append(infos, types.VolumeInfo{ Name: name, DefaultMount: types.DefaultVolumeMount(name), @@ -109,17 +115,29 @@ func (m *Manager) VolumePlacement(key types.PoolKey, tenant string, names []stri } local := true for _, name := range names { - volume, ok := m.volumes[name] + volume, ok, err := m.lookupVolume(name) + if !ok && err != nil { + return false, err + } if !ok { if tenant == "" { local = false continue } + if m.managed != nil { + return false, managedNotFound() + } return false, ErrVolumeUnavailable } if !volume.allowed(tenant) { + if m.managed != nil { + return false, managedNotFound() + } return false, ErrVolumeUnavailable } + if err != nil { + return false, err + } if _, err := os.Stat(volume.disk.Path); err != nil { local = false } @@ -138,10 +156,19 @@ func (m *Manager) resolveVolumes(ctx context.Context, tenant string, requested [ } resolved := make([]resolvedVolume, 0, len(applied)) for _, volume := range applied { - entry, ok := m.volumes[volume.Name] + entry, ok, lookupErr := m.lookupVolume(volume.Name) + if !ok && lookupErr != nil { + return nil, lookupErr + } if !ok || !entry.allowed(tenant) { + if m.managed != nil { + return nil, managedNotFound() + } return nil, ErrVolumeUnavailable } + if lookupErr != nil { + return nil, lookupErr + } resolvedEntry, resolveErr := m.resolveCatalogVolume(ctx, volume) if resolveErr != nil { return nil, resolveErr @@ -152,7 +179,10 @@ func (m *Manager) resolveVolumes(ctx context.Context, tenant string, requested [ } func (m *Manager) resolveCatalogVolume(ctx context.Context, volume types.Volume) (resolvedVolume, error) { - entry, ok := m.volumes[volume.Name] + entry, ok, lookupErr := m.lookupVolume(volume.Name) + if lookupErr != nil { + return resolvedVolume{}, lookupErr + } if !ok { return resolvedVolume{}, ErrVolumeUnavailable } @@ -167,7 +197,17 @@ func (m *Manager) resolveCatalogVolume(ctx context.Context, volume types.Volume) disk := entry.disk disk.RW = volume.RW() volume.AttachOnly = false // past validation the empty mount carries it - return resolvedVolume{disk: disk, applied: volume}, nil + return resolvedVolume{disk: disk, applied: volume, managedID: entry.managedID}, nil +} + +func (m *Manager) lookupVolume(name string) (catalogVolume, bool, error) { + if entry, ok := m.volumes[name]; ok { + if m.managedCatalogAlias(entry.disk.Path) { + return catalogVolume{}, false, managedStorageError() + } + return entry, true, nil + } + return m.lookupManaged(name) } // confirmVolumesClean refuses a read-only claim of a dirty image; run after holds are taken. @@ -185,6 +225,9 @@ func (m *Manager) applyVolumes(ctx context.Context, sb *types.Sandbox, volumes [ if len(volumes) == 0 { return nil } + if err := m.bindManaged(ctx, sb, volumes); err != nil { + return err + } group, groupCtx := errgroup.WithContext(ctx) for _, volume := range volumes { group.Go(func() error { return m.applyVolume(groupCtx, sb, volume, false) }) @@ -198,6 +241,33 @@ func (m *Manager) applyVolumes(ctx context.Context, sb *types.Sandbox, volumes [ // applyVolume keeps one volume's steps strictly ordered; siblings overlap freely. func (m *Manager) applyVolume(ctx context.Context, sb *types.Sandbox, volume resolvedVolume, live bool) error { + if volume.managedID == "" { + if m.managedCatalogAlias(volume.disk.Path) { + return managedStorageError() + } + if h, managed := m.managedHold(sb.ID); managed { + matched := false + for _, ref := range h.Refs { + if ref.Volume.Name == volume.applied.Name && ref.Path == volume.disk.Path { + identity, err := catalogBacking(volume.disk.Path) + matched = err == nil && identity == ref.Backing + break + } + } + if !matched { + return managedStorageError() + } + } + } + if volume.managedID != "" { + m.managed.mu.Lock() + v, ok := m.managed.data.Volumes[volume.applied.Name] + valid := ok && v.ID == volume.managedID && m.managed.fault == nil && m.managed.verify(v) == nil + m.managed.mu.Unlock() + if !valid { + return managedStorageError() + } + } // attach-only mounts nothing, so there is no umount to verify and no marker. attachOnly := volume.applied.Mount == "" // write-ahead: the marker must be durable before any guest write can be. @@ -228,6 +298,9 @@ func (m *Manager) applyVolume(ctx context.Context, sb *types.Sandbox, volume res // quiesceVolumes unmounts a claim's writable mounts in reverse; a failed unmount keeps the marker. func (m *Manager) quiesceVolumes(ctx context.Context, sb *types.Sandbox) volumeTeardown { td := volumeTeardown{holds: heldVolumes(sb)} + if h, managed := m.managedHold(sb.ID); managed { + td.managedID = h.SandboxID + } mounts := writableMounts(sb.Volumes) if mounts == 0 { return td @@ -246,7 +319,9 @@ func (m *Manager) quiesceVolumes(ctx context.Context, sb *types.Sandbox) volumeT stuck = true continue } - if entry, ok := m.volumes[volume.Name]; ok { + if path, ok := m.managedRefPath(sb.ID, volume.Name); ok { + td.clears = append(td.clears, path) + } else if entry, ok := m.volumes[volume.Name]; ok { td.clears = append(td.clears, entry.disk.Path) } } @@ -263,6 +338,10 @@ func (m *Manager) quiesceVolumes(ctx context.Context, sb *types.Sandbox) volumeT // finishVolumeTeardown runs only once the VM is gone, when its image lock is finally released. func (m *Manager) finishVolumeTeardown(ctx context.Context, td volumeTeardown) { + if td.managedID != "" { + m.finishManaged(td.managedID, td.clears) + return + } if len(td.holds) == 0 { return } diff --git a/sandboxd/pool/volume_mutation.go b/sandboxd/pool/volume_mutation.go index 33dc50e2..ff309432 100644 --- a/sandboxd/pool/volume_mutation.go +++ b/sandboxd/pool/volume_mutation.go @@ -85,7 +85,7 @@ func (m *Manager) changeVolumes(ctx context.Context, id, tenant string, change v view = m.finishVolumeRun(sb, run) }() m.mu.Lock() - current, held := slices.Clone(sb.VolumesWant), heldVolumes(sb) + current, held := slices.Clone(sb.VolumesWant), slices.Clone(run.holds) m.mu.Unlock() target, err := change(current) if err != nil { @@ -98,6 +98,9 @@ func (m *Manager) changeVolumes(ctx context.Context, id, tenant string, change v if err = validateLiveTarget(held, target); err != nil { return view, err } + if err = m.validateManagedTarget(sb.ID, target); err != nil { + return view, err + } additions := volumeDifference(target, held) if len(additions) != 0 { if err = m.tenantRemoved(ctx, sb.Tenant); err != nil { @@ -108,12 +111,17 @@ func (m *Manager) changeVolumes(ctx context.Context, id, tenant string, change v if err != nil { return view, err } + for _, volume := range resolved { + if volume.managedID != "" { + return view, fmt.Errorf("%w: managed volumes support claim-time attachment only", ErrBadVolume) + } + } if err = m.checkVolumeRun(ctx, sb, run); err != nil { return view, err } sb.Touch() m.mu.Lock() - uncertain := volumeDifference(volumeDifference(sb.VolumesWant, sb.Volumes), target) + uncertain := volumeDifference(volumeDifference(run.holds, sb.Volumes), target) m.mu.Unlock() for _, v := range uncertain { failedName = v.Name @@ -177,6 +185,16 @@ func (m *Manager) beginVolumeRun(ctx context.Context, sb *types.Sandbox) (contex m.mu.Unlock() return ctx, nil, ErrUnknownSandbox } + if h, managed := m.managedHold(sb.ID); managed { + if m.ManagedVolumeStatus() != nil { + m.mu.Unlock() + return ctx, nil, managedStorageError() + } + if h.State != holdClaimed { + m.mu.Unlock() + return ctx, nil, ErrFailed + } + } if active := m.volumeRuns[sb]; active != nil { done := active.done m.mu.Unlock() @@ -189,6 +207,9 @@ func (m *Manager) beginVolumeRun(ctx context.Context, sb *types.Sandbox) (contex } runCtx, cancel := context.WithCancel(ctx) run := &volumeRun{cancel: cancel, done: make(chan struct{}), holds: heldVolumes(sb)} + if held, managed := m.managedVolumesHeld(sb.ID); managed { + run.holds = held + } m.volumeRuns[sb] = run m.mu.Unlock() return runCtx, run, nil @@ -197,7 +218,7 @@ func (m *Manager) beginVolumeRun(ctx context.Context, sb *types.Sandbox) (contex func (m *Manager) finishVolumeRun(sb *types.Sandbox, run *volumeRun) types.SandboxVolumesResponse { m.mu.Lock() - if m.claimed[sb.ID] == sb { + if _, managed := m.managedHold(sb.ID); m.claimed[sb.ID] == sb && !managed { m.releaseVolumes(volumeDifference(run.holds, heldVolumes(sb))) } delete(m.volumeRuns, sb) @@ -209,6 +230,9 @@ func (m *Manager) finishVolumeRun(sb *types.Sandbox, run *volumeRun) types.Sandb } func (m *Manager) restoreVolumeHolds(sb *types.Sandbox, holds []types.Volume) { + if _, managed := m.managedHold(sb.ID); managed { + return + } if m.volumeRuns[sb] == nil { m.releaseVolumes(volumeDifference(holds, heldVolumes(sb))) } @@ -227,6 +251,9 @@ func (m *Manager) checkVolumeRunLocked(ctx context.Context, sb *types.Sandbox, r if !m.ownsRunLocked(sb, run) { return ErrUnknownSandbox } + if h, managed := m.managedHold(sb.ID); managed && h.State != holdClaimed { + return ErrFailed + } switch { case sb.Failed != "": return ErrFailed @@ -244,6 +271,9 @@ func (m *Manager) cancelVolumeRun(sb *types.Sandbox) []types.Volume { run.cancel() return slices.Clone(run.holds) } + if held, managed := m.managedVolumesHeld(sb.ID); managed { + return held + } return heldVolumes(sb) } @@ -272,6 +302,14 @@ func (m *Manager) stageVolumeTarget(ctx context.Context, sb *types.Sandbox, run return claimSnapshot{}, false, err } run.holds = append(run.holds, additions...) + if err := m.managedLiveAdd(sb.ID, resolved); err != nil { + if m.ManagedVolumeStatus() == nil { + m.releaseVolumes(additions) + run.holds = volumeDifference(run.holds, additions) + } + m.mu.Unlock() + return claimSnapshot{}, false, err + } m.mu.Unlock() cleanErr := m.confirmVolumesClean(resolved) m.mu.Lock() @@ -280,6 +318,9 @@ func (m *Manager) stageVolumeTarget(ctx context.Context, sb *types.Sandbox, run return claimSnapshot{}, false, err } if cleanErr != nil { + if err := m.managedLiveRemove(sb.ID, additions); err != nil { + return claimSnapshot{}, false, err + } m.releaseVolumes(additions) run.holds = volumeDifference(run.holds, additions) return claimSnapshot{}, false, cleanErr @@ -287,7 +328,7 @@ func (m *Manager) stageVolumeTarget(ctx context.Context, sb *types.Sandbox, run replaced := func(v types.Volume) bool { return volumeIndex(sb.VolumesWant, v.Name) < 0 && volumeIndex(target, v.Name) >= 0 } - if len(additions) == 0 && slices.Equal(target, sb.VolumesWant) && !slices.ContainsFunc(sb.Volumes, replaced) && m.store.synced() { + if len(additions) == 0 && slices.Equal(target, sb.VolumesWant) && !slices.ContainsFunc(sb.Volumes, replaced) && m.store.synced() && len(volumeDifference(run.holds, heldVolumes(sb))) == 0 { return claimSnapshot{}, false, nil } sb.Volumes = slices.DeleteFunc(slices.Clone(sb.Volumes), replaced) @@ -334,6 +375,9 @@ func (m *Manager) commitVolumeRun(ctx context.Context, sb *types.Sandbox, run *v return err } dropped := volumeDifference(run.holds, heldVolumes(sb)) + if err := m.managedLiveRemove(sb.ID, dropped); err != nil { + return err + } m.releaseVolumes(dropped) run.holds = volumeDifference(run.holds, dropped) return nil diff --git a/sandboxd/pool/volume_registry.go b/sandboxd/pool/volume_registry.go new file mode 100644 index 00000000..dc1e6c5f --- /dev/null +++ b/sandboxd/pool/volume_registry.go @@ -0,0 +1,402 @@ +package pool + +import ( + "encoding/json/v2" + "errors" + "fmt" + "os" + "path/filepath" + "regexp" + "strings" + "sync" + "syscall" + "time" + + "github.com/cocoonstack/sandbox/sandboxd/types" +) + +type volumeID string +type volumePhase string +type holdPhase string + +const ( + volumeCreating volumePhase = "creating" + volumeReady volumePhase = "ready" + volumeDeleting volumePhase = "deleting" + volumeRecovery volumePhase = "recovery_required" + holdAttaching holdPhase = "attaching" + holdClaimed holdPhase = "claimed" + holdRetiring holdPhase = "retiring" +) + +type backingIdentity struct { + Device uint64 `json:"device"` + Inode uint64 `json:"inode"` + Size int64 `json:"size"` +} + +type managedVolume struct { + ID volumeID `json:"id"` + Name string `json:"name"` + Tenant string `json:"tenant"` + Node string `json:"node"` + Size int64 `json:"size"` + Backing backingIdentity `json:"backing"` + State volumePhase `json:"state"` + CreatedAt time.Time `json:"created_at"` +} + +type volumeRef struct { + Volume types.Volume `json:"volume"` + ID volumeID `json:"id,omitempty"` + Path string `json:"path"` + Backing backingIdentity `json:"backing"` +} + +type processIdentity struct { + BootID string `json:"boot_id"` + PID int `json:"pid"` + Start uint64 `json:"start"` +} + +type volumeHold struct { + SandboxID string `json:"sandbox_id"` + Tenant string `json:"tenant"` + VMName string `json:"vm_name,omitempty"` + Process processIdentity `json:"process"` + State holdPhase `json:"state"` + Refs []volumeRef `json:"refs"` +} + +type volumeTombstone struct { + ID volumeID `json:"id"` + Name string `json:"name"` + Tenant string `json:"tenant"` +} + +type volumeRegistryData struct { + Orphans map[string]processIdentity `json:"orphans,omitempty"` + Version int `json:"version"` + Volumes map[string]managedVolume `json:"volumes"` + Holds map[string]volumeHold `json:"holds"` + Deleted map[volumeID]volumeTombstone `json:"deleted"` +} + +type volumeRegistry struct { + rootDir, disksDir os.FileInfo + mu sync.Mutex + dir, node string + lock *os.File + data volumeRegistryData + fault error + commitFile func(string, []byte) error + process func(int) (processIdentity, error) +} + +var managedIDPattern = regexp.MustCompile(`^vol_[0-9a-f]{32}$`) + +func openVolumeRegistry(dir, node string) (*volumeRegistry, error) { + if !filepath.IsAbs(dir) || filepath.Clean(dir) != dir || dir == "/" { + return nil, fmt.Errorf("invalid managed volume directory") + } + if err := privateVolumeDir(dir); err != nil { + return nil, err + } + if err := privateVolumeDir(filepath.Join(dir, "disks")); err != nil { + return nil, err + } + lock, err := openPrivateFile(filepath.Join(dir, "lock"), syscall.O_CREAT|syscall.O_RDWR) + if err != nil { + return nil, fmt.Errorf("open managed volume lock: %w", err) + } + if err = syscall.Flock(int(lock.Fd()), syscall.LOCK_EX|syscall.LOCK_NB); err != nil { + lock.Close() + return nil, fmt.Errorf("managed volume directory is already in use: %w", err) + } + r := &volumeRegistry{dir: dir, node: node, lock: lock, commitFile: writeVolumeSnapshot, process: readProcessIdentity} + r.rootDir, err = os.Lstat(dir) + if err != nil { + lock.Close() + return nil, err + } + r.disksDir, err = os.Lstat(filepath.Join(dir, "disks")) + if err != nil { + lock.Close() + return nil, err + } + r.data = volumeRegistryData{Version: 1, Volumes: map[string]managedVolume{}, Holds: map[string]volumeHold{}, Deleted: map[volumeID]volumeTombstone{}} + f, err := openPrivateFile(filepath.Join(dir, "registry.json"), syscall.O_RDONLY) + if errors.Is(err, os.ErrNotExist) { + entries, readErr := os.ReadDir(filepath.Join(dir, "disks")) + if readErr != nil || len(entries) != 0 { + r.fault = errors.New("missing registry with existing backing files") + return r, nil + } + if err := r.commit(); err != nil { + return r, nil + } + return r, nil + } + if err != nil { + r.fault = err + return r, nil + } + defer f.Close() + if err := json.UnmarshalRead(f, &r.data, json.RejectUnknownMembers(true)); err != nil { + r.fault = err + return r, nil + } + if err := r.validate(); err != nil { + r.fault = err + } + return r, nil +} + +func privateVolumeDir(path string) error { + for p := path; p != "/"; p = filepath.Dir(p) { + st, err := os.Lstat(p) + if errors.Is(err, os.ErrNotExist) { + continue + } + if err != nil { + return err + } + owner := st.Sys().(*syscall.Stat_t).Uid + if owner != 0 && owner != uint32(os.Geteuid()) { + return fmt.Errorf("managed directory ancestors must belong to root or the daemon user") + } + if st.Mode().Perm()&0022 != 0 && st.Mode()&os.ModeSticky == 0 { + return fmt.Errorf("managed directory ancestors must not be writable by others") + } + if !st.IsDir() || st.Mode()&os.ModeSymlink != 0 { + return fmt.Errorf("managed directory component is not a directory") + } + } + if err := os.MkdirAll(path, 0700); err != nil { + return err + } + st, err := os.Lstat(path) + if err != nil { + return err + } + if !st.IsDir() || st.Mode().Perm()&0077 != 0 { + return fmt.Errorf("managed directory must be private (0700)") + } + if st.Sys().(*syscall.Stat_t).Uid != uint32(os.Geteuid()) { + return fmt.Errorf("managed directory must belong to the daemon user") + } + return nil +} + +func openPrivateFile(path string, flags int) (*os.File, error) { + fd, err := syscall.Open(path, flags|syscall.O_CLOEXEC|syscall.O_NOFOLLOW, 0600) + if err != nil { + return nil, err + } + f := os.NewFile(uintptr(fd), path) + st, err := f.Stat() + if err != nil || !st.Mode().IsRegular() || st.Mode().Perm()&0077 != 0 || st.Sys().(*syscall.Stat_t).Nlink != 1 || st.Sys().(*syscall.Stat_t).Uid != uint32(os.Geteuid()) { + f.Close() + return nil, fmt.Errorf("managed file must be private, owned, regular, and unlinked elsewhere") + } + return f, nil +} + +func (r *volumeRegistry) path(id volumeID) string { + return filepath.Join(r.dir, "disks", string(id)+".raw") +} + +func inspectBacking(path string) (backingIdentity, error) { + f, err := openPrivateFile(path, syscall.O_RDONLY) + if err != nil { + return backingIdentity{}, err + } + defer f.Close() + st, err := f.Stat() + if err != nil { + return backingIdentity{}, err + } + raw := st.Sys().(*syscall.Stat_t) + return backingIdentity{Device: uint64(raw.Dev), Inode: raw.Ino, Size: st.Size()}, nil +} + +func (r *volumeRegistry) verify(v managedVolume) error { + if err := r.verifyDirectories(); err != nil { + return err + } + actual, err := inspectBacking(r.path(v.ID)) + if err != nil { + return err + } + if v.Backing.Inode == 0 || actual != v.Backing || actual.Size != v.Size { + return errors.New("managed backing identity changed") + } + return nil +} + +func (r *volumeRegistry) validate() error { + d := r.data + if d.Version != 1 || d.Volumes == nil || d.Holds == nil || d.Deleted == nil { + return errors.New("invalid volume registry format") + } + ids := map[volumeID]bool{} + for name, v := range d.Volumes { + if name != v.Name || !types.ValidVolumeName(name) || !managedIDPattern.MatchString(string(v.ID)) || !types.NameRe.MatchString(v.Tenant) || v.Size < 1<<30 || v.Size > 1024<<30 || v.Size%(1<<30) != 0 || v.CreatedAt.IsZero() || v.Node == "" || ids[v.ID] { + return errors.New("invalid managed volume record") + } + switch v.State { + case volumeCreating, volumeReady, volumeDeleting, volumeRecovery: + default: + return errors.New("invalid managed volume state") + } + if (v.State == volumeReady || v.Backing.Inode != 0) && (v.Backing.Inode == 0 || v.Backing.Size != v.Size) { + return errors.New("invalid managed backing record") + } + ids[v.ID] = true + } + vms := map[string]bool{} + holders := map[string]volumeHolders{} + for id, h := range d.Holds { + if id != h.SandboxID || !strings.HasPrefix(id, "sb_") || len(h.Refs) == 0 { + return errors.New("invalid volume holder") + } + switch h.State { + case holdAttaching, holdClaimed, holdRetiring: + default: + return errors.New("invalid volume holder state") + } + if h.VMName != "" { + if !strings.HasPrefix(h.VMName, vmPrefix) || vms[h.VMName] || h.Process.PID <= 0 || h.Process.Start == 0 || h.Process.BootID == "" { + return errors.New("invalid volume process identity") + } + vms[h.VMName] = true + } else if h.State == holdClaimed { + return errors.New("claimed volume lacks VM") + } + names := map[string]bool{} + managed := false + for _, ref := range h.Refs { + if !types.ValidVolumeName(ref.Volume.Name) || names[ref.Volume.Name] || !filepath.IsAbs(ref.Path) || (ref.Volume.Mode != "" && ref.Volume.Mode != types.VolumeModeRW) { + return errors.New("invalid volume reference") + } + names[ref.Volume.Name] = true + counts := holders[ref.Volume.Name] + if counts.writers > 0 || ref.Volume.RW() && counts.readers > 0 { + return errors.New("conflicting volume holder records") + } + if ref.Volume.RW() { + counts.writers++ + } else { + counts.readers++ + } + holders[ref.Volume.Name] = counts + if ref.ID != "" { + v, ok := d.Volumes[ref.Volume.Name] + if !ok || (v.State != volumeReady && v.State != volumeRecovery) || ref.ID != v.ID || ref.Path != r.path(v.ID) || ref.Backing != v.Backing || (h.Tenant != "" && h.Tenant != v.Tenant) { + return errors.New("inconsistent managed volume reference") + } + managed = true + } + } + if !managed { + return errors.New("managed holder has no managed resource") + } + } + for name, p := range d.Orphans { + if !strings.HasPrefix(name, vmPrefix) || p.PID <= 0 || p.Start == 0 || p.BootID == "" { + return errors.New("invalid orphan process record") + } + } + for id, t := range d.Deleted { + if id != t.ID || !managedIDPattern.MatchString(string(id)) || !types.ValidVolumeName(t.Name) || !types.NameRe.MatchString(t.Tenant) || ids[id] { + return errors.New("invalid volume tombstone") + } + } + entries, err := os.ReadDir(filepath.Join(r.dir, "disks")) + if err != nil { + return err + } + for _, entry := range entries { + name := strings.TrimSuffix(entry.Name(), volumeDirtySuffix) + id := volumeID(strings.TrimSuffix(name, ".raw")) + if name != string(id)+".raw" || !ids[id] { + return errors.New("unregistered managed backing requires quarantine") + } + } + return nil +} + +// Any failed snapshot poisons mutations until restart. The attempted state remains +// owned in memory because rename may already have published it. +func (r *volumeRegistry) commit() error { + if r.fault != nil { + return managedStorageError() + } + if err := r.verifyDirectories(); err != nil { + r.fault = err + return managedStorageError() + } + raw, err := json.Marshal(r.data) + if err == nil { + err = r.commitFile(filepath.Join(r.dir, "registry.json"), raw) + } + if err != nil { + r.fault = err + return managedStorageError() + } + return nil +} + +func writeVolumeSnapshot(path string, raw []byte) error { + tmp, err := os.CreateTemp(filepath.Dir(path), ".registry-") + if err != nil { + return err + } + defer os.Remove(tmp.Name()) + if _, err := tmp.Write(raw); err != nil { + tmp.Close() + return err + } + if err := errors.Join(tmp.Sync(), tmp.Close()); err != nil { + return err + } + if err := os.Rename(tmp.Name(), path); err != nil { + return err + } + return syncDir(filepath.Dir(path)) +} + +func (r *volumeRegistry) close() error { + r.mu.Lock() + defer r.mu.Unlock() + if r.lock == nil { + return nil + } + err := r.lock.Close() + r.lock = nil + return err +} + +// Close releases the managed directory lock after requests and workers have stopped. +func (m *Manager) Close() error { + if m.managed == nil { + return nil + } + return m.managed.close() +} + +func (r *volumeRegistry) verifyDirectories() error { + for _, pair := range []struct { + path string + original os.FileInfo + }{{r.dir, r.rootDir}, {filepath.Join(r.dir, "disks"), r.disksDir}} { + st, err := os.Lstat(pair.path) + if err != nil { + return err + } + if !st.IsDir() || st.Mode().Perm()&0077 != 0 || !os.SameFile(st, pair.original) { + return errors.New("managed directory identity or permissions changed") + } + } + return nil +} diff --git a/sandboxd/server/api.go b/sandboxd/server/api.go index 6e26e0c9..f612d71b 100644 --- a/sandboxd/server/api.go +++ b/sandboxd/server/api.go @@ -6,6 +6,10 @@ import ( "github.com/cocoonstack/sandbox/sandboxd/types" ) +type VolumeResponse struct { + Volume types.ManagedVolumeInfo `json:"volume"` +} + // TTLField is the shared requested-lease field; zero means the server default. type TTLField struct { TTLSeconds int `json:"ttl_seconds,omitzero"` diff --git a/sandboxd/server/managed_volumes.go b/sandboxd/server/managed_volumes.go new file mode 100644 index 00000000..f45b0d9e --- /dev/null +++ b/sandboxd/server/managed_volumes.go @@ -0,0 +1,129 @@ +package server + +import ( + "context" + "errors" + "io" + "net/http" + "strings" + + "github.com/cocoonstack/sandbox/sandboxd/pool" + "github.com/cocoonstack/sandbox/sandboxd/types" + "github.com/cocoonstack/sandbox/sandboxd/utils" +) + +type managedVolumeManager interface { + CreateVolume(context.Context, string, types.VolumeCreate, string) (types.ManagedVolumeInfo, bool, error) + GetVolume(string, string) (types.ManagedVolumeInfo, error) + DeleteVolume(context.Context, string, string, string) error + ManagedVolumeStatus() error + ManagedVolumesEnabled() bool +} + +func volumeHTTPError(w http.ResponseWriter, status int, code, message string) { + writeJSON(w, status, &pool.VolumeError{Status: status, Code: code, Message: message}) +} + +func (s *Server) requireVolumeListToken(next http.HandlerFunc) http.HandlerFunc { + if mgr, ok := s.mgr.(managedVolumeManager); ok && mgr.ManagedVolumesEnabled() { + return s.requireVolumeManager(next) + } + return s.requireToken(next) +} + +func (s *Server) requireVolumeManager(next http.HandlerFunc) http.HandlerFunc { + return func(w http.ResponseWriter, r *http.Request) { + token, ok := bearerToken(r) + if !ok { + volumeHTTPError(w, 401, "volume_unauthorized", "explicit root or tenant bearer token required") + return + } + if s.isRootToken(token) { + next(w, r.WithContext(withTenant(r.Context(), nil))) + return + } + tenant, err := s.mgr.TenantByToken(r.Context(), token) + if errors.Is(err, pool.ErrTenantStoreDown) { + w.Header().Set("Retry-After", retryAfterSeconds) + volumeHTTPError(w, 503, "tenant_store_unavailable", "tenant store unavailable; retry") + return + } + if err != nil || tenant == nil { + volumeHTTPError(w, 401, "volume_unauthorized", "explicit root or tenant bearer token required") + return + } + next(w, r.WithContext(withTenant(r.Context(), tenant))) + } +} + +func (s *Server) volumeManager(w http.ResponseWriter) (managedVolumeManager, bool) { + mgr, ok := s.mgr.(managedVolumeManager) + if !ok || !mgr.ManagedVolumesEnabled() { + volumeHTTPError(w, 503, "volume_storage_unavailable", "managed volume storage is not enabled") + return nil, false + } + return mgr, true +} + +func (s *Server) handleCreateVolume(w http.ResponseWriter, r *http.Request) { + mgr, ok := s.volumeManager(w) + if !ok { + return + } + raw, err := io.ReadAll(http.MaxBytesReader(w, r.Body, maxBodyBytes)) + var req types.VolumeCreate + if err != nil || utils.DecodeStrictJSON(raw, &req) != nil { + volumeHTTPError(w, 400, "invalid_volume", "invalid request body") + return + } + v, created, err := mgr.CreateVolume(r.Context(), r.PathValue("name"), req, tenantFrom(r.Context())) + if err != nil { + if !writePoolErr(w, err) { + volumeHTTPError(w, 503, "volume_storage_unavailable", "volume storage unavailable; retry") + } + return + } + status := http.StatusOK + if created { + status = http.StatusCreated + } + w.Header().Set("ETag", `"`+v.ID+`"`) + writeJSON(w, status, VolumeResponse{Volume: v}) +} + +func (s *Server) handleGetVolume(w http.ResponseWriter, r *http.Request) { + mgr, ok := s.volumeManager(w) + if !ok { + return + } + v, err := mgr.GetVolume(r.PathValue("name"), tenantFrom(r.Context())) + if err != nil { + writePoolErr(w, err) + return + } + w.Header().Set("ETag", `"`+v.ID+`"`) + writeJSON(w, http.StatusOK, VolumeResponse{Volume: v}) +} + +func (s *Server) handleDeleteVolume(w http.ResponseWriter, r *http.Request) { + mgr, ok := s.volumeManager(w) + if !ok { + return + } + expected := r.Header.Get("If-Match") + if expected == "" { + volumeHTTPError(w, 428, "volume_precondition_required", "If-Match must identify the volume") + return + } + if len(expected) < 3 || expected[0] != '"' || expected[len(expected)-1] != '"' || strings.ContainsAny(expected[1:len(expected)-1], "\",\\") { + volumeHTTPError(w, 412, "volume_precondition_failed", "If-Match must contain one quoted volume identity") + return + } + if err := mgr.DeleteVolume(r.Context(), r.PathValue("name"), expected[1:len(expected)-1], tenantFrom(r.Context())); err != nil { + if !writePoolErr(w, err) { + volumeHTTPError(w, 503, "volume_storage_unavailable", "volume storage unavailable; retry") + } + return + } + w.WriteHeader(http.StatusNoContent) +} diff --git a/sandboxd/server/server.go b/sandboxd/server/server.go index 78b44fd6..9a4f0669 100644 --- a/sandboxd/server/server.go +++ b/sandboxd/server/server.go @@ -268,7 +268,10 @@ func (s *Server) SetConfigReload(f ConfigReloader) { func (s *Server) Handler() http.Handler { mux := http.NewServeMux() mux.HandleFunc("POST /v1/claim", s.requireToken(s.handleClaim)) - mux.HandleFunc("GET /v1/volumes", s.requireToken(s.handleVolumes)) + mux.HandleFunc("GET /v1/volumes", s.requireVolumeListToken(s.handleVolumes)) + mux.HandleFunc("PUT /v1/volumes/{name}", s.requireVolumeManager(s.handleCreateVolume)) + mux.HandleFunc("GET /v1/volumes/{name}", s.requireVolumeManager(s.handleGetVolume)) + mux.HandleFunc("DELETE /v1/volumes/{name}", s.requireVolumeManager(s.handleDeleteVolume)) mux.HandleFunc("POST /v1/sandboxes/{id}/release", s.handleSandboxVerb("release", s.mgr.Release)) mux.HandleFunc("POST /v1/sandboxes/{id}/hibernate", s.handleSandboxVerb("hibernate", s.mgr.Hibernate)) mux.HandleFunc("POST /v1/sandboxes/{id}/wake", s.handleSandboxVerb("wake", s.mgr.Wake)) @@ -383,6 +386,12 @@ func (s *Server) handleVolumeClaim(w http.ResponseWriter, r *http.Request, req C } func (s *Server) handleVolumes(w http.ResponseWriter, r *http.Request) { + if mgr, ok := s.mgr.(managedVolumeManager); ok { + if err := mgr.ManagedVolumeStatus(); err != nil { + writePoolErr(w, err) + return + } + } var holders map[string]int if s.placer != nil { holders = s.placer.VolumeHolders() diff --git a/sandboxd/server/server_http.go b/sandboxd/server/server_http.go index 48bf2a94..59a90d98 100644 --- a/sandboxd/server/server_http.go +++ b/sandboxd/server/server_http.go @@ -15,6 +15,7 @@ import ( "github.com/projecteru2/core/log" + "github.com/cocoonstack/sandbox/sandboxd/pool" "github.com/cocoonstack/sandbox/sandboxd/tenants" "github.com/cocoonstack/sandbox/sandboxd/types" "github.com/cocoonstack/sandbox/sandboxd/utils" @@ -170,6 +171,13 @@ func metadataFilter(pairs []string) (types.Metadata, error) { } func writePoolErr(w http.ResponseWriter, err error) bool { + if volumeErr, ok := errors.AsType[*pool.VolumeError](err); ok { + if volumeErr.Status == http.StatusServiceUnavailable { + w.Header().Set("Retry-After", retryAfterSeconds) + } + writeJSON(w, volumeErr.Status, volumeErr) + return true + } code, msg, ok := mapPoolErr(w, err) if ok { writeErr(w, code, msg) @@ -186,6 +194,9 @@ func mapPoolErr(w http.ResponseWriter, err error) (code int, msg string, ok bool } func lookupPoolErr(err error) (code int, msg string, ok bool) { + if volumeErr, ok := errors.AsType[*pool.VolumeError](err); ok { + return volumeErr.Status, volumeErr.Message, true + } for _, m := range poolErrHTTP { if errors.Is(err, m.err) { return m.code, cmp.Or(m.msg, err.Error()), true diff --git a/sandboxd/types/api.go b/sandboxd/types/api.go index ac0cd6b4..0b0e8b11 100644 --- a/sandboxd/types/api.go +++ b/sandboxd/types/api.go @@ -13,15 +13,37 @@ const ( // VolumeInfo is the caller-visible, host-path-free catalog projection. type VolumeInfo struct { - Name string `json:"name"` - DefaultMount string `json:"default_mount"` - SizeBytes int64 `json:"size_bytes"` - Available bool `json:"available"` - Nodes int `json:"nodes"` + Kind string `json:"kind,omitempty"` + ID string `json:"id,omitempty"` + Tenant string `json:"tenant,omitempty"` + Node string `json:"node,omitempty"` + State string `json:"state,omitempty"` + CreatedAt time.Time `json:"created_at,omitzero"` + Name string `json:"name"` + DefaultMount string `json:"default_mount"` + SizeBytes int64 `json:"size_bytes"` + Available bool `json:"available"` + Nodes int `json:"nodes"` // Writable reports whether the operator allows rw claims of this name. Writable bool `json:"writable,omitzero"` } +type ManagedVolumeInfo struct { + VolumeInfo + Holders []VolumeHolder `json:"holders,omitempty"` +} + +type VolumeHolder struct { + SandboxID string `json:"sandbox_id"` + Mode string `json:"mode"` + State string `json:"state"` +} + +type VolumeCreate struct { + SizeGiB int64 `json:"size_gib"` + Tenant string `json:"tenant,omitempty"` +} + // VolumePatch is the wire body of PATCH /v1/sandboxes/{id}/volumes. type VolumePatch struct { Add []Volume `json:"add,omitzero"` diff --git a/sdk/go/client.go b/sdk/go/client.go index ccb932fd..082c7727 100644 --- a/sdk/go/client.go +++ b/sdk/go/client.go @@ -71,7 +71,10 @@ type renewResponse struct { } type errorResponse struct { - Error string `json:"error"` + Error string `json:"error"` + Code string `json:"code"` + Volume string `json:"volume"` + Holders []VolumeHolder `json:"holders"` } // APIError is a non-2xx control-plane reply. @@ -443,7 +446,11 @@ func encodeBody(verb string, v any) ([]byte, error) { func apiError(verb string, resp *http.Response) error { var er errorResponse - _ = json.UnmarshalRead(io.LimitReader(resp.Body, 4096), &er) + _ = json.UnmarshalRead(resp.Body, &er) _, _ = io.Copy(io.Discard, resp.Body) - return &APIError{Verb: verb, Status: resp.StatusCode, Message: er.Error} + base := &APIError{Verb: verb, Status: resp.StatusCode, Message: er.Error} + if er.Code != "" || len(er.Holders) != 0 { + return &ManagedVolumeError{APIError: base, Code: er.Code, Volume: er.Volume, Holders: er.Holders} + } + return base } diff --git a/sdk/go/managed_volume.go b/sdk/go/managed_volume.go new file mode 100644 index 00000000..05adbd17 --- /dev/null +++ b/sdk/go/managed_volume.go @@ -0,0 +1,129 @@ +package sandbox + +import ( + "bytes" + "context" + "encoding/json/v2" + "fmt" + "io" + "net/http" + "net/url" + "regexp" + "strings" +) + +type ManagedVolumeInfo struct { + VolumeInfo + Holders []VolumeHolder `json:"holders,omitempty"` +} + +type ManagedVolumeError struct { + *APIError + Code string + Volume string + Holders []VolumeHolder +} + +func (e *ManagedVolumeError) Unwrap() error { return e.APIError } + +var managedVolumeName = regexp.MustCompile(`^[a-z][a-z0-9_-]{0,19}$`) + +// VolumeCreate requests a raw managed disk owned by Tenant. A tenant token may leave Tenant empty. +type VolumeCreate struct { + SizeGiB int64 `json:"size_gib"` + Tenant string `json:"tenant,omitempty"` +} + +type volumeResponse struct { + Volume *ManagedVolumeInfo `json:"volume"` +} + +// CreateVolume creates a managed disk or returns the same disk for an identical retry. +func (c *Client) CreateVolume(ctx context.Context, name string, create VolumeCreate) (*ManagedVolumeInfo, error) { + if err := validManagedVolumeName(name); err != nil { + return nil, err + } + if create.SizeGiB < 1 || create.SizeGiB > 1024 { + return nil, fmt.Errorf("volume size_gib must be between 1 and 1024") + } + body, err := encodeBody("create volume", create) + if err != nil { + return nil, err + } + resp, err := c.roundTrip(ctx, http.MethodPut, c.addr, volumePath(name), bytes.NewReader(body), c.apiToken) + if err != nil { + return nil, err + } + defer resp.Body.Close() + if resp.StatusCode != http.StatusCreated && resp.StatusCode != http.StatusOK { + return nil, apiError("create volume", resp) + } + return decodeVolumeResponse("create volume", resp.Body) +} + +// GetVolume inspects one managed disk visible to this token. +func (c *Client) GetVolume(ctx context.Context, name string) (*ManagedVolumeInfo, error) { + if err := validManagedVolumeName(name); err != nil { + return nil, err + } + resp, err := c.roundTrip(ctx, http.MethodGet, c.addr, volumePath(name), nil, c.apiToken) + if err != nil { + return nil, err + } + defer resp.Body.Close() + if resp.StatusCode != http.StatusOK { + return nil, apiError("get volume", resp) + } + return decodeVolumeResponse("get volume", resp.Body) +} + +// DeleteVolume deletes exactly id, including when an earlier response was lost. +func (c *Client) DeleteVolume(ctx context.Context, name, id string) error { + if err := validManagedVolumeName(name); err != nil { + return err + } + if id == "" || strings.ContainsAny(id, "\"\r\n") { + return fmt.Errorf("volume id must be a nonempty unquoted resource id") + } + req, err := http.NewRequestWithContext(ctx, http.MethodDelete, c.requestURL(c.addr, volumePath(name)), nil) + if err != nil { + return err + } + req.Header.Set("If-Match", `"`+id+`"`) + if c.apiToken != "" { + req.Header.Set("Authorization", "Bearer "+c.apiToken) + } + resp, err := c.hc.Do(req) + if err != nil { + return err + } + defer resp.Body.Close() + if resp.StatusCode != http.StatusNoContent { + return apiError("delete volume", resp) + } + return nil +} + +func validManagedVolumeName(name string) error { + if !managedVolumeName.MatchString(name) || strings.HasPrefix(name, "cocoon-") { + return fmt.Errorf("volume name %q must match %s and not start with cocoon-", name, managedVolumeName) + } + return nil +} + +func volumePath(name string) string { + return "/v1/volumes/" + url.PathEscape(name) +} + +func decodeVolumeResponse(verb string, body io.Reader) (*ManagedVolumeInfo, error) { + var reply volumeResponse + if err := json.UnmarshalRead(body, &reply); err != nil { + return nil, fmt.Errorf("decode %s response: %w", verb, err) + } + if reply.Volume == nil || reply.Volume.Kind != "managed" || reply.Volume.ID == "" || + reply.Volume.Name == "" || reply.Volume.Tenant == "" || reply.Volume.Node == "" || + reply.Volume.State == "" || reply.Volume.CreatedAt.IsZero() || reply.Volume.SizeBytes < 1 { + return nil, fmt.Errorf("decode %s response: missing managed volume fields", verb) + } + return reply.Volume, nil +} diff --git a/sdk/go/managed_volume_test.go b/sdk/go/managed_volume_test.go new file mode 100644 index 00000000..10acfcef --- /dev/null +++ b/sdk/go/managed_volume_test.go @@ -0,0 +1,144 @@ +package sandbox + +import ( + "context" + "encoding/json/v2" + "errors" + "fmt" + "net/http" + "net/http/httptest" + "reflect" + "strings" + "testing" + "time" +) + +func TestManagedVolumeRequests(t *testing.T) { + want := ManagedVolumeInfo{VolumeInfo: VolumeInfo{Name: "data", Kind: "managed", ID: "vol_123", Tenant: "t1", Node: "node-1", State: "ready", CreatedAt: time.Date(2026, 10, 19, 0, 0, 0, 0, time.UTC), SizeBytes: 20 << 30, Available: true, Writable: true}, Holders: []VolumeHolder{{SandboxID: "sb_1", Mode: "rw", State: "attached"}}} + requests := 0 + ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + requests++ + if r.URL.Path != "/v1/volumes/data" || r.Header.Get("Authorization") != "Bearer token" { + t.Errorf("request = %s %s, authorization %q", r.Method, r.URL.Path, r.Header.Get("Authorization")) + } + switch r.Method { + case http.MethodPut: + var body map[string]any + if err := json.UnmarshalRead(r.Body, &body); err != nil { + t.Errorf("decode create: %v", err) + } + if body["size_gib"] != float64(20) || body["tenant"] != "t1" { + t.Errorf("create body = %v", body) + } + if requests == 1 { + w.WriteHeader(http.StatusCreated) + } + _ = json.MarshalWrite(w, volumeResponse{Volume: &want}) + case http.MethodGet: + w.Header().Set("ETag", `"vol_123"`) + _ = json.MarshalWrite(w, volumeResponse{Volume: &want}) + case http.MethodDelete: + if got := r.Header.Get("If-Match"); got != `"vol_123"` { + t.Errorf("If-Match = %q", got) + } + w.WriteHeader(http.StatusNoContent) + default: + t.Errorf("unexpected method %s", r.Method) + } + })) + t.Cleanup(ts.Close) + c := testClient(t, ts, WithAPIToken("token")) + for i := 0; i < 2; i++ { + got, err := c.CreateVolume(t.Context(), "data", VolumeCreate{SizeGiB: 20, Tenant: "t1"}) + if err != nil || !reflect.DeepEqual(got, &want) { + t.Fatalf("create = %+v, %v", got, err) + } + } + got, err := c.GetVolume(t.Context(), "data") + if err != nil || !reflect.DeepEqual(got, &want) { + t.Fatalf("get = %+v, %v", got, err) + } + if err := c.DeleteVolume(t.Context(), "data", want.ID); err != nil { + t.Fatal(err) + } + if err := c.DeleteVolume(t.Context(), "data", want.ID); err != nil { + t.Fatal(err) + } + if requests != 5 { + t.Fatalf("requests = %d, want 5", requests) + } +} + +func TestManagedVolumeFailures(t *testing.T) { + statuses := []int{400, 401, 403, 404, 409, 412, 428, 503} + for _, status := range statuses { + t.Run(http.StatusText(status), func(t *testing.T) { + ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(status) + _ = json.MarshalWrite(w, map[string]any{"error": "volume unavailable", "code": "volume_busy", "volume": "data", "holders": []map[string]string{{"sandbox_id": "sb_1", "mode": "rw", "state": "retiring", "host_path": "/secret"}}}) + })) + t.Cleanup(ts.Close) + _, err := testClient(t, ts).GetVolume(t.Context(), "data") + var apiErr *ManagedVolumeError + if !errors.As(err, &apiErr) || apiErr.Status != status || apiErr.Code != "volume_busy" || apiErr.Volume != "data" || !reflect.DeepEqual(apiErr.Holders, []VolumeHolder{{SandboxID: "sb_1", Mode: "rw", State: "retiring"}}) { + t.Fatalf("error = %#v", err) + } + if strings.Contains(apiErr.Error(), "/secret") { + t.Fatal("error exposed a host path") + } + }) + } +} + +func TestManagedVolumeRejectsInvalidRepliesAndInputs(t *testing.T) { + requests := 0 + ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + requests++ + _, _ = w.Write([]byte(`{}`)) + })) + t.Cleanup(ts.Close) + c := testClient(t, ts) + if _, err := c.CreateVolume(t.Context(), "bad/name", VolumeCreate{SizeGiB: 20}); err == nil { + t.Fatal("accepted invalid name") + } + if _, err := c.CreateVolume(t.Context(), "data", VolumeCreate{SizeGiB: 0}); err == nil { + t.Fatal("accepted zero size") + } + if err := c.DeleteVolume(t.Context(), "data", `"weak"`); err == nil { + t.Fatal("accepted quoted id") + } + if requests != 0 { + t.Fatalf("invalid input sent %d requests", requests) + } + if _, err := c.GetVolume(t.Context(), "data"); err == nil { + t.Fatal("accepted missing response volume") + } + ctx, cancel := context.WithCancel(t.Context()) + cancel() + if _, err := c.GetVolume(ctx, "data"); !errors.Is(err, context.Canceled) { + t.Fatalf("cancellation = %v", err) + } + if requests != 1 { + t.Fatalf("cancelled request reached server, count %d", requests) + } +} + +func TestManagedVolumeConflictPreservesAllHolders(t *testing.T) { + holders := make([]VolumeHolder, 100) + for i := range holders { + holders[i] = VolumeHolder{SandboxID: fmt.Sprintf("sb_%032d", i), Mode: "ro", State: "claimed"} + } + ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(http.StatusConflict) + _ = json.MarshalWrite(w, map[string]any{"error": "volume held", "code": "volume_busy", "volume": "data", "holders": holders}) + })) + t.Cleanup(ts.Close) + _, err := testClient(t, ts).GetVolume(t.Context(), "data") + var conflict *ManagedVolumeError + if !errors.As(err, &conflict) || conflict.Status != http.StatusConflict || conflict.Code != "volume_busy" { + t.Fatalf("expected typed volume conflict, got %#v", err) + } + if !reflect.DeepEqual(conflict.Holders, holders) { + t.Fatalf("conflict returned %d holders, want all %d with intact fields", len(conflict.Holders), len(holders)) + } +} diff --git a/sdk/go/options.go b/sdk/go/options.go index 20b75401..0d04873a 100644 --- a/sdk/go/options.go +++ b/sdk/go/options.go @@ -39,12 +39,24 @@ type Volume struct { // VolumeInfo describes one visible fleet catalog entry. type VolumeInfo struct { - Name string `json:"name"` - DefaultMount string `json:"default_mount"` - SizeBytes int64 `json:"size_bytes"` - Available bool `json:"available"` - Nodes int `json:"nodes"` - Writable bool `json:"writable,omitzero"` + Kind string `json:"kind,omitempty"` + ID string `json:"id,omitempty"` + Tenant string `json:"tenant,omitempty"` + Node string `json:"node,omitempty"` + State string `json:"state,omitempty"` + CreatedAt time.Time `json:"created_at,omitzero"` + Name string `json:"name"` + DefaultMount string `json:"default_mount"` + SizeBytes int64 `json:"size_bytes"` + Available bool `json:"available"` + Nodes int `json:"nodes"` + Writable bool `json:"writable,omitzero"` +} + +type VolumeHolder struct { + SandboxID string `json:"sandbox_id"` + Mode string `json:"mode"` + State string `json:"state"` } // EnvInject makes the node's egress proxy send a host-only entry's value to Hosts, exact or "*." suffix, as Header or as the Query parameter, where the guest sent a set Placeholder. diff --git a/sdk/python/cocoonsandbox/client.py b/sdk/python/cocoonsandbox/client.py index aa3c8fd1..80904c67 100644 --- a/sdk/python/cocoonsandbox/client.py +++ b/sdk/python/cocoonsandbox/client.py @@ -5,11 +5,13 @@ import http.client import json import queue +import re import ssl import threading import time import urllib.parse from collections.abc import Callable, Iterable, Mapping, Sequence +from datetime import datetime from typing import Any, TypeVar, cast from .checkpoint import Checkpoint @@ -21,6 +23,8 @@ T = TypeVar("T") +_VOLUME_NAME = re.compile(r"^[a-z][a-z0-9_-]{0,19}$") + _PEERS_TIMEOUT = 5.0 _IDLE_CONNS = 8 @@ -133,6 +137,37 @@ def volumes(self) -> list[dict[str, Any]]: reply = self._request(self.addr, "GET", "/v1/volumes", None, "list volumes") return [dict(volume) for volume in reply.get("volumes") or []] + def create_volume(self, name: str, size_gib: int, tenant: str = "") -> dict[str, Any]: + """Creates a raw managed disk or returns it for an identical retry.""" + path = _volume_path(name) + if isinstance(size_gib, bool) or not isinstance(size_gib, int) or not 1 <= size_gib <= 1024: + raise ValueError("volume size_gib must be an integer between 1 and 1024") + body: dict[str, Any] = {"size_gib": size_gib} + if tenant: + body["tenant"] = tenant + reply = self._request(self.addr, "PUT", path, body, "create volume", expected_status=(200, 201)) + return _managed_volume(reply, "create volume") + + def get_volume(self, name: str) -> dict[str, Any]: + """Inspects one managed disk visible to this token.""" + reply = self._request(self.addr, "GET", _volume_path(name), None, "get volume", expected_status=(200,)) + return _managed_volume(reply, "get volume") + + def delete_volume(self, name: str, resource_id: str) -> None: + """Deletes exactly resource_id, including when an earlier response was lost.""" + path = _volume_path(name) + if not resource_id or '"' in resource_id or "\r" in resource_id or "\n" in resource_id: + raise ValueError("volume resource_id must be a nonempty unquoted id") + self._request( + self.addr, + "DELETE", + path, + None, + "delete volume", + headers={"If-Match": f'"{resource_id}"'}, + expected_status=(204,), + ) + def info(self) -> dict[str, Any]: """The node's pool/claim counters, as served by GET /v1/info.""" return self._request(self.addr, "GET", "/v1/info", None, "info") @@ -198,11 +233,13 @@ def _request( bearer: str = "", timeout: float = 0.0, deadline: float | None = None, + headers: dict[str, str] | None = None, + expected_status: tuple[int, ...] | None = None, ) -> dict[str, Any]: timeout = remaining_timeout(timeout or self.timeout, deadline, verb) data = json.dumps(body).encode() if body is not None else None url = urllib.parse.urlsplit(addr + path if "://" in addr else f"{self._scheme}://{addr}{path}") - headers = {"Content-Type": "application/json"} if data is not None else {} + headers = {**(headers or {}), **({"Content-Type": "application/json"} if data is not None else {})} token = bearer or self.api_token if token: headers["Authorization"] = f"Bearer {token}" @@ -218,6 +255,8 @@ def _request( except (ValueError, TypeError): reply = None raise APIError(verb, status, _error_message(raw), reply=reply if isinstance(reply, dict) else None) + if expected_status is not None and status not in expected_status: + raise APIError(verb, status, "unexpected response status") if not raw: return {} try: @@ -347,3 +386,39 @@ def run(addr: str) -> None: return cast(T, value) last_error = error raise cast(Exception, last_error) + + +def _volume_path(name: str) -> str: + if not _VOLUME_NAME.fullmatch(name) or name.startswith("cocoon-"): + raise ValueError(f"invalid volume name {name!r}") + return "/v1/volumes/" + urllib.parse.quote(name, safe="") + + +def _managed_volume(reply: dict[str, Any], verb: str) -> dict[str, Any]: + if not isinstance(reply, dict): + raise APIError(verb, 0, "reply without managed volume fields") + volume = require_field(reply, "volume", verb) + required = ("id", "name", "tenant", "node", "state", "created_at") + if ( + not isinstance(volume, dict) + or volume.get("kind") != "managed" + or any(not isinstance(volume.get(key), str) or not volume[key] for key in required) + ): + raise APIError(verb, 0, "reply without managed volume fields") + size = volume.get("size_bytes") + if isinstance(size, bool) or not isinstance(size, int) or size < 1: + raise APIError(verb, 0, "reply with invalid managed volume capacity") + timestamp = re.fullmatch( + r"(\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2})(?:\.\d{1,9})?(Z|[+-]\d{2}:\d{2})", + volume["created_at"], + ) + if timestamp is None: + raise APIError(verb, 0, "reply with invalid managed volume timestamp") + offset = timestamp[2] + if offset != "Z" and (int(offset[1:3]) > 23 or int(offset[4:6]) > 59): + raise APIError(verb, 0, "reply with invalid managed volume timestamp") + try: + datetime.fromisoformat(timestamp[1] + timestamp[2].replace("Z", "+00:00")) + except ValueError as exc: + raise APIError(verb, 0, "reply with invalid managed volume timestamp") from exc + return dict(volume) diff --git a/sdk/python/cocoonsandbox/errors.py b/sdk/python/cocoonsandbox/errors.py index 4e1ed4dc..314b2fe5 100644 --- a/sdk/python/cocoonsandbox/errors.py +++ b/sdk/python/cocoonsandbox/errors.py @@ -25,6 +25,14 @@ def __init__(self, verb: str, status: int, message: str, *, reply: dict[str, Any self.status = status self.message = message self.reply = reply + self.code = (reply or {}).get("code", "") + self.volume = (reply or {}).get("volume", "") + holders = (reply or {}).get("holders") + self.holders = [ + {key: entry[key] for key in ("sandbox_id", "mode", "state") if isinstance(entry.get(key), str)} + for entry in (holders if isinstance(holders, list) else []) + if isinstance(entry, dict) + ] class VolumeError(APIError): diff --git a/sdk/python/tests/test_managed_volumes.py b/sdk/python/tests/test_managed_volumes.py new file mode 100644 index 00000000..198d367e --- /dev/null +++ b/sdk/python/tests/test_managed_volumes.py @@ -0,0 +1,185 @@ +import threading +from http.server import HTTPServer + +import pytest +from conftest import FakeNode + +from cocoonsandbox import APIError, Client + + +class VolumeNode(FakeNode): + def do_PUT(self): + self._dispatch("PUT") + + +@pytest.fixture +def node(): + server = HTTPServer(("127.0.0.1", 0), VolumeNode) + FakeNode.routes = {} + threading.Thread(target=server.serve_forever, daemon=True).start() + yield f"127.0.0.1:{server.server_port}" + server.shutdown() + + +def test_create_inspect_and_delete_use_immutable_identity(node): + volume = { + "name": "data", + "kind": "managed", + "id": "vol_123", + "tenant": "t1", + "node": "node-1", + "state": "ready", + "created_at": "2026-10-19T00:00:00Z", + "size_bytes": 20 << 30, + "available": True, + "holders": [{"sandbox_id": "sb_1", "mode": "rw", "state": "attached"}], + } + seen = [] + + def create(body, path): + seen.append(("PUT", body, dict(FakeNode.last_headers))) + return (201 if len(seen) == 1 else 200), {"volume": volume} + + def inspect(body, path): + seen.append(("GET", body, dict(FakeNode.last_headers))) + return 200, {"volume": volume} + + def delete(body, path): + seen.append(("DELETE", body, dict(FakeNode.last_headers))) + return 204, {} + + path = "/v1/volumes/data" + FakeNode.routes[("PUT", path)] = create + FakeNode.routes[("GET", path)] = inspect + FakeNode.routes[("DELETE", path)] = delete + client = Client(node, api_token="token") + assert client.create_volume("data", 20, tenant="t1") == volume + assert client.create_volume("data", 20, tenant="t1") == volume + assert client.get_volume("data") == volume + client.delete_volume("data", volume["id"]) + client.delete_volume("data", volume["id"]) + assert [request[0] for request in seen] == ["PUT", "PUT", "GET", "DELETE", "DELETE"] + assert all(request[2]["Authorization"] == "Bearer token" for request in seen) + assert seen[0][1] == {"size_gib": 20, "tenant": "t1"} + assert seen[3][2]["If-Match"] == seen[4][2]["If-Match"] == '"vol_123"' + + +@pytest.mark.parametrize("status", [400, 401, 403, 404, 409, 412, 428, 503]) +def test_management_error_keeps_safe_details(node, status): + FakeNode.routes[("GET", "/v1/volumes/data")] = lambda body, path: ( + status, + { + "error": "volume unavailable", + "code": "volume_busy", + "volume": "data", + "holders": [{"sandbox_id": "sb_1", "mode": "rw", "state": "retiring", "host_path": "/secret"}], + }, + ) + with pytest.raises(APIError) as raised: + Client(node).get_volume("data") + error = raised.value + assert error.status == status and error.message == "volume unavailable" + assert error.code == "volume_busy" and error.volume == "data" + assert error.holders == [{"sandbox_id": "sb_1", "mode": "rw", "state": "retiring"}] + assert "/secret" not in str(error) and "/secret" not in repr(error.holders) + + +def test_invalid_inputs_and_incomplete_reply(node): + calls = [] + + def incomplete(body, path): + calls.append(body) + return 200, {} + + FakeNode.routes[("GET", "/v1/volumes/data")] = incomplete + client = Client(node) + with pytest.raises(ValueError): + client.create_volume("bad/name", 20) + with pytest.raises(ValueError): + client.create_volume("data", 0) + with pytest.raises(ValueError): + client.create_volume("data", True) + with pytest.raises(ValueError): + client.delete_volume("data", '"weak"') + assert calls == [] + with pytest.raises(APIError, match="reply without volume"): + client.get_volume("data") + assert calls == [{}] + + +def test_management_timeout(black_hole): + with pytest.raises(APIError) as raised: + Client(black_hole, timeout=0.05).get_volume("data") + assert raised.value.status == 0 + + +@pytest.mark.parametrize("status", [200, 202, 302]) +def test_delete_requires_confirmed_no_content(node, status): + FakeNode.routes[("DELETE", "/v1/volumes/data")] = lambda body, path: (status, {}) + with pytest.raises(APIError) as raised: + Client(node).delete_volume("data", "vol_123") + assert raised.value.status == status + + +@pytest.mark.parametrize( + "field,value", + [ + ("id", 123), + ("created_at", "not-a-time"), + ("created_at", "2026-10-19T00:00:00+00:99"), + ("created_at", "2026-10-19T00:00:00+01:60"), + ("size_bytes", -1), + ("size_bytes", True), + ("size_bytes", 1.5), + ], +) +def test_managed_response_rejects_invalid_field_types(node, field, value): + volume = { + "name": "data", + "kind": "managed", + "id": "vol_123", + "tenant": "t1", + "node": "node-1", + "state": "ready", + "created_at": "2026-10-19T00:00:00Z", + "size_bytes": 20 << 30, + "available": True, + } + volume[field] = value + FakeNode.routes[("GET", "/v1/volumes/data")] = lambda body, path: (200, {"volume": volume}) + with pytest.raises(APIError, match="managed volume"): + Client(node).get_volume("data") + + +@pytest.mark.parametrize( + "timestamp", + [ + "2026-10-19T00:00:00.1Z", + "2026-10-19T00:00:00.123456789Z", + "2026-10-19T00:00:00.123456789+05:30", + ], +) +def test_managed_response_accepts_server_fractional_timestamp(node, timestamp): + volume = { + "name": "data", + "kind": "managed", + "id": "vol_123", + "tenant": "t1", + "node": "node-1", + "state": "ready", + "created_at": timestamp, + "size_bytes": 20 << 30, + "available": True, + } + FakeNode.routes[("GET", "/v1/volumes/data")] = lambda body, path: (200, {"volume": volume}) + assert Client(node).get_volume("data")["created_at"] == timestamp + + +@pytest.mark.parametrize("holders", [1, "invalid", {}, None]) +def test_malformed_holder_container_preserves_api_error(node, holders): + FakeNode.routes[("GET", "/v1/volumes/data")] = lambda body, path: (409, {"error": "busy", "holders": holders}) + with pytest.raises(APIError) as raised: + Client(node).get_volume("data") + assert raised.value.status == 409 and raised.value.message == "busy" + assert raised.value.holders == [] + assert raised.value.reply == {"error": "busy", "holders": holders} From a6ed90d62b1860e05adac0cde28b6f36381bb7c7 Mon Sep 17 00:00:00 2001 From: minatoaquaMK2 Date: Mon, 19 Oct 2026 09:26:35 +0000 Subject: [PATCH 02/75] fix(sandboxd): satisfy managed volume lint checks Preserve registry checks and native device identities across Linux and Darwin. Use the pinned CI toolchain to verify the full lint matrix. --- e2e/managed_volume_privacy_test.go | 8 +- e2e/wire_parity_test.go | 30 +-- sandboxd/config/config.go | 6 +- sandboxd/config/managed_volume_test.go | 30 +-- sandboxd/pool/claim.go | 6 +- sandboxd/pool/managed_attachment.go | 59 +++--- sandboxd/pool/managed_integration_test.go | 56 +++--- .../pool/managed_recovery_regression_test.go | 50 ++--- .../pool/managed_startup_boundary_test.go | 8 +- sandboxd/pool/managed_volume.go | 16 +- sandboxd/pool/managed_volume_test.go | 87 ++++---- sandboxd/pool/pool.go | 3 +- sandboxd/pool/reconcile.go | 35 ++-- sandboxd/pool/volume.go | 38 ++-- sandboxd/pool/volume_device_test.go | 31 +++ sandboxd/pool/volume_registry.go | 187 ++++++++++-------- sdk/go/managed_volume.go | 6 +- sdk/go/managed_volume_test.go | 12 +- 18 files changed, 385 insertions(+), 283 deletions(-) create mode 100644 sandboxd/pool/volume_device_test.go diff --git a/e2e/managed_volume_privacy_test.go b/e2e/managed_volume_privacy_test.go index 530ed292..3c5e3210 100644 --- a/e2e/managed_volume_privacy_test.go +++ b/e2e/managed_volume_privacy_test.go @@ -47,8 +47,10 @@ func TestLiveAttachManagedVolumePrivacy(t *testing.T) { type catalogPeerPlacer struct{ edgePlacer } func (*catalogPeerPlacer) VolumeCandidates(string, []string) []string { return []string{"peer:7777"} } -func (*catalogPeerPlacer) VolumeOwners([]string) []string { return []string{"peer:7777"} } -func (*catalogPeerPlacer) VolumeHolders() map[string]int { return map[string]int{"peer-only": 1} } + +func (*catalogPeerPlacer) VolumeOwners([]string) []string { return []string{"peer:7777"} } + +func (*catalogPeerPlacer) VolumeHolders() map[string]int { return map[string]int{"peer-only": 1} } func TestRootPeerCatalogRedirectWithManagedStore(t *testing.T) { for _, enabled := range []bool{false, true} { @@ -85,7 +87,7 @@ func TestStaticVolumePrivacyWithManagedStore(t *testing.T) { t.Run(fmt.Sprint(enabled), func(t *testing.T) { dir := t.TempDir() path := filepath.Join(dir, "catalog.raw") - if err := os.WriteFile(path, []byte("catalog"), 0600); err != nil { + if err := os.WriteFile(path, []byte("catalog"), 0o600); err != nil { t.Fatal(err) } cfg := &config.Config{Volumes: []config.VolumeSpec{{Name: "private-static", Path: path, Tenants: []string{"t1"}}}} diff --git a/e2e/wire_parity_test.go b/e2e/wire_parity_test.go index 7ab3937b..1dc2eb15 100644 --- a/e2e/wire_parity_test.go +++ b/e2e/wire_parity_test.go @@ -135,22 +135,24 @@ func parseSDKFields(t *testing.T) map[string]map[string]string { if !ok { return true } - if st, ok := spec.Type.(*ast.StructType); ok { - out[spec.Name.Name] = map[string]string{} - for _, f := range st.Fields.List { - if len(f.Names) == 0 { - if id, ok := f.Type.(*ast.Ident); ok { - embedded[spec.Name.Name] = append(embedded[spec.Name.Name], id.Name) - } - } - if f.Tag == nil { - continue - } - tag, _ := strconv.Unquote(f.Tag.Value) - if name, opts := splitJSONTag(reflect.StructTag(tag).Get("json")); name != "" && name != "-" { - out[spec.Name.Name][name] = opts + st, ok := spec.Type.(*ast.StructType) + if !ok { + return true + } + out[spec.Name.Name] = map[string]string{} + for _, f := range st.Fields.List { + if len(f.Names) == 0 { + if id, ok := f.Type.(*ast.Ident); ok { + embedded[spec.Name.Name] = append(embedded[spec.Name.Name], id.Name) } } + if f.Tag == nil { + continue + } + tag, _ := strconv.Unquote(f.Tag.Value) + if name, opts := splitJSONTag(reflect.StructTag(tag).Get("json")); name != "" && name != "-" { + out[spec.Name.Name][name] = opts + } } return true }) diff --git a/sandboxd/config/config.go b/sandboxd/config/config.go index fee0e129..63e9d4aa 100644 --- a/sandboxd/config/config.go +++ b/sandboxd/config/config.go @@ -724,9 +724,9 @@ func resolveExistingAncestors(path string) (string, error) { for { _, err := os.Lstat(path) if err == nil { - resolved, err := filepath.EvalSymlinks(path) - if err != nil { - return "", err + resolved, resolveErr := filepath.EvalSymlinks(path) + if resolveErr != nil { + return "", resolveErr } return filepath.Join(resolved, suffix), nil } diff --git a/sandboxd/config/managed_volume_test.go b/sandboxd/config/managed_volume_test.go index aec306e1..5bd02a80 100644 --- a/sandboxd/config/managed_volume_test.go +++ b/sandboxd/config/managed_volume_test.go @@ -36,12 +36,12 @@ func TestManagedVolumeStoreExcludesStartupCleanupPaths(t *testing.T) { func TestManagedVolumeStoreResolvesDataDirectoryAliases(t *testing.T) { root := t.TempDir() - real := filepath.Join(root, "real") + realDir := filepath.Join(root, "real") alias := filepath.Join(root, "alias") - if err := os.Mkdir(real, 0700); err != nil { + if err := os.Mkdir(realDir, 0o700); err != nil { t.Fatal(err) } - if err := os.Symlink(real, alias); err != nil { + if err := os.Symlink(realDir, alias); err != nil { t.Fatal(err) } t.Chdir(root) @@ -49,17 +49,17 @@ func TestManagedVolumeStoreResolvesDataDirectoryAliases(t *testing.T) { name, data, managed string allow bool }{ - {"alias-readers", alias, filepath.Join(real, "readers", "managed"), false}, - {"alias-goldens", alias, filepath.Join(real, "goldens"), false}, - {"alias-fork", alias, filepath.Join(real, "fork-persistent"), false}, - {"alias-new-suffix", filepath.Join(alias, "new", "data"), filepath.Join(real, "new", "data", "readers", "managed"), false}, - {"alias-safe", alias, filepath.Join(real, "managed"), true}, - {"relative", "real", filepath.Join(real, "readers", "managed"), false}, - {"relative-safe", "real", filepath.Join(real, "readers-safe"), true}, + {"alias-readers", alias, filepath.Join(realDir, "readers", "managed"), false}, + {"alias-goldens", alias, filepath.Join(realDir, "goldens"), false}, + {"alias-fork", alias, filepath.Join(realDir, "fork-persistent"), false}, + {"alias-new-suffix", filepath.Join(alias, "new", "data"), filepath.Join(realDir, "new", "data", "readers", "managed"), false}, + {"alias-safe", alias, filepath.Join(realDir, "managed"), true}, + {"relative", "real", filepath.Join(realDir, "readers", "managed"), false}, + {"relative-safe", "real", filepath.Join(realDir, "readers-safe"), true}, {"empty", "", filepath.Join(root, "fork-store"), false}, {"empty-safe", "", filepath.Join(root, "managed"), true}, - {"ancestor", filepath.Join(real, "data"), real, true}, - {"same-root", real, real, true}, + {"ancestor", filepath.Join(realDir, "data"), realDir, true}, + {"same-root", realDir, realDir, true}, } { t.Run(tc.name, func(t *testing.T) { cfg := Config{DataDir: tc.data, ManagedVolumeDir: tc.managed, APIToken: "root"} @@ -69,7 +69,7 @@ func TestManagedVolumeStoreResolvesDataDirectoryAliases(t *testing.T) { } }) } - if _, err := os.Stat(filepath.Join(real, "new")); !os.IsNotExist(err) { + if _, err := os.Stat(filepath.Join(realDir, "new")); !os.IsNotExist(err) { t.Fatalf("validation created data directories: %v", err) } } @@ -81,7 +81,7 @@ func TestManagedVolumeStoreRejectsCleanupDirectoryAliases(t *testing.T) { data := filepath.Join(root, "data") external := filepath.Join(root, "external") for _, dir := range []string{data, external} { - if err := os.Mkdir(dir, 0700); err != nil { + if err := os.Mkdir(dir, 0o700); err != nil { t.Fatal(err) } } @@ -119,7 +119,7 @@ func TestManagedVolumeStoreProtectsBackingDirectoryFromCleanupAliases(t *testing managed := filepath.Join(root, "managed") aliasTarget := filepath.Join(managed, target) for _, dir := range []string{data, aliasTarget} { - if err := os.MkdirAll(dir, 0700); err != nil { + if err := os.MkdirAll(dir, 0o700); err != nil { t.Fatal(err) } } diff --git a/sandboxd/pool/claim.go b/sandboxd/pool/claim.go index d7f64d8d..deefbf81 100644 --- a/sandboxd/pool/claim.go +++ b/sandboxd/pool/claim.go @@ -779,9 +779,9 @@ func (m *Manager) admitOptions(ctx context.Context, key types.PoolKey, o ClaimOp if err := m.checkEnv(o.Env); err != nil { return nil, nil, err } - volumeSpecs, err := m.resolveVolumes(ctx, o.Tenant, o.Volumes) - if err != nil { - return nil, nil, err + volumeSpecs, resolveErr := m.resolveVolumes(ctx, o.Tenant, o.Volumes) + if resolveErr != nil { + return nil, nil, resolveErr } for _, v := range volumeSpecs { if v.managedID != "" { diff --git a/sandboxd/pool/managed_attachment.go b/sandboxd/pool/managed_attachment.go index b75f4f84..df82e5fe 100644 --- a/sandboxd/pool/managed_attachment.go +++ b/sandboxd/pool/managed_attachment.go @@ -4,6 +4,7 @@ import ( "context" "errors" "fmt" + "io/fs" "os" "strconv" "strings" @@ -20,7 +21,8 @@ func readProcessIdentityAt(procDir string, pid int) (processIdentity, error) { if pid <= 0 { return processIdentity{}, errors.New("VMM PID unavailable") } - boot, err := os.ReadFile(procDir + "/sys/kernel/random/boot_id") + proc := os.DirFS(procDir) + boot, err := fs.ReadFile(proc, "sys/kernel/random/boot_id") if err != nil { // Missing boot metadata cannot prove that the PID is gone. return processIdentity{}, fmt.Errorf("read VMM boot identity: %v", err) @@ -29,7 +31,7 @@ func readProcessIdentityAt(procDir string, pid int) (processIdentity, error) { if bootID == "" { return processIdentity{}, errors.New("empty VMM boot identity") } - stat, err := os.ReadFile(procDir + "/" + strconv.Itoa(pid) + "/stat") + stat, err := fs.ReadFile(proc, strconv.Itoa(pid)+"/stat") if err != nil { return processIdentity{}, err } @@ -74,29 +76,11 @@ func (m *Manager) admitManaged(tenant string, volumes []resolvedVolume) ([]types } h := volumeHold{SandboxID: "sb_" + randHex(8), Tenant: tenant, State: holdAttaching} for _, volume := range volumes { - if volume.managedID != "" { - v, ok := r.data.Volumes[volume.applied.Name] - if !ok || v.ID != volume.managedID || (tenant != "" && v.Tenant != tenant) { - return nil, managedNotFound() - } - if v.State != volumeReady || r.verify(v) != nil { - return nil, managedStorageError() - } - for _, other := range r.data.Holds { - for _, ref := range other.Refs { - if ref.ID == v.ID && (ref.Volume.RW() || volume.applied.RW()) { - return nil, &VolumeError{Status: 409, Code: "volume_busy", Message: "volume is held by a sandbox", Volume: v.Name, Holders: r.holders(v.ID)} - } - } - } - h.Refs = append(h.Refs, volumeRef{Volume: volume.applied, ID: v.ID, Path: r.path(v.ID), Backing: v.Backing}) - } else { - identity, err := catalogBacking(volume.disk.Path) - if err != nil { - return nil, ErrVolumeUnavailable - } - h.Refs = append(h.Refs, volumeRef{Volume: volume.applied, Path: volume.disk.Path, Backing: identity}) + ref, err := r.referenceForAdmission(tenant, volume) + if err != nil { + return nil, err } + h.Refs = append(h.Refs, ref) } if err := m.reserveVolumes(applied); err != nil { return nil, err @@ -115,6 +99,31 @@ func (m *Manager) admitManaged(tenant string, volumes []resolvedVolume) ([]types return applied, nil } +func (r *volumeRegistry) referenceForAdmission(tenant string, volume resolvedVolume) (volumeRef, error) { + if volume.managedID == "" { + identity, err := catalogBacking(volume.disk.Path) + if err != nil { + return volumeRef{}, ErrVolumeUnavailable + } + return volumeRef{Volume: volume.applied, Path: volume.disk.Path, Backing: identity}, nil + } + v, ok := r.data.Volumes[volume.applied.Name] + if !ok || v.ID != volume.managedID || (tenant != "" && v.Tenant != tenant) { + return volumeRef{}, managedNotFound() + } + if v.State != volumeReady || r.verify(v) != nil { + return volumeRef{}, managedStorageError() + } + for _, other := range r.data.Holds { + for _, ref := range other.Refs { + if ref.ID == v.ID && (ref.Volume.RW() || volume.applied.RW()) { + return volumeRef{}, &VolumeError{Status: 409, Code: volumeBusyCode, Message: "volume is held by a sandbox", Volume: v.Name, Holders: r.holders(v.ID)} + } + } + } + return volumeRef{Volume: volume.applied, ID: v.ID, Path: r.path(v.ID), Backing: v.Backing}, nil +} + func (m *Manager) releaseUnbound(volumes []resolvedVolume, reserved []types.Volume) { if len(reserved) == 0 { return @@ -379,5 +388,5 @@ func catalogBacking(path string) (backingIdentity, error) { return backingIdentity{}, errors.New("catalog backing is not a regular file") } raw := st.Sys().(*syscall.Stat_t) - return backingIdentity{Device: uint64(raw.Dev), Inode: raw.Ino, Size: st.Size()}, nil + return backingIdentity{Device: backingDeviceID(raw.Dev), Inode: raw.Ino, Size: st.Size()}, nil } diff --git a/sandboxd/pool/managed_integration_test.go b/sandboxd/pool/managed_integration_test.go index f5c8c0a2..ef0c4bb9 100644 --- a/sandboxd/pool/managed_integration_test.go +++ b/sandboxd/pool/managed_integration_test.go @@ -97,7 +97,7 @@ func TestManagedRejectsCatalogReplacementBeforeAttach(t *testing.T) { if err := os.Symlink(m.managed.path(volumeID(v.ID)), path); err != nil { t.Fatal(err) } - } else if err := os.WriteFile(path, []byte("replacement"), 0600); err != nil { + } else if err := os.WriteFile(path, []byte("replacement"), 0o600); err != nil { t.Fatal(err) } for _, spec := range specs { @@ -124,16 +124,16 @@ func TestManagedReleaseDoesNotClearReplacementCatalogDirtyMarker(t *testing.T) { e := newManagedEngine() m := openManagedTest(t, cfg, e) createManagedTest(t, m, "data") - sb, err := m.ClaimProvision(t.Context(), testKey, ClaimOptions{Tenant: "t1", Volumes: []types.Volume{ + sb, claimErr := m.ClaimProvision(t.Context(), testKey, ClaimOptions{Tenant: "t1", Volumes: []types.Volume{ {Name: "data", Mode: "rw"}, {Name: "catalog", Mode: "rw"}, }}) - if err != nil { - t.Fatal(err) + if claimErr != nil { + t.Fatal(claimErr) } if err := os.Rename(path, path+".old"); err != nil { t.Fatal(err) } - if err := os.WriteFile(path, []byte("replacement"), 0600); err != nil { + if err := os.WriteFile(path, []byte("replacement"), 0o600); err != nil { t.Fatal(err) } if err := m.Release(t.Context(), sb.ID, Cred{Operator: true}); err != nil { @@ -142,9 +142,9 @@ func TestManagedReleaseDoesNotClearReplacementCatalogDirtyMarker(t *testing.T) { if !volumeDirty(path) { t.Fatal("release cleared a replacement backing's dirty marker") } - _, err = m.ClaimProvision(t.Context(), testKey, ClaimOptions{Tenant: "t1", Volumes: []types.Volume{{Name: "catalog"}}}) - if !errors.Is(err, ErrVolumeNeedsRecovery) { - t.Fatalf("replacement reader admitted: %v", err) + _, claimErr = m.ClaimProvision(t.Context(), testKey, ClaimOptions{Tenant: "t1", Volumes: []types.Volume{{Name: "catalog"}}}) + if !errors.Is(claimErr, ErrVolumeNeedsRecovery) { + t.Fatalf("replacement reader admitted: %v", claimErr) } } @@ -154,21 +154,21 @@ func TestManagedSettledRetryDrainsPreviouslyUncertainCatalogRef(t *testing.T) { e := newManagedEngine() m := openManagedTest(t, cfg, e) createManagedTest(t, m, "data") - sb, err := m.ClaimProvision(t.Context(), testKey, managedOptions("data")) - if err != nil { - t.Fatal(err) + sb, claimErr := m.ClaimProvision(t.Context(), testKey, managedOptions("data")) + if claimErr != nil { + t.Fatal(claimErr) } e.diskAttachErrFor, e.diskAttachErr = "catalog", errors.New("uncertain attach") - _, err = m.PatchVolumes(t.Context(), sb.ID, "", types.VolumePatch{Add: []types.Volume{{Name: "catalog"}}}) - if err == nil { + _, claimErr = m.PatchVolumes(t.Context(), sb.ID, "", types.VolumePatch{Add: []types.Volume{{Name: "catalog"}}}) + if claimErr == nil { t.Fatal("injected attach succeeded") } e.diskAttachErr, e.diskAttachErrFor = nil, "" - if err := os.Mkdir(m.store.path+".tmp", 0700); err != nil { + if err := os.Mkdir(m.store.path+".tmp", 0o700); err != nil { t.Fatal(err) } - _, err = m.PatchVolumes(t.Context(), sb.ID, "", types.VolumePatch{Remove: []string{"catalog"}}) - if err == nil { + _, claimErr = m.PatchVolumes(t.Context(), sb.ID, "", types.VolumePatch{Remove: []string{"catalog"}}) + if claimErr == nil { t.Fatal("injected claim commit succeeded") } if err := os.Remove(m.store.path + ".tmp"); err != nil { @@ -182,9 +182,9 @@ func TestManagedSettledRetryDrainsPreviouslyUncertainCatalogRef(t *testing.T) { if !ok || len(h.Refs) != 1 { t.Fatalf("settled retry leaked uncertain reference: %+v", h) } - writer, err := m.ClaimProvision(t.Context(), testKey, managedOptions("catalog")) - if err != nil { - t.Fatalf("confirmed detached catalog still busy: %v", err) + writer, claimErr := m.ClaimProvision(t.Context(), testKey, managedOptions("catalog")) + if claimErr != nil { + t.Fatalf("confirmed detached catalog still busy: %v", claimErr) } if err := m.Release(t.Context(), writer.ID, Cred{Operator: true}); err != nil { t.Fatal(err) @@ -209,9 +209,9 @@ func TestManagedReleaseCancelsExistingVolumeRunAndRetainsFullUnion(t *testing.T) cfg := managedConfig(t) cfg.Volumes = []config.VolumeSpec{{Name: "catalog", Path: writeVolumeImage(t, "catalog.img", "catalog"), Writable: true}} e := &managedCancelEngine{managedTestEngine: newManagedEngine(), entered: make(chan struct{})} - m, err := NewManager(t.Context(), cfg, e, testSecrets(t), testBackends(t, cfg)) - if err != nil { - t.Fatal(err) + m, openErr := NewManager(t.Context(), cfg, e, testSecrets(t), testBackends(t, cfg)) + if openErr != nil { + t.Fatal(openErr) } t.Cleanup(func() { m.Close() }) m.managed.process = e.identity @@ -219,9 +219,9 @@ func TestManagedReleaseCancelsExistingVolumeRunAndRetainsFullUnion(t *testing.T) t.Fatal(err) } createManagedTest(t, m, "data") - sb, err := m.ClaimProvision(t.Context(), testKey, managedOptions("data")) - if err != nil { - t.Fatal(err) + sb, openErr := m.ClaimProvision(t.Context(), testKey, managedOptions("data")) + if openErr != nil { + t.Fatal(openErr) } ctx, cancel := context.WithCancel(t.Context()) defer cancel() @@ -247,9 +247,9 @@ func TestManagedReleaseCancelsExistingVolumeRunAndRetainsFullUnion(t *testing.T) if !ok || h.State != holdRetiring || len(h.Refs) != 2 { t.Fatalf("retirement lost possible device references: %+v", h) } - _, err = m.ClaimProvision(t.Context(), testKey, managedOptions("catalog")) - if !errors.Is(err, ErrVolumeBusy) { - t.Fatalf("uncertain catalog writer admitted: %v", err) + _, openErr = m.ClaimProvision(t.Context(), testKey, managedOptions("catalog")) + if !errors.Is(openErr, ErrVolumeBusy) { + t.Fatalf("uncertain catalog writer admitted: %v", openErr) } e.falseRemove.Store(false) m.retryRemovals(t.Context()).Wait() diff --git a/sandboxd/pool/managed_recovery_regression_test.go b/sandboxd/pool/managed_recovery_regression_test.go index b44a2617..8bc9d66d 100644 --- a/sandboxd/pool/managed_recovery_regression_test.go +++ b/sandboxd/pool/managed_recovery_regression_test.go @@ -27,9 +27,9 @@ func TestManagedQuarantineRemovalSurvivesFailedRetirementRestart(t *testing.T) { e := newManagedEngine() m := openManagedTest(t, cfg, e) createManagedTest(t, m, "data") - sb, err := m.ClaimProvision(t.Context(), testKey, managedOptions("data")) - if err != nil { - t.Fatal(err) + sb, claimErr := m.ClaimProvision(t.Context(), testKey, managedOptions("data")) + if claimErr != nil { + t.Fatal(claimErr) } sb.Key.Net = types.NetEgress if err := m.store.commit(m.store.set(sb)); err != nil { @@ -55,16 +55,16 @@ func TestManagedQuarantineRemovalSurvivesFailedRetirementRestart(t *testing.T) { if err := m.Reconcile(t.Context()); err != nil { t.Fatal(err) } - _, err = m.ClaimProvision(t.Context(), testKey, managedOptions("data")) - requireVolumeStatus(t, err, 409) + _, claimErr = m.ClaimProvision(t.Context(), testKey, managedOptions("data")) + requireVolumeStatus(t, claimErr, 409) e.falseRemove.Store(false) m.retryRemovals(t.Context()).Wait() if _, present, err := e.Inspect(t.Context(), sb.VMName); err != nil || present { t.Fatalf("quarantined VM survived removal retry after restart: present=%v err=%v", present, err) } - replacement, err := m.ClaimProvision(t.Context(), testKey, managedOptions("data")) - if err != nil { - t.Fatalf("confirmed removal did not free managed disk: %v", err) + replacement, claimErr := m.ClaimProvision(t.Context(), testKey, managedOptions("data")) + if claimErr != nil { + t.Fatalf("confirmed removal did not free managed disk: %v", claimErr) } if err := m.Release(t.Context(), replacement.ID, Cred{Operator: true}); err != nil { t.Fatal(err) @@ -79,9 +79,9 @@ func TestManagedOrdinaryFailedClaimRetainsOwnershipOnRestart(t *testing.T) { e := newManagedEngine() m := openManagedTest(t, cfg, e) createManagedTest(t, m, "data") - sb, err := m.ClaimProvision(t.Context(), testKey, managedOptions("data")) - if err != nil { - t.Fatal(err) + sb, claimErr := m.ClaimProvision(t.Context(), testKey, managedOptions("data")) + if claimErr != nil { + t.Fatal(claimErr) } sb.Failed = "ordinary failure requiring explicit release" if err := m.store.commit(m.store.set(sb)); err != nil { @@ -98,8 +98,8 @@ func TestManagedOrdinaryFailedClaimRetainsOwnershipOnRestart(t *testing.T) { if _, present, err := e.Inspect(t.Context(), sb.VMName); err != nil || !present { t.Fatalf("ordinary failed claim removed: %v %v", present, err) } - _, err = m.ClaimProvision(t.Context(), testKey, managedOptions("data")) - requireVolumeStatus(t, err, 409) + _, claimErr = m.ClaimProvision(t.Context(), testKey, managedOptions("data")) + requireVolumeStatus(t, claimErr, 409) if err := m.Release(t.Context(), sb.ID, Cred{Operator: true}); err != nil { t.Fatal(err) } @@ -138,16 +138,16 @@ func TestManagedStartupCleansDeadOrphansAndRefusesUnknownProcess(t *testing.T) { m.managed.process = readProcessIdentity } e.falseRemove.Store(tc.ineffectiveRemove) - err := m.Reconcile(t.Context()) + reconcileErr := m.Reconcile(t.Context()) if tc.allow { - if err != nil { - t.Fatalf("dead orphan blocked startup: %v", err) + if reconcileErr != nil { + t.Fatalf("dead orphan blocked startup: %v", reconcileErr) } if _, present, err := e.Inspect(t.Context(), name); err != nil || present { t.Fatalf("dead orphan survived cleanup: %v %v", present, err) } } else { - requireVolumeStatus(t, err, 503) + requireVolumeStatus(t, reconcileErr, 503) } if tc.processError == os.ErrPermission || (!tc.stopped && tc.pid == 0) { e.mu.Lock() @@ -188,9 +188,9 @@ func TestManagedStorePreservesBackingWhenCleanupAliasesDisks(t *testing.T) { } v := createManagedTest(t, m, "data") backing := m.managed.path(volumeID(v.ID)) - f, err := os.OpenFile(backing, os.O_WRONLY, 0) - if err != nil { - t.Fatal(err) + f, openErr := os.OpenFile(backing, os.O_WRONLY, 0) + if openErr != nil { + t.Fatal(openErr) } _, writeErr := f.WriteAt([]byte("preserved"), 4096) if err := errors.Join(writeErr, f.Close(), m.Close()); err != nil { @@ -203,17 +203,17 @@ func TestManagedStorePreservesBackingWhenCleanupAliasesDisks(t *testing.T) { if err := os.Symlink(filepath.Join(cfg.ManagedVolumeDir, "disks"), cleanup); err != nil { t.Fatal(err) } - next, err := NewManager(t.Context(), cfg, e, testSecrets(t), testBackends(t, cfg)) - if err == nil { + next, openErr := NewManager(t.Context(), cfg, e, testSecrets(t), testBackends(t, cfg)) + if openErr == nil { t.Error("constructor accepted cleanup alias into managed backing directory") if err := next.Reconcile(t.Context()); err != nil { t.Errorf("reconcile: %v", err) } next.Close() } - f, err = os.Open(backing) - if err != nil { - t.Fatalf("managed backing lost: %v", err) + f, openErr = os.Open(backing) + if openErr != nil { + t.Fatalf("managed backing lost: %v", openErr) } defer f.Close() marker := make([]byte, len("preserved")) diff --git a/sandboxd/pool/managed_startup_boundary_test.go b/sandboxd/pool/managed_startup_boundary_test.go index c0491740..e438e0f6 100644 --- a/sandboxd/pool/managed_startup_boundary_test.go +++ b/sandboxd/pool/managed_startup_boundary_test.go @@ -75,18 +75,18 @@ func TestManagedDirectoryRequiresTrustedAncestorOwner(t *testing.T) { } { t.Run(tc.name, func(t *testing.T) { parent := filepath.Join(t.TempDir(), "parent") - if err := os.Mkdir(parent, 0700); err != nil { + if err := os.Mkdir(parent, 0o700); err != nil { t.Fatal(err) } - mode := os.FileMode(0755) + mode := os.FileMode(0o755) if tc.sticky { - mode = 0777 | os.ModeSticky + mode = 0o777 | os.ModeSticky } if err := os.Chmod(parent, mode); err != nil { t.Fatal(err) } managed := filepath.Join(parent, "managed") - if err := os.Mkdir(managed, 0700); err != nil { + if err := os.Mkdir(managed, 0o700); err != nil { t.Fatal(err) } if err := os.Chown(parent, tc.owner, tc.owner); err != nil { diff --git a/sandboxd/pool/managed_volume.go b/sandboxd/pool/managed_volume.go index e20ef258..01d4fe8f 100644 --- a/sandboxd/pool/managed_volume.go +++ b/sandboxd/pool/managed_volume.go @@ -16,6 +16,8 @@ import ( // VolumeError carries only caller-safe details. Storage paths and process IDs // remain in the registry and never become API diagnostics. +const volumeBusyCode = "volume_busy" + type VolumeError struct { Status int `json:"-"` Message string `json:"error"` @@ -26,14 +28,17 @@ type VolumeError struct { func (e *VolumeError) Error() string { return e.Message } func (e *VolumeError) Is(target error) bool { - return target == ErrVolumeBusy && e.Code == "volume_busy" + return target == ErrVolumeBusy && e.Code == volumeBusyCode } + func volumeError(status int, code, message string) *VolumeError { return &VolumeError{Status: status, Code: code, Message: message} } + func managedStorageError() *VolumeError { return volumeError(503, "volume_storage_unavailable", "volume storage requires recovery; retry after recovery") } + func managedNotFound() *VolumeError { return volumeError(404, "volume_not_found", "unknown volume") } func (m *Manager) CreateVolume(ctx context.Context, name string, req types.VolumeCreate, tenant string) (types.ManagedVolumeInfo, bool, error) { @@ -124,7 +129,7 @@ func (r *volumeRegistry) completeCreate(v managedVolume) error { return managedStorageError() } raw := st.Sys().(*syscall.Stat_t) - v.Backing = backingIdentity{Device: uint64(raw.Dev), Inode: raw.Ino, Size: st.Size()} + v.Backing = backingIdentity{Device: backingDeviceID(raw.Dev), Inode: raw.Ino, Size: st.Size()} v.State = volumeReady r.data.Volumes[v.Name] = v return r.commit() @@ -190,7 +195,7 @@ func (m *Manager) DeleteVolume(ctx context.Context, name, expected, tenant strin return volumeError(412, "volume_precondition_failed", "volume identity does not match") } if holders := r.holders(v.ID); len(holders) > 0 { - return &VolumeError{Status: 409, Code: "volume_busy", Message: "volume is held by a sandbox", Volume: name, Holders: holders} + return &VolumeError{Status: 409, Code: volumeBusyCode, Message: "volume is held by a sandbox", Volume: name, Holders: holders} } if err := r.verifyDirectories(); err != nil { return managedStorageError() @@ -243,8 +248,9 @@ func (r *volumeRegistry) holders(id volumeID) []types.VolumeHolder { slices.SortFunc(holders, func(a, b types.VolumeHolder) int { return strings.Compare(a.SandboxID, b.SandboxID) }) return holders } + func (r *volumeRegistry) info(v managedVolume) types.ManagedVolumeInfo { - return types.ManagedVolumeInfo{VolumeInfo: types.VolumeInfo{Name: v.Name, DefaultMount: types.DefaultVolumeMount(v.Name), SizeBytes: v.Size, Available: v.State == volumeReady, Nodes: 1, Writable: true, Kind: "managed", ID: string(v.ID), Tenant: v.Tenant, Node: v.Node, State: string(v.State), CreatedAt: v.CreatedAt}, Holders: r.holders(v.ID)} + return types.ManagedVolumeInfo{Name: v.Name, DefaultMount: types.DefaultVolumeMount(v.Name), SizeBytes: v.Size, Available: v.State == volumeReady, Nodes: 1, Writable: true, Kind: "managed", ID: string(v.ID), Tenant: v.Tenant, Node: v.Node, State: string(v.State), CreatedAt: v.CreatedAt, Holders: r.holders(v.ID)} } // ManagedVolumeStatus lets listing fail clearly rather than presenting a corrupt @@ -394,7 +400,7 @@ func (m *Manager) managedCatalogAlias(path string) bool { r.mu.Lock() defer r.mu.Unlock() for _, v := range r.data.Volumes { - if v.Backing.Inode != 0 && identity.Ino == v.Backing.Inode && uint64(identity.Dev) == v.Backing.Device { + if v.Backing.Inode != 0 && identity.Ino == v.Backing.Inode && backingDeviceID(identity.Dev) == v.Backing.Device { return true } if backing, err := os.Lstat(r.path(v.ID)); err == nil && os.SameFile(info, backing) { diff --git a/sandboxd/pool/managed_volume_test.go b/sandboxd/pool/managed_volume_test.go index 715d706a..04c6715a 100644 --- a/sandboxd/pool/managed_volume_test.go +++ b/sandboxd/pool/managed_volume_test.go @@ -21,7 +21,6 @@ import ( ) type managedTestEngine struct { - liveEntered, liveResume chan struct{} *fakeEngine next atomic.Int64 falseRemove atomic.Bool @@ -33,24 +32,28 @@ type managedTestEngine struct { } func newManagedEngine() *managedTestEngine { return &managedTestEngine{fakeEngine: newFakeEngine()} } + func (e *managedTestEngine) RunCold(ctx context.Context, name string, key types.PoolKey, storage string) (types.VMRecord, error) { e.mu.Lock() e.pids[name] = 1000 + int(e.next.Add(1)) e.mu.Unlock() return e.fakeEngine.RunCold(ctx, name, key, storage) } + func (e *managedTestEngine) Remove(ctx context.Context, name string) error { if e.falseRemove.Load() { return nil } return e.fakeEngine.Remove(ctx, name) } + func (e *managedTestEngine) Inspect(ctx context.Context, name string) (types.VMRecord, bool, error) { if e.missingInventory.Load() { return types.VMRecord{}, false, nil } return e.fakeEngine.Inspect(ctx, name) } + func (e *managedTestEngine) DiskAttach(ctx context.Context, name string, spec engine.VolumeSpec) error { if e.attachEntered != nil { e.attachEntered <- struct{}{} @@ -58,6 +61,7 @@ func (e *managedTestEngine) DiskAttach(ctx context.Context, name string, spec en } return e.fakeEngine.DiskAttach(ctx, name, spec) } + func (e *managedTestEngine) identity(pid int) (processIdentity, error) { if e.processUnknown.Load() { return processIdentity{}, os.ErrPermission @@ -73,11 +77,13 @@ func (e *managedTestEngine) identity(pid int) (processIdentity, error) { } return processIdentity{}, os.ErrNotExist } + func managedConfig(t *testing.T) *config.Config { t.Helper() dir := t.TempDir() return &config.Config{DataDir: filepath.Join(dir, "data"), ManagedVolumeDir: filepath.Join(dir, "managed"), APIToken: "root", Pools: []config.PoolSpec{{PoolKey: testKey, Warm: 1}}} } + func openManagedTest(t *testing.T, cfg *config.Config, e *managedTestEngine) *Manager { t.Helper() m, err := NewManager(t.Context(), cfg, e, testSecrets(t), testBackends(t, cfg)) @@ -94,6 +100,7 @@ func openManagedTest(t *testing.T, cfg *config.Config, e *managedTestEngine) *Ma } return m } + func createManagedTest(t *testing.T, m *Manager, name string) types.ManagedVolumeInfo { t.Helper() v, created, err := m.CreateVolume(t.Context(), name, types.VolumeCreate{SizeGiB: 1}, "t1") @@ -102,6 +109,7 @@ func createManagedTest(t *testing.T, m *Manager, name string) types.ManagedVolum } return v } + func managedOptions(names ...string) ClaimOptions { o := ClaimOptions{Tenant: "t1"} for _, name := range names { @@ -109,6 +117,7 @@ func managedOptions(names ...string) ClaimOptions { } return o } + func requireVolumeStatus(t *testing.T, err error, status int) *VolumeError { t.Helper() var v *VolumeError @@ -131,7 +140,7 @@ func TestManagedCreateReplaySparseOwnershipAndDeleteIdentity(t *testing.T) { if st.Size() != 1<<30 || st.Sys().(*syscall.Stat_t).Blocks*512 >= st.Size() { t.Fatal("backing is not a sparse GiB raw image") } - f, err := os.OpenFile(path, os.O_WRONLY, 0600) + f, err := os.OpenFile(path, os.O_WRONLY, 0o600) if err != nil { t.Fatal(err) } @@ -195,12 +204,12 @@ func TestManagedConcurrentCreateAndWriterAdmission(t *testing.T) { ids := make(chan string, 12) for range 12 { wg.Go(func() { - v, new, err := m.CreateVolume(t.Context(), "race", types.VolumeCreate{SizeGiB: 1}, "t1") + v, wasCreated, err := m.CreateVolume(t.Context(), "race", types.VolumeCreate{SizeGiB: 1}, "t1") if err != nil { t.Error(err) return } - if new { + if wasCreated { created.Add(1) } ids <- v.ID @@ -261,9 +270,9 @@ func TestManagedRetirementSurvivesRestartAndStaleClaim(t *testing.T) { e := newManagedEngine() m := openManagedTest(t, cfg, e) v := createManagedTest(t, m, "data") - sb, err := m.ClaimProvision(t.Context(), testKey, managedOptions("data")) - if err != nil { - t.Fatal(err) + sb, claimErr := m.ClaimProvision(t.Context(), testKey, managedOptions("data")) + if claimErr != nil { + t.Fatal(claimErr) } e.falseRemove.Store(true) if err := m.retireManaged(sb.ID); err != nil { @@ -277,8 +286,8 @@ func TestManagedRetirementSurvivesRestartAndStaleClaim(t *testing.T) { if _, ok := m.Sandbox(sb.ID); ok { t.Fatal("retiring stale claim was adopted") } - _, err = m.ClaimProvision(t.Context(), testKey, managedOptions("data")) - busy := requireVolumeStatus(t, err, 409) + _, claimErr = m.ClaimProvision(t.Context(), testKey, managedOptions("data")) + busy := requireVolumeStatus(t, claimErr, 409) if busy.Holders[0].State != "retiring" || busy.Holders[0].SandboxID != sb.ID { t.Fatalf("holder=%+v", busy) } @@ -360,10 +369,12 @@ func TestManagedPreclaimCrashRetainsOwnership(t *testing.T) { if err := m.bindManaged(t.Context(), sb, specs); err != nil { t.Fatal(err) } + var applyErr error if boundary == "attached" || boundary == "mounted" { - if err := m.applyVolume(t.Context(), sb, specs[0], false); err != nil { - t.Fatal(err) - } + applyErr = m.applyVolume(t.Context(), sb, specs[0], false) + } + if applyErr != nil { + t.Fatal(applyErr) } e.falseRemove.Store(true) } @@ -429,27 +440,27 @@ func TestManagedCreateUnrecordedInodeRequiresQuarantine(t *testing.T) { } return writeVolumeSnapshot(path, raw) } - _, _, err := m.CreateVolume(t.Context(), "data", types.VolumeCreate{SizeGiB: 1}, "t1") - requireVolumeStatus(t, err, 503) + _, _, createErr := m.CreateVolume(t.Context(), "data", types.VolumeCreate{SizeGiB: 1}, "t1") + requireVolumeStatus(t, createErr, 503) m.Close() m = openManagedTest(t, cfg, e) - v, err := m.GetVolume("data", "t1") - if err != nil || v.State != "creating" { - t.Fatalf("interrupted create=%+v %v", v, err) + v, createErr := m.GetVolume("data", "t1") + if createErr != nil || v.State != "creating" { + t.Fatalf("interrupted create=%+v %v", v, createErr) } path := m.managed.path(volumeID(v.ID)) - before, err := os.Stat(path) - if err != nil { - t.Fatal(err) + before, createErr := os.Stat(path) + if createErr != nil { + t.Fatal(createErr) } - _, _, err = m.CreateVolume(t.Context(), "data", types.VolumeCreate{SizeGiB: 1}, "t1") - ve := requireVolumeStatus(t, err, 503) + _, _, createErr = m.CreateVolume(t.Context(), "data", types.VolumeCreate{SizeGiB: 1}, "t1") + ve := requireVolumeStatus(t, createErr, 503) if ve.Code != "volume_create_incomplete" { t.Fatal(ve) } requireVolumeStatus(t, m.DeleteVolume(t.Context(), "data", v.ID, "t1"), 503) - after, err := os.Stat(path) - if err != nil || !os.SameFile(before, after) { + after, createErr := os.Stat(path) + if createErr != nil || !os.SameFile(before, after) { t.Fatal("unknown inode modified") } m.Close() @@ -457,9 +468,9 @@ func TestManagedCreateUnrecordedInodeRequiresQuarantine(t *testing.T) { t.Fatal(err) } m = openManagedTest(t, cfg, e) - repeat, created, err := m.CreateVolume(t.Context(), "data", types.VolumeCreate{SizeGiB: 1}, "t1") - if err != nil || created || repeat.ID != v.ID { - t.Fatalf("quarantine repair=%+v %v %v", repeat, created, err) + repeat, created, createErr := m.CreateVolume(t.Context(), "data", types.VolumeCreate{SizeGiB: 1}, "t1") + if createErr != nil || created || repeat.ID != v.ID { + t.Fatalf("quarantine repair=%+v %v %v", repeat, created, createErr) } } @@ -474,7 +485,7 @@ func TestManagedCorruptAndUnsafeStorage(t *testing.T) { m.Close() switch fault { case "truncated": - if err := os.WriteFile(filepath.Join(cfg.ManagedVolumeDir, "registry.json"), []byte("{"), 0600); err != nil { + if err := os.WriteFile(filepath.Join(cfg.ManagedVolumeDir, "registry.json"), []byte("{"), 0o600); err != nil { t.Fatal(err) } case "version": @@ -488,7 +499,7 @@ func TestManagedCorruptAndUnsafeStorage(t *testing.T) { } data.Version = 99 raw, _ = json.Marshal(data) - if err := os.WriteFile(filepath.Join(cfg.ManagedVolumeDir, "registry.json"), raw, 0600); err != nil { + if err := os.WriteFile(filepath.Join(cfg.ManagedVolumeDir, "registry.json"), raw, 0o600); err != nil { t.Fatal(err) } case "missing-registry": @@ -499,7 +510,7 @@ func TestManagedCorruptAndUnsafeStorage(t *testing.T) { if err := os.Rename(path, path+".original"); err != nil { t.Fatal(err) } - f, err := os.OpenFile(path, os.O_CREATE|os.O_EXCL|os.O_RDWR, 0600) + f, err := os.OpenFile(path, os.O_CREATE|os.O_EXCL|os.O_RDWR, 0o600) if err != nil { t.Fatal(err) } @@ -519,7 +530,7 @@ func TestManagedCorruptAndUnsafeStorage(t *testing.T) { t.Fatal(err) } case "permissions": - if err := os.Chmod(path, 0000); err != nil { + if err := os.Chmod(path, 0o000); err != nil { t.Fatal(err) } } @@ -544,7 +555,7 @@ func TestManagedStoreExclusiveAndDirectoryPermissions(t *testing.T) { t.Fatal("second daemon accepted locked store") } m.Close() - if err := os.Chmod(cfg.ManagedVolumeDir, 0755); err != nil { + if err := os.Chmod(cfg.ManagedVolumeDir, 0o755); err != nil { t.Fatal(err) } if _, err := NewManager(t.Context(), cfg, e, testSecrets(t), testBackends(t, cfg)); err == nil { @@ -557,9 +568,9 @@ func TestManagedStalledAttachDoesNotBlockOtherClaims(t *testing.T) { e := newManagedEngine() m := openManagedTest(t, cfg, e) v := createManagedTest(t, m, "data") - unrelated, err := m.ClaimProvision(t.Context(), testKey, ClaimOptions{}) - if err != nil { - t.Fatal(err) + unrelated, claimErr := m.ClaimProvision(t.Context(), testKey, ClaimOptions{}) + if claimErr != nil { + t.Fatal(claimErr) } e.attachEntered = make(chan struct{}, 1) e.attachResume = make(chan struct{}) @@ -572,9 +583,9 @@ func TestManagedStalledAttachDoesNotBlockOtherClaims(t *testing.T) { if err := m.Release(t.Context(), unrelated.ID, Cred{Operator: true}); err != nil { t.Fatal(err) } - got, err := m.GetVolume("data", "t1") - if err != nil || len(got.Holders) != 1 || got.Holders[0].State != "attaching" { - t.Fatalf("attaching projection=%+v %v", got, err) + got, claimErr := m.GetVolume("data", "t1") + if claimErr != nil || len(got.Holders) != 1 || got.Holders[0].State != "attaching" { + t.Fatalf("attaching projection=%+v %v", got, claimErr) } requireVolumeStatus(t, m.DeleteVolume(t.Context(), "data", v.ID, "t1"), 409) close(e.attachResume) diff --git a/sandboxd/pool/pool.go b/sandboxd/pool/pool.go index 6f0ded95..3d49c116 100644 --- a/sandboxd/pool/pool.go +++ b/sandboxd/pool/pool.go @@ -570,8 +570,7 @@ func NewManager(ctx context.Context, cfg *config.Config, eng Engine, secrets *eg for name, entry := range m.volumes { _, collision := m.managed.data.Volumes[name] if collision || m.managedCatalogAlias(entry.disk.Path) { - m.managed.close() - return nil, fmt.Errorf("catalog conflicts with existing managed resource") + return nil, errors.Join(fmt.Errorf("catalog conflicts with existing managed resource"), m.managed.close()) } } } diff --git a/sandboxd/pool/reconcile.go b/sandboxd/pool/reconcile.go index 69d8b615..95564b48 100644 --- a/sandboxd/pool/reconcile.go +++ b/sandboxd/pool/reconcile.go @@ -119,21 +119,7 @@ func (m *Manager) Reconcile(ctx context.Context) error { m.retryArchiveDeletes(ctx) removed := m.sweepStaleVMs(ctx, live, owned) - // an unreferenced hibernate snapshot is an orphan; fork and golden snapshots never span a restart - if snapsErr != nil { - logger.Warnf(ctx, "snapshot sweep skipped: %v", snapsErr) - } else { - orphans := slices.DeleteFunc(snaps, func(snap string) bool { - if strings.HasPrefix(snap, hibernatePrefix) { - return referenced[snap] - } - return !strings.HasPrefix(snap, forkPrefix) && !strings.HasPrefix(snap, goldenPrefix) - }) - m.runBounded(ctx, len(orphans), func(ctx context.Context, i int) { - m.dropSnap(ctx, orphans[i]) - logger.Infof(ctx, "removed orphan snapshot %s", orphans[i]) - }).Wait() - } + m.sweepOrphanSnapshots(ctx, snaps, snapsErr, referenced) m.resyncEgress(ctx, live, removed) m.warnDeniedClaims(ctx) logger.Infof(ctx, "adopted %d claims, %d VMs live", len(m.claimed), len(live)) @@ -344,3 +330,22 @@ func (m *Manager) reclaimOrphanArchiveCks(ctx context.Context, claims map[string } } } + +func (m *Manager) sweepOrphanSnapshots(ctx context.Context, snaps []string, snapsErr error, referenced map[string]bool) { + logger := log.WithFunc("pool.Reconcile") + // an unreferenced hibernate snapshot is an orphan; fork and golden snapshots never span a restart + if snapsErr != nil { + logger.Warnf(ctx, "snapshot sweep skipped: %v", snapsErr) + } else { + orphans := slices.DeleteFunc(snaps, func(snap string) bool { + if strings.HasPrefix(snap, hibernatePrefix) { + return referenced[snap] + } + return !strings.HasPrefix(snap, forkPrefix) && !strings.HasPrefix(snap, goldenPrefix) + }) + m.runBounded(ctx, len(orphans), func(ctx context.Context, i int) { + m.dropSnap(ctx, orphans[i]) + logger.Infof(ctx, "removed orphan snapshot %s", orphans[i]) + }).Wait() + } +} diff --git a/sandboxd/pool/volume.go b/sandboxd/pool/volume.go index 1075ce04..de82875e 100644 --- a/sandboxd/pool/volume.go +++ b/sandboxd/pool/volume.go @@ -242,21 +242,8 @@ func (m *Manager) applyVolumes(ctx context.Context, sb *types.Sandbox, volumes [ // applyVolume keeps one volume's steps strictly ordered; siblings overlap freely. func (m *Manager) applyVolume(ctx context.Context, sb *types.Sandbox, volume resolvedVolume, live bool) error { if volume.managedID == "" { - if m.managedCatalogAlias(volume.disk.Path) { - return managedStorageError() - } - if h, managed := m.managedHold(sb.ID); managed { - matched := false - for _, ref := range h.Refs { - if ref.Volume.Name == volume.applied.Name && ref.Path == volume.disk.Path { - identity, err := catalogBacking(volume.disk.Path) - matched = err == nil && identity == ref.Backing - break - } - } - if !matched { - return managedStorageError() - } + if err := m.verifyCatalogBeforeAttach(sb.ID, volume); err != nil { + return err } } if volume.managedID != "" { @@ -295,6 +282,27 @@ func (m *Manager) applyVolume(ctx context.Context, sb *types.Sandbox, volume res return nil } +func (m *Manager) verifyCatalogBeforeAttach(id string, volume resolvedVolume) error { + if m.managedCatalogAlias(volume.disk.Path) { + return managedStorageError() + } + h, managed := m.managedHold(id) + if !managed { + return nil + } + for _, ref := range h.Refs { + if ref.Volume.Name != volume.applied.Name || ref.Path != volume.disk.Path { + continue + } + identity, err := catalogBacking(volume.disk.Path) + if err != nil || identity != ref.Backing { + return managedStorageError() + } + return nil + } + return managedStorageError() +} + // quiesceVolumes unmounts a claim's writable mounts in reverse; a failed unmount keeps the marker. func (m *Manager) quiesceVolumes(ctx context.Context, sb *types.Sandbox) volumeTeardown { td := volumeTeardown{holds: heldVolumes(sb)} diff --git a/sandboxd/pool/volume_device_test.go b/sandboxd/pool/volume_device_test.go new file mode 100644 index 00000000..998f68a8 --- /dev/null +++ b/sandboxd/pool/volume_device_test.go @@ -0,0 +1,31 @@ +package pool + +import ( + "encoding/json/v2" + "testing" +) + +func TestBackingDeviceIDPreservesPersistedNumbers(t *testing.T) { + for _, tc := range []struct { + name string + device uint64 + want string + }{ + {"linux-zero", backingDeviceID(uint64(0)), "0"}, + {"linux-high-bit", backingDeviceID(uint64(1) << 63), "9223372036854775808"}, + {"linux-max", backingDeviceID(^uint64(0)), "18446744073709551615"}, + {"darwin-positive", backingDeviceID(int32(16777234)), "16777234"}, + {"darwin-negative", backingDeviceID(int32(-1)), "18446744073709551615"}, + {"darwin-min", backingDeviceID(int32(-2147483648)), "18446744071562067968"}, + } { + t.Run(tc.name, func(t *testing.T) { + raw, err := json.Marshal(backingIdentity{Device: tc.device}) + if err != nil { + t.Fatal(err) + } + if want := `{"device":` + tc.want + `,"inode":0,"size":0}`; string(raw) != want { + t.Fatalf("persisted backing identity = %s, want %s", raw, want) + } + }) + } +} diff --git a/sandboxd/pool/volume_registry.go b/sandboxd/pool/volume_registry.go index dc1e6c5f..5fc25a32 100644 --- a/sandboxd/pool/volume_registry.go +++ b/sandboxd/pool/volume_registry.go @@ -15,9 +15,11 @@ import ( "github.com/cocoonstack/sandbox/sandboxd/types" ) -type volumeID string -type volumePhase string -type holdPhase string +type ( + volumeID string + volumePhase string + holdPhase string +) const ( volumeCreating volumePhase = "creating" @@ -110,19 +112,16 @@ func openVolumeRegistry(dir, node string) (*volumeRegistry, error) { return nil, fmt.Errorf("open managed volume lock: %w", err) } if err = syscall.Flock(int(lock.Fd()), syscall.LOCK_EX|syscall.LOCK_NB); err != nil { - lock.Close() - return nil, fmt.Errorf("managed volume directory is already in use: %w", err) + return nil, errors.Join(fmt.Errorf("managed volume directory is already in use: %w", err), lock.Close()) } r := &volumeRegistry{dir: dir, node: node, lock: lock, commitFile: writeVolumeSnapshot, process: readProcessIdentity} r.rootDir, err = os.Lstat(dir) if err != nil { - lock.Close() - return nil, err + return nil, errors.Join(err, lock.Close()) } r.disksDir, err = os.Lstat(filepath.Join(dir, "disks")) if err != nil { - lock.Close() - return nil, err + return nil, errors.Join(err, lock.Close()) } r.data = volumeRegistryData{Version: 1, Volumes: map[string]managedVolume{}, Holds: map[string]volumeHold{}, Deleted: map[volumeID]volumeTombstone{}} f, err := openPrivateFile(filepath.Join(dir, "registry.json"), syscall.O_RDONLY) @@ -132,7 +131,7 @@ func openVolumeRegistry(dir, node string) (*volumeRegistry, error) { r.fault = errors.New("missing registry with existing backing files") return r, nil } - if err := r.commit(); err != nil { + if err = r.commit(); err != nil { return r, nil } return r, nil @@ -141,7 +140,7 @@ func openVolumeRegistry(dir, node string) (*volumeRegistry, error) { r.fault = err return r, nil } - defer f.Close() + defer func() { _ = f.Close() }() if err := json.UnmarshalRead(f, &r.data, json.RejectUnknownMembers(true)); err != nil { r.fault = err return r, nil @@ -162,41 +161,41 @@ func privateVolumeDir(path string) error { return err } owner := st.Sys().(*syscall.Stat_t).Uid - if owner != 0 && owner != uint32(os.Geteuid()) { + if owner != 0 && int64(owner) != int64(os.Geteuid()) { return fmt.Errorf("managed directory ancestors must belong to root or the daemon user") } - if st.Mode().Perm()&0022 != 0 && st.Mode()&os.ModeSticky == 0 { + if st.Mode().Perm()&0o022 != 0 && st.Mode()&os.ModeSticky == 0 { return fmt.Errorf("managed directory ancestors must not be writable by others") } if !st.IsDir() || st.Mode()&os.ModeSymlink != 0 { return fmt.Errorf("managed directory component is not a directory") } } - if err := os.MkdirAll(path, 0700); err != nil { + if err := os.MkdirAll(path, 0o700); err != nil { return err } st, err := os.Lstat(path) if err != nil { return err } - if !st.IsDir() || st.Mode().Perm()&0077 != 0 { + if !st.IsDir() || st.Mode().Perm()&0o077 != 0 { return fmt.Errorf("managed directory must be private (0700)") } - if st.Sys().(*syscall.Stat_t).Uid != uint32(os.Geteuid()) { + if int64(st.Sys().(*syscall.Stat_t).Uid) != int64(os.Geteuid()) { return fmt.Errorf("managed directory must belong to the daemon user") } return nil } func openPrivateFile(path string, flags int) (*os.File, error) { - fd, err := syscall.Open(path, flags|syscall.O_CLOEXEC|syscall.O_NOFOLLOW, 0600) + fd, err := syscall.Open(path, flags|syscall.O_CLOEXEC|syscall.O_NOFOLLOW, 0o600) if err != nil { return nil, err } f := os.NewFile(uintptr(fd), path) st, err := f.Stat() - if err != nil || !st.Mode().IsRegular() || st.Mode().Perm()&0077 != 0 || st.Sys().(*syscall.Stat_t).Nlink != 1 || st.Sys().(*syscall.Stat_t).Uid != uint32(os.Geteuid()) { - f.Close() + if err != nil || !st.Mode().IsRegular() || st.Mode().Perm()&0o077 != 0 || st.Sys().(*syscall.Stat_t).Nlink != 1 || int64(st.Sys().(*syscall.Stat_t).Uid) != int64(os.Geteuid()) { + _ = f.Close() return nil, fmt.Errorf("managed file must be private, owned, regular, and unlinked elsewhere") } return f, nil @@ -211,13 +210,21 @@ func inspectBacking(path string) (backingIdentity, error) { if err != nil { return backingIdentity{}, err } - defer f.Close() + defer func() { _ = f.Close() }() st, err := f.Stat() if err != nil { return backingIdentity{}, err } raw := st.Sys().(*syscall.Stat_t) - return backingIdentity{Device: uint64(raw.Dev), Inode: raw.Ino, Size: st.Size()}, nil + return backingIdentity{Device: backingDeviceID(raw.Dev), Inode: raw.Ino, Size: st.Size()}, nil +} + +// Darwin uses signed device numbers; retain their existing uint64 representation. +func backingDeviceID[T ~uint64 | ~int32](device T) uint64 { + if device < 0 { + return ^uint64(^device) + } + return uint64(device) } func (r *volumeRegistry) verify(v managedVolume) error { @@ -241,65 +248,15 @@ func (r *volumeRegistry) validate() error { } ids := map[volumeID]bool{} for name, v := range d.Volumes { - if name != v.Name || !types.ValidVolumeName(name) || !managedIDPattern.MatchString(string(v.ID)) || !types.NameRe.MatchString(v.Tenant) || v.Size < 1<<30 || v.Size > 1024<<30 || v.Size%(1<<30) != 0 || v.CreatedAt.IsZero() || v.Node == "" || ids[v.ID] { - return errors.New("invalid managed volume record") - } - switch v.State { - case volumeCreating, volumeReady, volumeDeleting, volumeRecovery: - default: - return errors.New("invalid managed volume state") - } - if (v.State == volumeReady || v.Backing.Inode != 0) && (v.Backing.Inode == 0 || v.Backing.Size != v.Size) { - return errors.New("invalid managed backing record") + if err := validateVolumeRecord(name, v, ids); err != nil { + return err } - ids[v.ID] = true } vms := map[string]bool{} holders := map[string]volumeHolders{} for id, h := range d.Holds { - if id != h.SandboxID || !strings.HasPrefix(id, "sb_") || len(h.Refs) == 0 { - return errors.New("invalid volume holder") - } - switch h.State { - case holdAttaching, holdClaimed, holdRetiring: - default: - return errors.New("invalid volume holder state") - } - if h.VMName != "" { - if !strings.HasPrefix(h.VMName, vmPrefix) || vms[h.VMName] || h.Process.PID <= 0 || h.Process.Start == 0 || h.Process.BootID == "" { - return errors.New("invalid volume process identity") - } - vms[h.VMName] = true - } else if h.State == holdClaimed { - return errors.New("claimed volume lacks VM") - } - names := map[string]bool{} - managed := false - for _, ref := range h.Refs { - if !types.ValidVolumeName(ref.Volume.Name) || names[ref.Volume.Name] || !filepath.IsAbs(ref.Path) || (ref.Volume.Mode != "" && ref.Volume.Mode != types.VolumeModeRW) { - return errors.New("invalid volume reference") - } - names[ref.Volume.Name] = true - counts := holders[ref.Volume.Name] - if counts.writers > 0 || ref.Volume.RW() && counts.readers > 0 { - return errors.New("conflicting volume holder records") - } - if ref.Volume.RW() { - counts.writers++ - } else { - counts.readers++ - } - holders[ref.Volume.Name] = counts - if ref.ID != "" { - v, ok := d.Volumes[ref.Volume.Name] - if !ok || (v.State != volumeReady && v.State != volumeRecovery) || ref.ID != v.ID || ref.Path != r.path(v.ID) || ref.Backing != v.Backing || (h.Tenant != "" && h.Tenant != v.Tenant) { - return errors.New("inconsistent managed volume reference") - } - managed = true - } - } - if !managed { - return errors.New("managed holder has no managed resource") + if err := r.validateHoldRecord(id, h, vms, holders); err != nil { + return err } } for name, p := range d.Orphans { @@ -326,6 +283,79 @@ func (r *volumeRegistry) validate() error { return nil } +func validateVolumeRecord(name string, v managedVolume, ids map[volumeID]bool) error { + if name != v.Name || !types.ValidVolumeName(name) || !managedIDPattern.MatchString(string(v.ID)) || !types.NameRe.MatchString(v.Tenant) || v.Size < 1<<30 || v.Size > 1024<<30 || v.Size%(1<<30) != 0 || v.CreatedAt.IsZero() || v.Node == "" || ids[v.ID] { + return errors.New("invalid managed volume record") + } + switch v.State { + case volumeCreating, volumeReady, volumeDeleting, volumeRecovery: + default: + return errors.New("invalid managed volume state") + } + if (v.State == volumeReady || v.Backing.Inode != 0) && (v.Backing.Inode == 0 || v.Backing.Size != v.Size) { + return errors.New("invalid managed backing record") + } + ids[v.ID] = true + return nil +} + +func (r *volumeRegistry) validateHoldRecord(id string, h volumeHold, vms map[string]bool, holders map[string]volumeHolders) error { + if id != h.SandboxID || !strings.HasPrefix(id, "sb_") || len(h.Refs) == 0 { + return errors.New("invalid volume holder") + } + switch h.State { + case holdAttaching, holdClaimed, holdRetiring: + default: + return errors.New("invalid volume holder state") + } + if h.VMName != "" { + if !strings.HasPrefix(h.VMName, vmPrefix) || vms[h.VMName] || h.Process.PID <= 0 || h.Process.Start == 0 || h.Process.BootID == "" { + return errors.New("invalid volume process identity") + } + vms[h.VMName] = true + } else if h.State == holdClaimed { + return errors.New("claimed volume lacks VM") + } + names := map[string]bool{} + managed := false + for _, ref := range h.Refs { + isManaged, err := r.validateHoldRef(h, ref, names, holders) + if err != nil { + return err + } + managed = managed || isManaged + } + if !managed { + return errors.New("managed holder has no managed resource") + } + return nil +} + +func (r *volumeRegistry) validateHoldRef(h volumeHold, ref volumeRef, names map[string]bool, holders map[string]volumeHolders) (bool, error) { + if !types.ValidVolumeName(ref.Volume.Name) || names[ref.Volume.Name] || !filepath.IsAbs(ref.Path) || (ref.Volume.Mode != "" && ref.Volume.Mode != types.VolumeModeRW) { + return false, errors.New("invalid volume reference") + } + names[ref.Volume.Name] = true + counts := holders[ref.Volume.Name] + if counts.writers > 0 || ref.Volume.RW() && counts.readers > 0 { + return false, errors.New("conflicting volume holder records") + } + if ref.Volume.RW() { + counts.writers++ + } else { + counts.readers++ + } + holders[ref.Volume.Name] = counts + if ref.ID == "" { + return false, nil + } + v, ok := r.data.Volumes[ref.Volume.Name] + if !ok || (v.State != volumeReady && v.State != volumeRecovery) || ref.ID != v.ID || ref.Path != r.path(v.ID) || ref.Backing != v.Backing || (h.Tenant != "" && h.Tenant != v.Tenant) { + return false, errors.New("inconsistent managed volume reference") + } + return true, nil +} + // Any failed snapshot poisons mutations until restart. The attempted state remains // owned in memory because rename may already have published it. func (r *volumeRegistry) commit() error { @@ -352,10 +382,9 @@ func writeVolumeSnapshot(path string, raw []byte) error { if err != nil { return err } - defer os.Remove(tmp.Name()) + defer func() { _ = os.Remove(tmp.Name()) }() if _, err := tmp.Write(raw); err != nil { - tmp.Close() - return err + return errors.Join(err, tmp.Close()) } if err := errors.Join(tmp.Sync(), tmp.Close()); err != nil { return err @@ -394,7 +423,7 @@ func (r *volumeRegistry) verifyDirectories() error { if err != nil { return err } - if !st.IsDir() || st.Mode().Perm()&0077 != 0 || !os.SameFile(st, pair.original) { + if !st.IsDir() || st.Mode().Perm()&0o077 != 0 || !os.SameFile(st, pair.original) { return errors.New("managed directory identity or permissions changed") } } diff --git a/sdk/go/managed_volume.go b/sdk/go/managed_volume.go index 05adbd17..376667f4 100644 --- a/sdk/go/managed_volume.go +++ b/sdk/go/managed_volume.go @@ -54,7 +54,7 @@ func (c *Client) CreateVolume(ctx context.Context, name string, create VolumeCre if err != nil { return nil, err } - defer resp.Body.Close() + defer func() { _ = resp.Body.Close() }() if resp.StatusCode != http.StatusCreated && resp.StatusCode != http.StatusOK { return nil, apiError("create volume", resp) } @@ -70,7 +70,7 @@ func (c *Client) GetVolume(ctx context.Context, name string) (*ManagedVolumeInfo if err != nil { return nil, err } - defer resp.Body.Close() + defer func() { _ = resp.Body.Close() }() if resp.StatusCode != http.StatusOK { return nil, apiError("get volume", resp) } @@ -97,7 +97,7 @@ func (c *Client) DeleteVolume(ctx context.Context, name, id string) error { if err != nil { return err } - defer resp.Body.Close() + defer func() { _ = resp.Body.Close() }() if resp.StatusCode != http.StatusNoContent { return apiError("delete volume", resp) } diff --git a/sdk/go/managed_volume_test.go b/sdk/go/managed_volume_test.go index 10acfcef..cbb428d7 100644 --- a/sdk/go/managed_volume_test.go +++ b/sdk/go/managed_volume_test.go @@ -14,7 +14,7 @@ import ( ) func TestManagedVolumeRequests(t *testing.T) { - want := ManagedVolumeInfo{VolumeInfo: VolumeInfo{Name: "data", Kind: "managed", ID: "vol_123", Tenant: "t1", Node: "node-1", State: "ready", CreatedAt: time.Date(2026, 10, 19, 0, 0, 0, 0, time.UTC), SizeBytes: 20 << 30, Available: true, Writable: true}, Holders: []VolumeHolder{{SandboxID: "sb_1", Mode: "rw", State: "attached"}}} + want := ManagedVolumeInfo{Name: "data", Kind: "managed", ID: "vol_123", Tenant: "t1", Node: "node-1", State: "ready", CreatedAt: time.Date(2026, 10, 19, 0, 0, 0, 0, time.UTC), SizeBytes: 20 << 30, Available: true, Writable: true, Holders: []VolumeHolder{{SandboxID: "sb_1", Mode: "rw", State: "attached"}}} requests := 0 ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { requests++ @@ -48,7 +48,7 @@ func TestManagedVolumeRequests(t *testing.T) { })) t.Cleanup(ts.Close) c := testClient(t, ts, WithAPIToken("token")) - for i := 0; i < 2; i++ { + for range 2 { got, err := c.CreateVolume(t.Context(), "data", VolumeCreate{SizeGiB: 20, Tenant: "t1"}) if err != nil || !reflect.DeepEqual(got, &want) { t.Fatalf("create = %+v, %v", got, err) @@ -73,7 +73,7 @@ func TestManagedVolumeFailures(t *testing.T) { statuses := []int{400, 401, 403, 404, 409, 412, 428, 503} for _, status := range statuses { t.Run(http.StatusText(status), func(t *testing.T) { - ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { w.WriteHeader(status) _ = json.MarshalWrite(w, map[string]any{"error": "volume unavailable", "code": "volume_busy", "volume": "data", "holders": []map[string]string{{"sandbox_id": "sb_1", "mode": "rw", "state": "retiring", "host_path": "/secret"}}}) })) @@ -92,7 +92,7 @@ func TestManagedVolumeFailures(t *testing.T) { func TestManagedVolumeRejectsInvalidRepliesAndInputs(t *testing.T) { requests := 0 - ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { requests++ _, _ = w.Write([]byte(`{}`)) })) @@ -119,7 +119,7 @@ func TestManagedVolumeRejectsInvalidRepliesAndInputs(t *testing.T) { t.Fatalf("cancellation = %v", err) } if requests != 1 { - t.Fatalf("cancelled request reached server, count %d", requests) + t.Fatalf("canceled request reached server, count %d", requests) } } @@ -128,7 +128,7 @@ func TestManagedVolumeConflictPreservesAllHolders(t *testing.T) { for i := range holders { holders[i] = VolumeHolder{SandboxID: fmt.Sprintf("sb_%032d", i), Mode: "ro", State: "claimed"} } - ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { w.WriteHeader(http.StatusConflict) _ = json.MarshalWrite(w, map[string]any{"error": "volume held", "code": "volume_busy", "volume": "data", "holders": holders}) })) From 8360beb56ea00cef94e4da051bab434701cbb5e7 Mon Sep 17 00:00:00 2001 From: minatoaquaMK2 Date: Mon, 19 Oct 2026 12:54:09 +0000 Subject: [PATCH 03/75] feat(sandboxd): hibernate mounted managed volumes --- docs/deploy.md | 1 + docs/managed-volumes.md | 89 +- docs/sandboxd-api.md | 15 +- docs/sdk-python.md | 4 +- docs/sdk.md | 4 +- sandboxd/config/config.go | 17 +- sandboxd/config/managed_hibernate.go | 24 + sandboxd/config/managed_hibernate_test.go | 28 + sandboxd/engine/cocoond_test.go | 2 +- sandboxd/engine/managed_sleep.go | 85 ++ sandboxd/engine/managed_sleep_test.go | 88 ++ sandboxd/pool/archive.go | 2 +- sandboxd/pool/claim.go | 53 +- sandboxd/pool/claims.go | 17 +- sandboxd/pool/claims_test.go | 23 + sandboxd/pool/env.go | 2 +- sandboxd/pool/hibernate.go | 26 +- sandboxd/pool/lease.go | 6 +- sandboxd/pool/managed_attachment.go | 24 +- sandboxd/pool/managed_integration_test.go | 5 + sandboxd/pool/managed_sleep.go | 381 ++++++++ .../pool/managed_sleep_boundaries_test.go | 322 +++++++ sandboxd/pool/managed_sleep_recovery.go | 142 +++ sandboxd/pool/managed_sleep_registry.go | 144 +++ sandboxd/pool/managed_sleep_test.go | 851 ++++++++++++++++++ sandboxd/pool/managed_sleep_vm.go | 110 +++ sandboxd/pool/managed_volume.go | 5 +- sandboxd/pool/operator.go | 2 +- sandboxd/pool/pool.go | 88 +- sandboxd/pool/pool_test.go | 11 + sandboxd/pool/reconcile.go | 7 +- sandboxd/pool/remove.go | 9 + sandboxd/pool/volume_registry.go | 11 +- sandboxd/types/types.go | 16 +- 34 files changed, 2511 insertions(+), 103 deletions(-) create mode 100644 sandboxd/config/managed_hibernate.go create mode 100644 sandboxd/config/managed_hibernate_test.go create mode 100644 sandboxd/engine/managed_sleep.go create mode 100644 sandboxd/engine/managed_sleep_test.go create mode 100644 sandboxd/pool/managed_sleep.go create mode 100644 sandboxd/pool/managed_sleep_boundaries_test.go create mode 100644 sandboxd/pool/managed_sleep_recovery.go create mode 100644 sandboxd/pool/managed_sleep_registry.go create mode 100644 sandboxd/pool/managed_sleep_test.go create mode 100644 sandboxd/pool/managed_sleep_vm.go diff --git a/docs/deploy.md b/docs/deploy.md index f8020f0e..cbf2fdb7 100644 --- a/docs/deploy.md +++ b/docs/deploy.md @@ -112,6 +112,7 @@ gets the intersection of the two. | `cocoon_bin` | `cocoon` | cocoon CLI binary | | `restore_mode` | unset | clone and wake-restore memory mode: `copy`, `ondemand`, or `mmap`; use `mmap` for dense pools | | `no_direct_io` | false | use buffered writable disks for Cloud Hypervisor cold boots and clones; recommended for dense ephemeral pools to avoid direct-I/O CoW journal contention | +| `managed_volume_hibernate` | absent | Enable explicit hibernate for one mounted managed writable volume with `hook_argv` and `hook_timeout_seconds` from 1 through 300. Requires `managed_volume_dir` and a guest-installed hook. Changes require restart. See [managed volumes](managed-volumes.md#hibernate-one-mounted-managed-volume). | | `sync_claims` | false | fsync `claims.json` and its directory on every commit (claim, release, renew, hibernate, wake, fork, archive, reap), about 0.3 ms each on NVMe; without it a host power loss can leave an empty journal, which fails the next start's reconcile until the file is removed, after which hibernate snapshots are reaped as orphans and archive checkpoints stay stranded. Off by default; turn it on when running claims must come back from a power loss (see `vmm_restart`) | | `cocoond_socket` | unset | `cocoon daemon`'s read-only API socket (by default `/cocoond.sock`); set, the node detects VMM exits from its event stream and polls only as a 60 s backstop. See [dead VMMs](#dead-vmms-and-host-restarts) | | `vmm_restart` | `cold` | what the node does with a running claim whose VMM exited outside a transition (a VMM crash, or a host restart that took it): `cold` cold-boots the claim's own VM from its disk under the same sandbox id and token (guest memory, processes, and the instance-metadata document are lost), `none` keeps the claim listed as `failed` for the caller to release or wake. Egress-lane and volume claims, a VM record gone from cocoon, and a VMM that exits more than 3 times in 10 minutes are marked `failed` under either value. See [dead VMMs](#dead-vmms-and-host-restarts) | diff --git a/docs/managed-volumes.md b/docs/managed-volumes.md index 87fea148..2de217d1 100644 --- a/docs/managed-volumes.md +++ b/docs/managed-volumes.md @@ -7,9 +7,10 @@ single-writer exclusion. It builds on the existing [read-only live volume API](live-volumes.md). This is a subset of [issue #300](https://github.com/cocoonstack/sandbox/issues/300). -Hibernate/wake, online growth, placement by volume and archive export remain -follow-up work. Attached volumes still block capture. Do not use this branch to -claim that all of #300 is complete. +Explicit hibernate and wake support one mounted writable managed volume when +the node enables a workload hook. Online growth, placement by volume, and +archive export remain follow-up work. Other capture operations still refuse +volume holders. This does not complete all of #300. ## Configure one node @@ -113,6 +114,78 @@ through the existing `GET/PUT/PATCH /v1/sandboxes/{id}/volumes` interface. Attach-only managed entries retain their empty mount across those changes and across daemon restart. +## Hibernate one mounted managed volume + +The node can enable explicit Hibernate and Wake for one managed writable disk +on a `net: none` claim. The applied, desired, and held volume sets must match. +Attach-only volumes, additional disks, catalog mixtures, egress, replication, +and archive remain unsupported. Idle hibernation continues to skip all volume +claims. Existing API and SDK methods take no new arguments. + +Install the same workload hook at an absolute path in every supported guest +image, together with GNU `timeout` and util-linux `flock` at `/usr/bin`. +Enable the hook in node configuration and restart sandboxd. + +```json +{ + "managed_volume_dir": "/var/lib/sandboxd/managed-volumes", + "managed_volume_hibernate": { + "hook_argv": ["/usr/local/bin/workload-volume"], + "hook_timeout_seconds": 10 + } +} +``` + +`hook_timeout_seconds` accepts 1 through 300. The daemon appends five separate +arguments to `hook_argv`. + +```text +prepare|resume operation_id volume_id volume_name exact_mount_path +``` + +`prepare` must stop workload writers and close their filesystem references +before returning success. The daemon then verifies the mount, unmounts without +force, detaches the disk, and captures VM memory. `resume` restarts writers +after the exact backing is attached and mounted. It also compensates for a +failed preparation. Both actions must be idempotent for the same operation ID. +The hook must not detach from its supervised process group or close inherited +lock descriptors while descendants continue working. + +Hook calls execute as structured argv under a guest lock at +`/run/sandboxd-volume-hook.lock`. GNU `timeout` kills the hook's process group +at its execution limit. Lock acquisition has the same time limit, and the host +RPC allows both limits plus a drain interval. A failed or disconnected call +keeps access paused until serialized compensation or resume succeeds. A hook +that deliberately escapes supervision is outside this contract. + +A successful hibernate keeps the managed hold and backing file. Another claim +or a volume deletion receives 409 while the owner sleeps. Wake verifies the +backing's path, device, inode, and size before restoring memory. A missing or +replaced backing leaves the claim paused. Restore the original backing before +retrying wake. A partial wake keeps the restored VM alive and retries attach, +mount, and resume on that same process. It never restores old memory over a +running VM. If that process dies after resume may have run, recovery refuses +rather than replaying stale memory against changed disk contents. + +Existing data streams close before preparation. Explicit and data-plane wake +publish access only after remount, resume, and durable ownership publication. +Passive operator access stays paused during incomplete transitions. Readers +never serve the detached snapshot. + +An effective outbound policy, including a proxy on the `none` lane, blocks managed +hibernation before it changes the running guest. If a policy reload enables +outbound access while the claim sleeps or resumes, wake stays paused until the +effective policy is removed. Release remains available. Release and lease expiry +remove the VM and snapshot before freeing the hold, while preserving the managed +disk. + +The selected hook and operation ID remain in the durable sleep record. Retries +use them even after a configuration change or disabling new managed sleeps. +The first sleep durably upgrades the managed registry to version 2. Older +binaries refuse that registry. Keep the upgraded binary when restarting it. +Claim barriers preserve this lifecycle across daemon crashes with either value +of `sync_claims`. They do not add a host-power-loss recovery guarantee. + ## Ownership and recovery The managed registry supplements the existing flat `claims.json` journal. @@ -142,11 +215,9 @@ remain in force. ## Follow-up order -1. Add quiesce/unmount/detach/capture and restore/reattach/remount with failure - rollback. Keep single-writer ownership while a claim is hibernated. -2. Add online growth with explicit block-device and guest-filesystem outcomes. -3. Add owner-node placement and a clear unavailable-owner response. -4. Add archive/export and restore of external data disks. -5. Add Kubernetes lifecycle/status ownership once the daemon contracts are stable. +1. Add online growth with explicit block-device and guest-filesystem outcomes. +2. Add owner-node placement and a clear unavailable-owner response. +3. Add archive/export and restore of external data disks. +4. Add Kubernetes lifecycle/status ownership once the daemon contracts are stable. These are proposed next steps. None is implemented or qualified by this phase. diff --git a/docs/sandboxd-api.md b/docs/sandboxd-api.md index ebf4c96a..ffbbcfe0 100644 --- a/docs/sandboxd-api.md +++ b/docs/sandboxd-api.md @@ -402,12 +402,15 @@ freeing its memory; the next agent access restores it transparently snapshot point and the stop coincident), except a read-only file call that a [reader](#get-v1sandboxesidagent) serves without waking it. Idempotent on an already-hibernated sandbox. The TTL keeps running: a hibernated sandbox is still reaped (VM and -snapshot) at its deadline, unless it claimed `on_expire: archive` or its pool -archives. When to hibernate is the caller's policy — the -node only provides the transition. 204 on success, 404 unknown id or wrong -token, 409 on the egress lane or when volumes are attached (neither kind of -sandbox hibernates; see [egress](egress.md)), 429 while the disk is under -[`disk_reserve`](deploy.md). +snapshot) at its deadline. Claims without volume holds are excepted when they +claimed `on_expire: archive` or their pool archives. Hibernated managed-volume +claims expire through release, which preserves the backing disk. When to hibernate +is the caller's policy. The node only provides the transition. 204 on success, 404 unknown id or wrong +token, 409 on the egress lane or unsupported volume claims, 429 while the disk is under +[`disk_reserve`](deploy.md). An opt-in node hook supports one mounted writable +managed volume on the `none` lane. See [managed volume hibernation](managed-volumes.md#hibernate-one-mounted-managed-volume) +for hook installation, exclusive ownership, and failure recovery. Other volume +capture operations remain blocked. ## POST /v1/sandboxes/{id}/wake diff --git a/docs/sdk-python.md b/docs/sdk-python.md index e168fc4f..12d6cdd2 100644 --- a/docs/sdk-python.md +++ b/docs/sdk-python.md @@ -126,7 +126,9 @@ branched). `sb.template_digest` is the exact content identity when the claim cloned a promoted template; it is empty for other sources. `Sandbox` is a context manager; `sb.close()` releases it (releasing one already gone is not an error — double-release and reap races stay silent). -Volume sandboxes cannot hibernate, fork, checkpoint, or promote. Checkpoint +Volume sandboxes cannot fork, checkpoint, or promote. Explicit hibernate supports +one mounted managed writable volume when the node enables a workload hook. +See [managed volume hibernation](managed-volumes.md#hibernate-one-mounted-managed-volume). Checkpoint branches do not accept volumes of either mode in this version. `mount=False` on `client.new` or `template.new` claims the same volumes diff --git a/docs/sdk.md b/docs/sdk.md index c7da4017..f951e28a 100644 --- a/docs/sdk.md +++ b/docs/sdk.md @@ -230,7 +230,9 @@ promoted template; it is empty for other sources. `Owner()` names the owning node, and `Token()` returns the per-sandbox bearer to persist with `ID` for a later `Lookup`; `Close()` releases the sandbox (releasing one already gone is not an error, and `Close` is bounded internally so it stays defer-friendly). -Volume sandboxes cannot hibernate, fork, checkpoint, or promote. Passing +Volume sandboxes cannot fork, checkpoint, or promote. Explicit hibernate supports +one mounted managed writable volume when the node enables a workload hook. +See [managed volume hibernation](managed-volumes.md#hibernate-one-mounted-managed-volume). Passing `WithVolumes` or `WithClaimRef` to `Checkpoint.New` returns a local error: checkpoint branches support neither volumes nor a claim reference in this version. `WithMetadata`, `WithArchiveOnExpire` and `WithEnv` apply to `New`, `Template.New`, and diff --git a/sandboxd/config/config.go b/sandboxd/config/config.go index 63e9d4aa..d14a037f 100644 --- a/sandboxd/config/config.go +++ b/sandboxd/config/config.go @@ -274,10 +274,11 @@ type Config struct { DiskReserve *DiskReserve `json:"disk_reserve,omitempty"` - ManagedVolumeDir string `json:"managed_volume_dir,omitempty"` - Volumes []VolumeSpec `json:"volumes,omitempty"` - Mesh *MeshConfig `json:"mesh,omitempty"` - Pools []PoolSpec `json:"pools"` + ManagedVolumeHibernate *ManagedVolumeHibernate `json:"managed_volume_hibernate,omitempty"` + ManagedVolumeDir string `json:"managed_volume_dir,omitempty"` + Volumes []VolumeSpec `json:"volumes,omitempty"` + Mesh *MeshConfig `json:"mesh,omitempty"` + Pools []PoolSpec `json:"pools"` } // HasEgress reports whether the node can attach egress-lane VMs. @@ -427,6 +428,14 @@ func (c *Config) validateVolumes() error { } func (c *Config) ValidateManagedVolumeStore() error { + if c.ManagedVolumeHibernate != nil { + if err := c.ManagedVolumeHibernate.Validate(); err != nil { + return err + } + if c.ManagedVolumeDir == "" { + return fmt.Errorf("managed_volume_hibernate requires managed_volume_dir") + } + } if c.ManagedVolumeDir == "" { return nil } diff --git a/sandboxd/config/managed_hibernate.go b/sandboxd/config/managed_hibernate.go new file mode 100644 index 00000000..7582e646 --- /dev/null +++ b/sandboxd/config/managed_hibernate.go @@ -0,0 +1,24 @@ +package config + +import ( + "fmt" + "path/filepath" + "strings" +) + +type ManagedVolumeHibernate struct { + HookArgv []string `json:"hook_argv"` + HookTimeoutSeconds int `json:"hook_timeout_seconds"` +} + +func (h ManagedVolumeHibernate) Validate() error { + if len(h.HookArgv) == 0 || !filepath.IsAbs(h.HookArgv[0]) || filepath.Clean(h.HookArgv[0]) != h.HookArgv[0] || h.HookTimeoutSeconds < 1 || h.HookTimeoutSeconds > 300 { + return fmt.Errorf("managed_volume_hibernate requires absolute hook_argv and hook_timeout_seconds from 1 through 300") + } + for _, arg := range h.HookArgv { + if strings.ContainsRune(arg, 0) { + return fmt.Errorf("managed_volume_hibernate hook_argv contains NUL") + } + } + return nil +} diff --git a/sandboxd/config/managed_hibernate_test.go b/sandboxd/config/managed_hibernate_test.go new file mode 100644 index 00000000..ea1151a3 --- /dev/null +++ b/sandboxd/config/managed_hibernate_test.go @@ -0,0 +1,28 @@ +package config + +import "testing" + +func TestManagedHibernateHookValidation(t *testing.T) { + for _, tc := range []struct { + name string + hook ManagedVolumeHibernate + valid bool + }{ + {"valid", ManagedVolumeHibernate{[]string{"/usr/local/bin/hook", "literal $(arg)"}, 10}, true}, + {"missing command", ManagedVolumeHibernate{nil, 10}, false}, + {"relative command", ManagedVolumeHibernate{[]string{"hook"}, 10}, false}, + {"NUL argument", ManagedVolumeHibernate{[]string{"/hook", "bad\x00arg"}, 10}, false}, + {"zero timeout", ManagedVolumeHibernate{[]string{"/hook"}, 0}, false}, + {"excess timeout", ManagedVolumeHibernate{[]string{"/hook"}, 301}, false}, + } { + t.Run(tc.name, func(t *testing.T) { + if err := tc.hook.Validate(); (err == nil) != tc.valid { + t.Fatalf("valid=%v error=%v", tc.valid, err) + } + }) + } + cfg := Config{ManagedVolumeHibernate: &ManagedVolumeHibernate{[]string{"/hook"}, 1}} + if err := cfg.ValidateManagedVolumeStore(); err == nil { + t.Fatal("enabled hibernate without managed storage") + } +} diff --git a/sandboxd/engine/cocoond_test.go b/sandboxd/engine/cocoond_test.go index 053dc205..77044695 100644 --- a/sandboxd/engine/cocoond_test.go +++ b/sandboxd/engine/cocoond_test.go @@ -48,7 +48,7 @@ func TestInspectReadsOneVMAndReportsAMissingOne(t *testing.T) { "echo \"Error: inspect: vm $3: vm not found\" >&2; exit 1\n") e := New("cocoon", nil, nil, false, false, "") rec, ok, err := e.Inspect(t.Context(), "sbx-1") - wantDisks := []types.VMDisk{{Name: "imagenet", ReadOnly: true}, {Name: "scratch"}} + wantDisks := []types.VMDisk{{Name: "imagenet", Path: "/v/a.img", ReadOnly: true}, {Name: "scratch", Path: "/v/b.img"}} if err != nil || !ok || rec.State != "running" || rec.Config.Name != "sbx-1" || !slices.Equal(rec.AttachedDevices.Disks, wantDisks) { t.Errorf("Inspect sbx-1 = %+v, %v, %v", rec, ok, err) } diff --git a/sandboxd/engine/managed_sleep.go b/sandboxd/engine/managed_sleep.go new file mode 100644 index 00000000..5c6b95cd --- /dev/null +++ b/sandboxd/engine/managed_sleep.go @@ -0,0 +1,85 @@ +package engine + +import ( + "context" + "encoding/json/v2" + "fmt" + "slices" + "strconv" + "strings" + "time" +) + +type VolumeHookCall struct { + Argv []string + Action, Operation, VolumeID, VolumeName, Mount string + Timeout time.Duration +} + +type HibernateSource struct { + ID string `json:"id"` + Name string `json:"name"` +} + +func (e *Engine) RunVolumeHook(ctx context.Context, socket string, call VolumeHookCall) error { + seconds := strconv.FormatFloat(call.Timeout.Seconds(), 'f', -1, 64) + argv := []string{"/usr/bin/flock", "--timeout", seconds, "/run/sandboxd-volume-hook.lock", "/usr/bin/timeout", "--signal=KILL", "--", seconds + "s"} + argv = append(argv, call.Argv...) + argv = append(argv, call.Action, call.Operation, call.VolumeID, call.VolumeName, call.Mount) + ctx, cancel := context.WithTimeout(ctx, 2*call.Timeout+10*time.Second) + defer cancel() + _, err := e.silkdOutput(ctx, socket, argv...) + return err +} + +func (e *Engine) InspectHibernateSource(ctx context.Context, name string) (HibernateSource, bool, error) { + names, err := e.SnapshotList(ctx) + if err != nil { + return HibernateSource{}, false, err + } + if !slices.Contains(names, name) { + return HibernateSource{}, false, nil + } + raw, err := e.run(ctx, "snapshot", "inspect", name) + if err != nil { + return HibernateSource{}, false, err + } + var source HibernateSource + if err = json.Unmarshal(raw, &source); err != nil { + return source, false, fmt.Errorf("parse snapshot inspect: %w", err) + } + if source.ID == "" || source.Name != name { + return source, false, fmt.Errorf("snapshot identity unavailable") + } + return source, true, nil +} + +func (e *Engine) VerifyMountedVolume(ctx context.Context, socket, name, mount string) error { + ctx, cancel := context.WithTimeout(ctx, volumeSetupTimeout) + defer cancel() + device, found, err := e.findVolumeDevice(ctx, socket, name) + if err != nil { + return err + } + if !found { + return &VolumeMountConflict{Message: "managed volume device is absent"} + } + devNum, mounts, err := e.volumeMounts(ctx, socket, device) + if err != nil { + return err + } + matched := false + for _, entry := range mounts { + if entry.path != mount && entry.devNum != devNum { + continue + } + if matched || entry.path != mount || entry.devNum != devNum || entry.root != "/" || !slices.Contains(strings.Split(entry.options, ","), "rw") { + return &VolumeMountConflict{Message: "managed volume must have one exact writable root mount"} + } + matched = true + } + if !matched { + return &VolumeMountConflict{Message: "managed volume mount is absent"} + } + return nil +} diff --git a/sandboxd/engine/managed_sleep_test.go b/sandboxd/engine/managed_sleep_test.go new file mode 100644 index 00000000..d9c5cf14 --- /dev/null +++ b/sandboxd/engine/managed_sleep_test.go @@ -0,0 +1,88 @@ +package engine + +import ( + "errors" + "slices" + "testing" + "time" +) + +func TestManagedHookUsesBoundedSerializedArgv(t *testing.T) { + sock := sockPath(t) + fake := serveFakeSilkd(t, sock) + e := New("cocoon", nil, nil, false, false, "") + call := VolumeHookCall{Argv: []string{"/opt/hook", "$(touch /tmp/escape); literal"}, Action: "prepare", Operation: "op", VolumeID: "vol", VolumeName: "data", Mount: "/data with spaces", Timeout: 10 * time.Second} + if err := e.RunVolumeHook(t.Context(), sock, call); err != nil { + t.Fatal(err) + } + fake.mu.Lock() + got := slices.Clone(fake.execCalls[0]) + fake.execCode = 137 + fake.mu.Unlock() + want := []string{"/usr/bin/flock", "--timeout", "10", "/run/sandboxd-volume-hook.lock", "/usr/bin/timeout", "--signal=KILL", "--", "10s", "/opt/hook", "$(touch /tmp/escape); literal", "prepare", "op", "vol", "data", "/data with spaces"} + if !slices.Equal(got, want) { + t.Fatalf("guest argv %q", got) + } + if err := e.RunVolumeHook(t.Context(), sock, call); err == nil { + t.Fatal("hook termination reported success") + } +} + +func TestManagedMountRequiresLiteralWritableRoot(t *testing.T) { + for _, tc := range []struct { + name, mountinfo string + valid bool + }{ + {"exact", "2 1 8:32 / /data rw - ext4 /dev/vdc rw\n", true}, + {"absent", "", false}, + {"read only", "2 1 8:32 / /data ro - ext4 /dev/vdc ro\n", false}, + {"wrong device", "2 1 8:48 / /data rw - ext4 /dev/vdd rw\n", false}, + {"subdirectory bind", "2 1 8:32 /sub /data rw - ext4 /dev/vdc rw\n", false}, + {"alias", "2 1 8:32 / /elsewhere rw - ext4 /dev/vdc rw\n", false}, + {"extra mount", "2 1 8:32 / /data rw - ext4 /dev/vdc rw\n3 1 8:32 / /alias rw - ext4 /dev/vdc rw\n", false}, + } { + t.Run(tc.name, func(t *testing.T) { + sock := sockPath(t) + fake := serveFakeSilkd(t, sock) + configureVolumeDevices(fake) + fake.mu.Lock() + fake.serial["/sys/block/vdc/dev"] = []byte("8:32\n") + fake.serial["/proc/self/mountinfo"] = []byte(rootMountInfo + tc.mountinfo) + fake.mu.Unlock() + err := New("cocoon", nil, nil, false, false, "").VerifyMountedVolume(t.Context(), sock, "imagenet", "/data") + if tc.valid && err != nil { + t.Fatal(err) + } + if !tc.valid { + if _, ok := errors.AsType[*VolumeMountConflict](err); !ok { + t.Fatalf("mount accepted or error wrong: %v", err) + } + } + fake.mu.Lock() + defer fake.mu.Unlock() + if len(fake.execCalls) != 0 { + t.Fatal("mount preflight changed guest") + } + }) + } +} + +func TestHibernateSourceRequiresExactInspectMetadata(t *testing.T) { + for _, tc := range []struct { + name, listed, metadata string + found, fail bool + }{ + {"present", `[{"name":"sleep"}]`, `{"name":"sleep","id":"immutable-source"}`, true, false}, + {"missing", `[]`, `{}`, false, false}, + {"wrong name", `[{"name":"sleep"}]`, `{"name":"other","id":"immutable-source"}`, false, true}, + {"missing id", `[{"name":"sleep"}]`, `{"name":"sleep"}`, false, true}, + } { + t.Run(tc.name, func(t *testing.T) { + fakeCocoon(t, "#!/bin/sh\ncase \"$1 $2\" in\n'snapshot list') echo '"+tc.listed+"';;\n'snapshot inspect') echo '"+tc.metadata+"';;\n*) exit 1;;\nesac\n") + source, found, err := New("cocoon", nil, nil, false, false, "").InspectHibernateSource(t.Context(), "sleep") + if found != tc.found || (err != nil) != tc.fail { + t.Fatalf("source=%+v found=%v err=%v", source, found, err) + } + }) + } +} diff --git a/sandboxd/pool/archive.go b/sandboxd/pool/archive.go index af26aa17..d7f87e05 100644 --- a/sandboxd/pool/archive.go +++ b/sandboxd/pool/archive.go @@ -49,7 +49,7 @@ func (m *Manager) archiveOnce(ctx context.Context) { var victims []*types.Sandbox m.mu.Lock() for _, sb := range m.claimed { - if sb.HibernateSnap == "" || sb.ArchiveCk != "" || m.tenantGone(sb.Tenant) { + if m.hasVolumeHolds(sb) || sb.HibernateSnap == "" || sb.ArchiveCk != "" || m.tenantGone(sb.Tenant) { continue } if _, busy := m.archiving[sb.ID]; busy { diff --git a/sandboxd/pool/claim.go b/sandboxd/pool/claim.go index deefbf81..ac38b599 100644 --- a/sandboxd/pool/claim.go +++ b/sandboxd/pool/claim.go @@ -217,6 +217,9 @@ func (m *Manager) AgentSocket(id, token string) (string, error) { return "", ErrLeaseLost } // no activity stamp: a control-plane poll must not keep an idle sandbox awake + if m.managedPaused(sb) { + return "", ErrPaused + } return sb.VsockSocket, nil } @@ -233,7 +236,12 @@ func (m *Manager) dialPort(ctx context.Context, id string, cred Cred, port uint1 sb.Touch() m.recordAudit(ctx, id, auditFrame{Op: op, Port: port}) sb.Hold() - sock, err := m.wakeResolved(ctx, sb) + sb.Transition.Lock() + sock, err := m.wakeOrReadLocked(ctx, sb, false) + if err == nil { + fenced, err = m.fenceWatch(sb) + } + sb.Transition.Unlock() if err != nil { sb.Unhold() return nil, err @@ -256,15 +264,19 @@ func (m *Manager) dialPassive(ctx context.Context, id string, port uint16) (net. if !ok { return nil, ErrUnknownSandbox } - fenced, err := m.fenceWatch(sb) - if err != nil { + if _, err := m.fenceWatch(sb); err != nil { return nil, err } if !sb.Transition.TryLock() { return nil, ErrPaused } + fenced, err := m.fenceWatch(sb) + if err != nil { + sb.Transition.Unlock() + return nil, err + } failed := sb.Failed != "" - paused, sock := sb.HibernateSnap != "" || sb.PendingSnap != "" || sb.ArchiveCk != "", sb.VsockSocket + paused, sock := m.managedPaused(sb) || sb.HibernateSnap != "" || sb.PendingSnap != "" || sb.ArchiveCk != "", sb.VsockSocket if !paused && !failed { sb.Hold() } @@ -291,6 +303,11 @@ func (m *Manager) dialPassive(ctx context.Context, id string, port uint16) (net. // releaseResolved re-checks under m.mu that sb is still the live claim: no double teardown. func (m *Manager) releaseResolved(ctx context.Context, id string, sb *types.Sandbox) error { + if m.managedLifecycle(sb) { + sb.Transition.Lock() + defer sb.Transition.Unlock() + return m.releaseManagedSleepLocked(ctx, sb, time.Time{}) + } var marked string m.mu.Lock() for { @@ -433,6 +450,9 @@ func (m *Manager) tenantDelta(tenant string, delta int) { } func (m *Manager) addClaimLocked(sb *types.Sandbox) { + if _, ok := m.managedHold(sb.ID); ok { + m.managedStreams[sb] = make(chan struct{}) + } if m.archives != nil { _, replicating := m.replicaPools[sb.PolicyKey()] sb.SetReplicated(replicating || sb.HasReplica()) @@ -442,6 +462,14 @@ func (m *Manager) addClaimLocked(sb *types.Sandbox) { } func (m *Manager) dropClaimLocked(sb *types.Sandbox) { + if ch := m.managedStreams[sb]; ch != nil { + select { + case <-ch: + default: + close(ch) + } + delete(m.managedStreams, sb) + } delete(m.claimed, sb.ID) m.tenantDelta(sb.Tenant, -1) } @@ -570,6 +598,7 @@ func (m *Manager) reapOnce(ctx context.Context) { } m.mu.Lock() var expired []victim + var managedExpired []*types.Sandbox var dropped, purges []string var unrows []*types.Sandbox for id, sb := range m.claimed { @@ -577,6 +606,10 @@ func (m *Manager) reapOnce(ctx context.Context) { if sb.Deadline.IsZero() || !now.After(sb.Deadline) { continue } + if m.managedLifecycle(sb) { + managedExpired = append(managedExpired, sb) + continue + } switch { case m.expiryDropsRow(sb): if _, busy := m.archiving[id]; !busy { @@ -610,9 +643,13 @@ func (m *Manager) reapOnce(ctx context.Context) { m.store.del(dropped...) } m.mu.Unlock() - if len(unrows) > 0 { - m.runBounded(ctx, len(unrows), func(ctx context.Context, i int) { m.unrowExpired(ctx, unrows[i]) }) - } + m.runBounded(ctx, len(managedExpired), func(ctx context.Context, i int) { + sb := managedExpired[i] + sb.Transition.Lock() + defer sb.Transition.Unlock() + _ = m.releaseManagedSleepLocked(ctx, sb, now) + }) + m.runBounded(ctx, len(unrows), func(ctx context.Context, i int) { m.unrowExpired(ctx, unrows[i]) }) if len(expired) == 0 { return } @@ -703,7 +740,7 @@ func (m *Manager) reapOnce(ctx context.Context) { } func (m *Manager) archivesAtDeadline(sb *types.Sandbox) bool { - return sb.Failed == "" && !m.tenantGone(sb.Tenant) && (sb.OnExpire == types.ExpireArchive || sb.HibernateSnap != "" && m.archiveAfterFor(sb.Key) > 0) + return !m.hasVolumeHolds(sb) && sb.Failed == "" && !m.tenantGone(sb.Tenant) && (sb.OnExpire == types.ExpireArchive || sb.HibernateSnap != "" && m.archiveAfterFor(sb.Key) > 0) } func (m *Manager) pauseExpired(ctx context.Context, sb *types.Sandbox) error { diff --git a/sandboxd/pool/claims.go b/sandboxd/pool/claims.go index 5d7111df..e1e85672 100644 --- a/sandboxd/pool/claims.go +++ b/sandboxd/pool/claims.go @@ -162,10 +162,18 @@ func (s *claimStore) reset(claims map[string]*types.Sandbox) claimSnapshot { // commit atomically replaces claims.json, encoding only the rows changed since the last commit; fsync is opt-in (sync_claims). func (s *claimStore) commit(snap claimSnapshot) error { + return s.commitSnapshot(snap, false) +} + +func (s *claimStore) commitDurable(snap claimSnapshot) error { + return s.commitSnapshot(snap, true) +} + +func (s *claimStore) commitSnapshot(snap claimSnapshot, durable bool) error { s.writeMu.Lock() defer s.writeMu.Unlock() s.mu.Lock() - if snap.seq <= s.written { + if !durable && snap.seq <= s.written { s.mu.Unlock() return nil } @@ -186,7 +194,7 @@ func (s *claimStore) commit(snap claimSnapshot) error { s.buf = append(s.buf, row.raw...) } s.buf = append(s.buf, '}') - if err := s.replace(s.buf); err != nil { + if err := s.replaceMode(s.buf, durable || s.sync); err != nil { return err } s.mu.Lock() @@ -218,10 +226,9 @@ func (s *claimStore) recommit(ctx context.Context, snap claimSnapshot) { }() } -// replace swaps claims.json for raw through the fixed temp name, durably when sync is set. -func (s *claimStore) replace(raw []byte) error { +func (s *claimStore) replaceMode(raw []byte, durable bool) error { tmp := s.path + ".tmp" - if !s.sync { + if !durable { if err := os.WriteFile(tmp, raw, 0o600); err != nil { return fmt.Errorf("write claims: %w", err) } diff --git a/sandboxd/pool/claims_test.go b/sandboxd/pool/claims_test.go index 0539a537..f570d72d 100644 --- a/sandboxd/pool/claims_test.go +++ b/sandboxd/pool/claims_test.go @@ -231,3 +231,26 @@ func TestClaimStoreConcurrentCommits(t *testing.T) { t.Fatalf("after concurrent commits: %+v, %v", got, err) } } + +func TestDurableClaimBarrierRewritesLatestAlreadyWrittenRows(t *testing.T) { + s := newClaimStore(t.TempDir(), false) + sb := &types.Sandbox{ID: "sb-barrier", Token: "first"} + old := s.set(sb) + if err := s.commit(old); err != nil { + t.Fatal(err) + } + sb.Token = "latest" + if err := s.commit(s.set(sb)); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(s.path, []byte("{}"), 0o600); err != nil { + t.Fatal(err) + } + if err := s.commitDurable(old); err != nil { + t.Fatal(err) + } + got, err := s.load() + if err != nil || got[sb.ID] == nil || got[sb.ID].Token != "latest" { + t.Fatalf("durable barrier did not rewrite latest credentials: %v %v", got, err) + } +} diff --git a/sandboxd/pool/env.go b/sandboxd/pool/env.go index a6f0f2f9..f689346a 100644 --- a/sandboxd/pool/env.go +++ b/sandboxd/pool/env.go @@ -70,7 +70,7 @@ func (m *Manager) writeEnv(ctx context.Context, id, tenant, verb string, next fu } } changed := !env.SameGuest(prev) - paused := !locked || sb.HibernateSnap != "" || sb.PendingSnap != "" || sb.ArchiveCk != "" || failed + paused := !locked || m.managedPaused(sb) || sb.HibernateSnap != "" || sb.PendingSnap != "" || sb.ArchiveCk != "" || failed if changed && paused { m.mu.Unlock() if failed { diff --git a/sandboxd/pool/hibernate.go b/sandboxd/pool/hibernate.go index cac827ed..6b26e76b 100644 --- a/sandboxd/pool/hibernate.go +++ b/sandboxd/pool/hibernate.go @@ -44,7 +44,12 @@ func (m *Manager) agentSocket(ctx context.Context, id, token string, readerOK bo return "", nil, nil, err } sb.Hold() - sock, err := m.wakeOrRead(ctx, sb, readerOK) + sb.Transition.Lock() + sock, err := m.wakeOrReadLocked(ctx, sb, readerOK) + if err == nil { + fenced, err = m.fenceWatch(sb) + } + sb.Transition.Unlock() switch { case errors.Is(err, ErrSleeping): sb.UnholdIdle() @@ -69,6 +74,9 @@ func (m *Manager) hibernateLocked(ctx context.Context, sb *types.Sandbox) error volumeBlocked := m.hasVolumeHolds(sb) m.mu.Unlock() if volumeBlocked { + if m.managedLifecycle(sb) { + return m.driveManagedSleepLocked(ctx, sb, managedSleeping) + } return ErrVolumeCapture } if sb.Key.Net == types.NetEgress { @@ -150,15 +158,25 @@ func (m *Manager) wakeResolved(ctx context.Context, sb *types.Sandbox) (string, func (m *Manager) wakeOrRead(ctx context.Context, sb *types.Sandbox, readerOK bool) (string, error) { sb.Transition.Lock() defer sb.Transition.Unlock() + return m.wakeOrReadLocked(ctx, sb, readerOK) +} + +func (m *Manager) wakeOrReadLocked(ctx context.Context, sb *types.Sandbox, readerOK bool) (string, error) { + if sb.Failed != "" { + return "", fmt.Errorf("%w: %s", ErrFailed, sb.Failed) + } + if m.managedPaused(sb) { + if err := m.driveManagedSleepLocked(ctx, sb, managedRunning); err != nil { + return "", err + } + return sb.VsockSocket, nil + } if sb.ArchiveCk != "" { if readerOK && sb.ReaderSafe { return "", ErrSleeping } return m.wakeArchived(ctx, sb) } - if sb.Failed != "" { - return "", fmt.Errorf("%w: %s", ErrFailed, sb.Failed) - } // settle a dangling intent before choosing running vs hibernated if _, err := m.settlePendingSnap(ctx, sb); err != nil { return "", fmt.Errorf("wake %s: %w", sb.ID, err) diff --git a/sandboxd/pool/lease.go b/sandboxd/pool/lease.go index 4a136af3..05c7c90e 100644 --- a/sandboxd/pool/lease.go +++ b/sandboxd/pool/lease.go @@ -179,10 +179,12 @@ func (m *Manager) confirmOwnership(ctx context.Context, sb *types.Sandbox, rec a return false, fmt.Errorf("confirm %s: its row kept moving under this node", sb.ID) } -// fenceWatch gates sb at resolution and returns the channel that closes when its streams must end; nil for an unreplicated claim. func (m *Manager) fenceWatch(sb *types.Sandbox) (<-chan struct{}, error) { if !sb.Replicated() { - return nil, nil + m.mu.Lock() + ch := m.managedStreams[sb] + m.mu.Unlock() + return ch, nil } st := m.fence.Load() if st.fenced { diff --git a/sandboxd/pool/managed_attachment.go b/sandboxd/pool/managed_attachment.go index df82e5fe..03b835df 100644 --- a/sandboxd/pool/managed_attachment.go +++ b/sandboxd/pool/managed_attachment.go @@ -6,6 +6,7 @@ import ( "fmt" "io/fs" "os" + "slices" "strconv" "strings" "syscall" @@ -210,7 +211,11 @@ func (m *Manager) finishManaged(id string, clears []string) bool { if r.fault != nil { return false } - if err := m.store.commit(m.store.del(id)); err != nil { + commit := m.store.commit + if h.Sleep != nil { + commit = m.store.commitDurable + } + if err := commit(m.store.del(id)); err != nil { return false } for _, path := range clears { @@ -259,6 +264,14 @@ func (m *Manager) recoverManaged(ctx context.Context, claims map[string]*types.S } m.mu.Unlock() sb, exists := claims[h.SandboxID] + if h.Sleep != nil && h.State != holdRetiring { + if !exists || !sleepMatchesClaim(h, sb) { + return owned, managedStorageError() + } + projectSleepState(sb, h.Sleep) + owned[h.VMName] = true + continue + } rec, present := live[h.VMName] valid := exists && sb.VMName == h.VMName && sb.Tenant == h.Tenant && sb.Failed != egressQuarantineFailure && h.State != holdRetiring && present if valid { @@ -270,14 +283,7 @@ func (m *Manager) recoverManaged(ctx context.Context, claims map[string]*types.S } if valid { for _, volume := range heldVolumes(sb) { - matched := false - for _, ref := range h.Refs { - if ref.Volume == volume { - matched = true - break - } - } - if !matched { + if !slices.ContainsFunc(h.Refs, func(ref volumeRef) bool { return ref.Volume == volume }) { return owned, managedStorageError() } } diff --git a/sandboxd/pool/managed_integration_test.go b/sandboxd/pool/managed_integration_test.go index ef0c4bb9..fc84a893 100644 --- a/sandboxd/pool/managed_integration_test.go +++ b/sandboxd/pool/managed_integration_test.go @@ -281,4 +281,9 @@ func TestManagedQuarantineStillRemovesVMWhenRetirementCannotCommit(t *testing.T) if _, held := m.managedHold(sb.ID); !held { t.Fatal("failed retirement discarded disk ownership") } + waitFor(t, func() bool { return !m.store.recommitting.Load() && m.store.synced() }) + persisted, loadErr := m.store.load() + if loadErr != nil || persisted[sb.ID] == nil || persisted[sb.ID].Failed != egressQuarantineFailure { + t.Fatalf("quarantine marker was not persisted: %v %v", persisted[sb.ID], loadErr) + } } diff --git a/sandboxd/pool/managed_sleep.go b/sandboxd/pool/managed_sleep.go new file mode 100644 index 00000000..d7fb1992 --- /dev/null +++ b/sandboxd/pool/managed_sleep.go @@ -0,0 +1,381 @@ +package pool + +import ( + "context" + "errors" + "fmt" + "slices" + "strings" + + "github.com/cocoonstack/sandbox/sandboxd/types" +) + +type managedSleepTarget bool + +const ( + managedRunning managedSleepTarget = false + managedSleeping managedSleepTarget = true +) + +func sleepRecoveryError() error { + return volumeError(409, "managed_sleep_recovery_required", "managed sleep ownership or VM state is uncertain; retain the backing and retry or release the sandbox") +} + +func (m *Manager) managedSleepPreflight(ctx context.Context, sb *types.Sandbox) (volumeHold, types.VMRecord, error) { + h, ok := m.managedHold(sb.ID) + m.mu.Lock() + live := m.claimed[sb.ID] == sb + busy := m.volumeRuns[sb] != nil + _, outbound := m.view.Load().out.Resolve(sb, m.claimPooledLocked) + m.mu.Unlock() + if !ok || m.managedSleepHook == nil || h.State != holdClaimed || !sleepMatchesClaim(h, sb) || h.Refs[0].Volume.Mount == "" || !h.Refs[0].Volume.RW() || h.Refs[0].ID == "" || busy || outbound { + return h, types.VMRecord{}, ErrVolumeCapture + } + if !live { + return h, types.VMRecord{}, ErrUnknownSandbox + } + if err := m.diskErr(); err != nil { + return h, types.VMRecord{}, err + } + if err := m.verifySleepBacking(h); err != nil { + return h, types.VMRecord{}, err + } + if err := m.eng.CheckLiveVolumeSupport(ctx); err != nil { + return h, types.VMRecord{}, err + } + rec, present, err := m.eng.Inspect(ctx, h.VMName) + if err != nil { + return h, rec, err + } + if !present || rec.Config.Name != h.VMName || rec.ID == "" || rec.LastTransitionReason == "" || rec.TransitionGeneration == 0 || rec.TransitionGeneration > ^uint64(0)-2 || rec.State != vmStateRunning || !exactSleepDisk(rec, h.Refs[0]) { + return h, rec, sleepRecoveryError() + } + p, err := m.managed.process(rec.PID) + if err != nil || p != h.Process { + return h, rec, sleepRecoveryError() + } + if err := m.eng.VerifyMountedVolume(ctx, rec.VsockSocket, h.Refs[0].Volume.Name, h.Refs[0].Volume.Mount); err != nil { + return h, rec, err + } + return h, rec, nil +} + +func (m *Manager) driveManagedSleepLocked(ctx context.Context, sb *types.Sandbox, target managedSleepTarget) error { + h, ok := m.managedHold(sb.ID) + if !ok { + return ErrVolumeCapture + } + m.mu.Lock() + live := m.claimed[sb.ID] == sb + m.mu.Unlock() + if !live { + return ErrUnknownSandbox + } + if err := m.ManagedVolumeStatus(); err != nil { + return err + } + if h.State != holdClaimed || !sleepMatchesClaim(h, sb) { + return sleepRecoveryError() + } + if _, outbound := m.view.Load().out.Resolve(sb, m.claimPooled); outbound { + return ErrVolumeCapture + } + ctx = context.WithoutCancel(ctx) + if h.Sleep == nil { + if err := m.beginManagedSleep(ctx, sb); err != nil { + return err + } + } + + for { + h, _ = m.managedHold(sb.ID) + if h.Sleep == nil { + return nil + } + if err := m.verifySleepBacking(h); err != nil { + return err + } + switch h.Sleep.Phase { + case sleepPreparing: + if err := m.setSleepPhase(h, sleepAborting); err != nil { + return err + } + case sleepAborting: + if err := m.resumeManagedSleep(ctx, sb, h, false); err != nil { + return err + } + if target == managedRunning { + return nil + } + return m.driveManagedSleepLocked(ctx, sb, target) + case sleepCapturing: + if err := m.captureManagedSleep(ctx, sb, h); err != nil { + return err + } + case sleepSleeping: + if err := m.verifySleepingVM(ctx, h); err != nil { + return err + } + if err := m.projectManagedSleep(sb, h); err != nil { + return err + } + if target == managedSleeping { + return nil + } + if err := m.setSleepPhase(h, sleepWaking); err != nil { + return err + } + case sleepWaking: + if err := m.wakeManagedSleep(ctx, sb, h); err != nil { + return err + } + case sleepResuming: + if err := m.resumeManagedSleep(ctx, sb, h, true); err != nil { + return err + } + if target == managedRunning { + return nil + } + return m.driveManagedSleepLocked(ctx, sb, target) + default: + return sleepRecoveryError() + } + } +} + +func (m *Manager) beginManagedSleep(ctx context.Context, sb *types.Sandbox) error { + h, rec, err := m.managedSleepPreflight(ctx, sb) + if err != nil { + return err + } + if err = m.store.commitDurable(m.store.mark()); err != nil { + return err + } + hook := *m.managedSleepHook + hook.HookArgv = slices.Clone(hook.HookArgv) + next := &managedSleep{Operation: randHex(16), Phase: sleepPreparing, Snapshot: hibernatePrefix + strings.TrimPrefix(sb.VMName, vmPrefix) + "-" + randHex(8), Hook: hook, VMID: rec.ID, Generation: rec.TransitionGeneration, BeforeRestore: h.Process} + if err = m.managed.commitManagedSleep(h, next); err != nil { + return err + } + h.Sleep = next + if err = m.projectManagedSleep(sb, h); err != nil { + return err + } + m.closeManagedStreams(sb) + m.out.Disarm(sb.ID, false) + return m.prepareManagedSleep(ctx, sb, h) +} + +func (m *Manager) setSleepPhase(h volumeHold, phase sleepPhase) error { + next := *h.Sleep + next.Phase = phase + return m.managed.commitManagedSleep(h, &next) +} + +func (m *Manager) prepareManagedSleep(ctx context.Context, sb *types.Sandbox, h volumeHold) error { + rec, err := m.sleepVM(ctx, h, false) + if err == nil { + err = m.sleepHook(ctx, h, rec.VsockSocket, "prepare") + } + if err == nil { + ref := h.Refs[0] + err = m.eng.VerifyMountedVolume(ctx, rec.VsockSocket, ref.Volume.Name, ref.Volume.Mount) + if err == nil { + err = m.eng.DetachVolume(ctx, h.VMName, rec.VsockSocket, ref.Volume.Name, ref.Volume.Mount) + } + } + if err == nil { + rec, err = m.sleepVM(ctx, h, false) + if err == nil && len(rec.AttachedDevices.Disks) != 0 { + err = sleepRecoveryError() + } + } + if err != nil { + if persistErr := m.setSleepPhase(h, sleepAborting); persistErr != nil { + return errors.Join(err, persistErr) + } + h, _ = m.managedHold(sb.ID) + return errors.Join(err, m.resumeManagedSleep(ctx, sb, h, false)) + } + return m.setSleepPhase(h, sleepCapturing) +} + +func (m *Manager) captureManagedSleep(ctx context.Context, sb *types.Sandbox, h volumeHold) error { + rec, found, err := m.eng.Inspect(ctx, h.VMName) + if err != nil { + return err + } + if !found || rec.ID != h.Sleep.VMID || rec.Config.Name != h.VMName { + return sleepRecoveryError() + } + source, exists, err := m.eng.InspectHibernateSource(ctx, h.Sleep.Snapshot) + if err != nil { + return err + } + if rec.State == vmStateRunning { + if exists || len(rec.AttachedDevices.Disks) != 0 { + return sleepRecoveryError() + } + return m.captureRunningManagedSleep(ctx, sb, h) + } + + if !found || rec.ID != h.Sleep.VMID || rec.Config.Name != h.VMName || rec.State != vmStateStopped || rec.TransitionGeneration != h.Sleep.Generation+1 || rec.LastTransitionReason != vmReasonStopUser || !exists || !m.managed.processGone(h.Sleep.BeforeRestore) { + return sleepRecoveryError() + } + next := *h.Sleep + next.Phase, next.SourceID = sleepSleeping, source.ID + if err = m.managed.commitManagedSleep(h, &next); err != nil { + return err + } + h.Sleep = &next + if err = m.projectManagedSleep(sb, h); err != nil { + return err + } + m.recordHibernate(ctx, sb) + return nil +} + +func (m *Manager) captureRunningManagedSleep(ctx context.Context, sb *types.Sandbox, h volumeHold) error { + if _, err := m.sleepVM(ctx, h, false); err != nil { + return err + } + effectErr := m.eng.Hibernate(ctx, h.VMName, h.Sleep.Snapshot) + rec, found, err := m.eng.Inspect(ctx, h.VMName) + if err != nil { + return errors.Join(effectErr, err) + } + _, exists, err := m.eng.InspectHibernateSource(ctx, h.Sleep.Snapshot) + if err != nil { + return errors.Join(effectErr, err) + } + if !found { + return errors.Join(effectErr, sleepRecoveryError()) + } + if rec.State != vmStateRunning { + return m.captureManagedSleep(ctx, sb, h) + } + if exists { + return errors.Join(effectErr, sleepRecoveryError()) + } + if err = m.setSleepPhase(h, sleepAborting); err != nil { + return errors.Join(effectErr, err) + } + h, _ = m.managedHold(sb.ID) + return errors.Join(effectErr, sleepRecoveryError(), m.resumeManagedSleep(ctx, sb, h, false)) +} + +func (m *Manager) wakeManagedSleep(ctx context.Context, sb *types.Sandbox, h volumeHold) error { + rec, err := m.restoreManagedSleep(ctx, h) + if err != nil { + return err + } + if h.Sleep.Restored == (processIdentity{}) { + p, identityErr := m.managed.process(rec.PID) + if identityErr != nil { + return identityErr + } + next := *h.Sleep + next.Restored = p + h.Process = p + if err = m.managed.commitManagedSleep(h, &next); err != nil { + return err + } + h.Sleep = &next + } + sock, err := m.probeReady(ctx, h.VMName, rec.VsockSocket, claimProbeTimeout) + if err != nil { + return err + } + if err = m.attachManagedSleep(ctx, h, sock, true); err != nil { + return err + } + m.mu.Lock() + sb.VsockSocket = sock + m.mu.Unlock() + return m.setSleepPhase(h, sleepResuming) +} + +func (m *Manager) resumeManagedSleep(ctx context.Context, sb *types.Sandbox, h volumeHold, restored bool) error { + rec, err := m.sleepVM(ctx, h, restored) + if err != nil { + return err + } + if err = m.attachManagedSleep(ctx, h, rec.VsockSocket, restored); err != nil { + return err + } + if err = m.sleepHook(ctx, h, rec.VsockSocket, "resume"); err != nil { + return err + } + sock, err := m.probeReady(ctx, h.VMName, rec.VsockSocket, claimProbeTimeout) + if err != nil { + return err + } + rec, err = m.sleepVM(ctx, h, restored) + if err != nil { + return err + } + if !exactSleepDisk(rec, h.Refs[0]) { + return sleepRecoveryError() + } + if err = m.eng.VerifyMountedVolume(ctx, sock, h.Refs[0].Volume.Name, h.Refs[0].Volume.Mount); err != nil { + return err + } + m.mu.Lock() + sb.VsockSocket, sb.HibernateSnap, sb.PendingSnap, sb.ReaderSafe = sock, "", "", false + js := m.store.set(sb) + m.mu.Unlock() + if err = m.store.commitDurable(js); err != nil { + m.pauseManagedSleep(sb, h) + return err + } + m.mu.Lock() + if _, outbound := m.view.Load().out.Resolve(sb, m.claimPooledLocked); outbound { + err = ErrVolumeCapture + } else { + err = m.managed.commitManagedSleep(h, nil) + } + m.mu.Unlock() + if err != nil { + m.pauseManagedSleep(sb, h) + return err + } + m.rotateManagedStreams(sb) + if restored { + m.counters.wakes.Add(1) + m.recordUsage(ctx, usageEvent{Event: "wake", ID: sb.ID, VMName: sb.VMName}) + } + m.dropSnap(ctx, h.Sleep.Snapshot) + return nil +} + +func (m *Manager) pauseManagedSleep(sb *types.Sandbox, h volumeHold) claimSnapshot { + m.mu.Lock() + defer m.mu.Unlock() + projectSleepState(sb, h.Sleep) + return m.store.set(sb) +} + +func (m *Manager) projectManagedSleep(sb *types.Sandbox, h volumeHold) error { + if err := m.store.commitDurable(m.pauseManagedSleep(sb, h)); err != nil { + return fmt.Errorf("persist managed sleep: %w", err) + } + return nil +} + +func (m *Manager) closeManagedStreams(sb *types.Sandbox) { + m.mu.Lock() + defer m.mu.Unlock() + if ch := m.managedStreams[sb]; ch != nil { + select { + case <-ch: + default: + close(ch) + } + } +} + +func (m *Manager) rotateManagedStreams(sb *types.Sandbox) { + m.mu.Lock() + defer m.mu.Unlock() + m.managedStreams[sb] = make(chan struct{}) +} diff --git a/sandboxd/pool/managed_sleep_boundaries_test.go b/sandboxd/pool/managed_sleep_boundaries_test.go new file mode 100644 index 00000000..1969a2d8 --- /dev/null +++ b/sandboxd/pool/managed_sleep_boundaries_test.go @@ -0,0 +1,322 @@ +package pool + +import ( + "context" + "errors" + "net" + "path/filepath" + "slices" + "testing" + "time" + + "github.com/cocoonstack/sandbox/sandboxd/config" + "github.com/cocoonstack/sandbox/sandboxd/egress" + "github.com/cocoonstack/sandbox/sandboxd/engine" + "github.com/cocoonstack/sandbox/sandboxd/outbound/outboundtest" +) + +func TestManagedSleepFailedClaimRefusesDataPlane(t *testing.T) { + for _, enabled := range []bool{false, true} { + t.Run(map[bool]string{false: "disabled", true: "enabled"}[enabled], func(t *testing.T) { + m, e, sb, _ := newSleepTest(t) + if !enabled { + m.managedSleepHook = nil + } + e.mu.Lock() + e.stopped[sb.VMName] = true + e.mu.Unlock() + m.recoverVMM(t.Context(), sb) + if sb.Failed == "" { + t.Fatal("missing VMM did not fail the mounted claim") + } + sock, _, release, err := m.WakeAgentSocket(t.Context(), sb.ID, sb.Token) + if release != nil { + release() + } + if !errors.Is(err, ErrFailed) || sock != "" { + t.Fatalf("failed claim exposed data plane: socket=%q err=%v", sock, err) + } + }) + } +} + +func TestManagedSleepOptInPreservesRunningEgress(t *testing.T) { + for _, enabled := range []bool{false, true} { + t.Run(map[bool]string{false: "disabled", true: "enabled"}[enabled], func(t *testing.T) { + cfg := managedConfig(t) + cfg.Networks = []string{"test-network"} + if enabled { + cfg.ManagedVolumeHibernate = &config.ManagedVolumeHibernate{HookArgv: []string{"/hook"}, HookTimeoutSeconds: 1} + } + e := newManagedEngine() + m := openManagedTest(t, cfg, e) + createManagedTest(t, m, "data") + opts := managedOptions("data") + opts.Volumes[0].AttachOnly = false + sb, err := m.ClaimProvision(t.Context(), egKey, opts) + if err != nil { + t.Fatal(err) + } + sock, _, release, err := m.WakeAgentSocket(t.Context(), sb.ID, sb.Token) + if release != nil { + release() + } + if err != nil || sock != sb.VsockSocket { + t.Fatalf("running egress claim lost data plane: socket=%q err=%v", sock, err) + } + }) + } +} + +func TestManagedSleepRefusesEffectiveOutboundBeforeEffects(t *testing.T) { + for _, armed := range []bool{false, true} { + t.Run(map[bool]string{false: "unbound", true: "working proxy"}[armed], func(t *testing.T) { + m, e, sb, _ := newSleepTest(t) + editEgress(t, m, func(cfg *config.Config) { + cfg.Pools[0].Egress = &egress.Policy{Allow: []egress.Rule{{Host: "example.com"}}} + cfg.EgressClasses = []config.EgressClass{{Name: "desk", Egress: cfg.Pools[0].Egress}} + }) + sb.EgressClass = "desk" + sock := filepath.Join(outboundtest.SockRoot(t), "v") + sb.VsockSocket = sock + e.mu.Lock() + e.vms[sb.VMName] = sock + e.mu.Unlock() + if armed { + if err := m.out.ArmProxy(t.Context(), sb); err != nil { + t.Fatal(err) + } + t.Cleanup(func() { m.out.Disarm(sb.ID, false) }) + requireProxyDoor(t, sock) + } + before := slices.Clone(e.volumeOps) + if err := m.Hibernate(t.Context(), sb.ID, Cred{Token: sb.Token}); !errors.Is(err, ErrVolumeCapture) { + t.Fatalf("effective outbound policy accepted for sleep: %v", err) + } + if armed { + requireProxyDoor(t, sock) + } + h, _ := m.managedHold(sb.ID) + if h.Sleep != nil || len(e.hooks) != 0 || len(e.snapshots) != 0 || !slices.Equal(before, e.volumeOps) { + t.Fatal("outbound refusal had lifecycle effects") + } + got, _, release, err := m.WakeAgentSocket(t.Context(), sb.ID, sb.Token) + if release != nil { + release() + } + if err != nil || got != sock { + t.Fatalf("outbound refusal lost running guest: socket=%q err=%v", got, err) + } + }) + } +} + +func requireProxyDoor(t *testing.T, sock string) { + t.Helper() + c, err := net.DialTimeout("unix", engine.EgressSocketPath(sock), time.Second) + if err != nil { + t.Fatalf("outbound proxy unavailable: %v", err) + } + c.Close() +} + +func TestManagedSleepPolicyReloadKeepsWakePaused(t *testing.T) { + m, e, sb, cfg := newSleepTest(t) + sb.EgressClass = "desk" + requireSleep(t, m, sb) + next := *cfg + next.Pools = slices.Clone(cfg.Pools) + next.Pools[0].Egress = &egress.Policy{Allow: []egress.Rule{{Host: "example.com"}}} + next.EgressClasses = []config.EgressClass{{Name: "desk", Egress: next.Pools[0].Egress}} + if _, err := m.ReloadConfig(t.Context(), &next); err != nil { + t.Fatal(err) + } + before := slices.Clone(e.volumeOps) + hooks := len(e.hooks) + if err := m.Wake(t.Context(), sb.ID, Cred{Token: sb.Token}); !errors.Is(err, ErrVolumeCapture) { + t.Fatalf("wake accepted new outbound policy: %v", err) + } + if _, err := m.AgentSocket(sb.ID, sb.Token); !errors.Is(err, ErrPaused) { + t.Fatalf("policy-blocked wake exposed guest: %v", err) + } + if len(e.restores) != 0 || len(e.hooks) != hooks || !slices.Equal(before, e.volumeOps) { + t.Fatal("policy-blocked wake touched guest") + } + cleared := next + cleared.EgressClasses = nil + if _, err := m.ReloadConfig(t.Context(), &cleared); err != nil { + t.Fatal(err) + } + if err := m.Wake(t.Context(), sb.ID, Cred{Token: sb.Token}); err != nil { + t.Fatal(err) + } +} + +type sleepUnmountGate struct { + *sleepTestEngine + entered, resume chan struct{} +} + +func (e *sleepUnmountGate) UnmountVolume(ctx context.Context, sock, mount string) error { + close(e.entered) + <-e.resume + return e.sleepTestEngine.UnmountVolume(ctx, sock, mount) +} + +func TestManagedSleepTeardownRefusesRenewal(t *testing.T) { + for _, expiry := range []bool{false, true} { + t.Run(map[bool]string{false: "release", true: "expiry"}[expiry], func(t *testing.T) { + m, e, sb, cfg := newSleepTest(t) + gate := &sleepUnmountGate{sleepTestEngine: e, entered: make(chan struct{}), resume: make(chan struct{})} + m.eng = gate + done := make(chan error, 1) + if expiry { + m.mu.Lock() + sb.Deadline = time.Now().Add(-time.Minute) + m.mu.Unlock() + m.reapOnce(t.Context()) + } else { + go func() { done <- m.Release(t.Context(), sb.ID, Cred{Token: sb.Token}) }() + } + select { + case <-gate.entered: + case <-time.After(time.Second): + close(gate.resume) + t.Fatal("release did not reach teardown") + } + _, renewErr := m.Renew(t.Context(), sb.ID, Cred{Token: sb.Token}, time.Hour, "") + rows, loadErr := newClaimStore(cfg.DataDir, true).load() + _, heldDuringTeardown := m.managedHold(sb.ID) + close(gate.resume) + if !expiry { + if err := <-done; err != nil { + t.Fatal(err) + } + } + waitFor(t, func() bool { _, held := m.managedHold(sb.ID); return !held }) + if !errors.Is(renewErr, ErrUnknownSandbox) { + t.Errorf("teardown accepted renewal it discarded: %v", renewErr) + } + if loadErr != nil || rows[sb.ID] != nil { + t.Errorf("claim still durable during guest teardown: rows=%v err=%v", rows, loadErr) + } + if !heldDuringTeardown { + t.Fatal("backing ownership freed before guest teardown") + } + }) + } +} + +func TestManagedSleepExpiryHonorsRenewalWhileTransitionBusy(t *testing.T) { + m, e, sb, cfg := newSleepTest(t) + sb.Transition.Lock() + m.mu.Lock() + sb.Deadline = time.Now().Add(-time.Minute) + m.mu.Unlock() + before := slices.Clone(e.volumeOps) + m.reapOnce(t.Context()) + waitFor(t, func() bool { return len(m.maintSem) > 0 }) + deadline, err := m.Renew(t.Context(), sb.ID, Cred{Token: sb.Token}, time.Hour, "") + sb.Transition.Unlock() + waitFor(t, func() bool { return len(m.maintSem) == 0 }) + if err != nil { + t.Fatal(err) + } + rows, err := newClaimStore(cfg.DataDir, true).load() + if err != nil || rows[sb.ID] == nil || !rows[sb.ID].Deadline.Equal(deadline) { + t.Fatalf("expiry discarded accepted renewal: rows=%v err=%v", rows, err) + } + if _, ok := m.Sandbox(sb.ID); !ok || !slices.Equal(before, e.volumeOps) { + t.Fatal("expiry tore down renewed claim") + } +} + +func TestManagedSleepReleasePersistenceFailureKeepsGuestAndHold(t *testing.T) { + m, e, sb, _ := newSleepTest(t) + before := slices.Clone(e.volumeOps) + path := m.store.path + m.store.path = filepath.Join(t.TempDir(), "missing", "claims.json") + if err := m.Release(t.Context(), sb.ID, Cred{Token: sb.Token}); err == nil { + t.Fatal("failed durable deletion reported success") + } + m.store.path = path + if _, ok := m.Sandbox(sb.ID); !ok || !slices.Equal(before, e.volumeOps) { + t.Fatal("failed durable deletion dropped claim or touched guest") + } + if _, held := m.managedHold(sb.ID); !held { + t.Fatal("failed durable deletion freed backing ownership") + } + m.managedSleepHook = nil + sock, _, release, err := m.WakeAgentSocket(t.Context(), sb.ID, sb.Token) + if release != nil { + release() + } + if err == nil || sock != "" { + t.Fatalf("retiring claim exposed guest with feature disabled: socket=%q err=%v", sock, err) + } + if err := m.Release(t.Context(), sb.ID, Cred{Token: sb.Token}); err != nil { + t.Fatal(err) + } +} + +func TestManagedSleepAllowsIneffectiveOutboundPolicy(t *testing.T) { + for _, noEgress := range []bool{false, true} { + t.Run(map[bool]string{false: "missing tenant policy", true: "no egress claim"}[noEgress], func(t *testing.T) { + m, _, sb, _ := newSleepTest(t) + editEgress(t, m, func(cfg *config.Config) { + cfg.Pools[0].Egress = &egress.Policy{Allow: []egress.Rule{{Host: "example.com"}}} + if noEgress { + cfg.EgressClasses = []config.EgressClass{{Name: "desk", Egress: cfg.Pools[0].Egress}} + } + }) + sb.EgressClass, sb.NoEgress = "desk", noEgress + requireSleep(t, m, sb) + if err := m.Wake(t.Context(), sb.ID, Cred{Token: sb.Token}); err != nil { + t.Fatal(err) + } + }) + } +} + +func TestManagedSleepPolicyReloadDuringResumeKeepsGuestPaused(t *testing.T) { + m, e, sb, cfg := newSleepTest(t) + sb.EgressClass = "desk" + requireSleep(t, m, sb) + e.hookEntered = make(chan struct{}, 1) + e.hookContinue = make(chan struct{}) + done := make(chan error, 1) + go func() { done <- m.Wake(t.Context(), sb.ID, Cred{Token: sb.Token}) }() + select { + case <-e.hookEntered: + case <-time.After(time.Second): + close(e.hookContinue) + t.Fatal("wake did not reach resume hook") + } + next := *cfg + next.Pools = slices.Clone(cfg.Pools) + next.Pools[0].Egress = &egress.Policy{Allow: []egress.Rule{{Host: "example.com"}}} + next.EgressClasses = []config.EgressClass{{Name: "desk", Egress: next.Pools[0].Egress}} + _, reloadErr := m.ReloadConfig(t.Context(), &next) + close(e.hookContinue) + wakeErr := <-done + if reloadErr != nil { + t.Fatal(reloadErr) + } + if !errors.Is(wakeErr, ErrVolumeCapture) { + t.Fatalf("resume published readiness after policy reload: %v", wakeErr) + } + if _, err := m.AgentSocket(sb.ID, sb.Token); !errors.Is(err, ErrPaused) { + t.Fatalf("policy-blocked resume exposed guest: %v", err) + } + cleared := next + cleared.EgressClasses = nil + if _, err := m.ReloadConfig(t.Context(), &cleared); err != nil { + t.Fatal(err) + } + if err := m.Wake(t.Context(), sb.ID, Cred{Token: sb.Token}); err != nil { + t.Fatal(err) + } + if len(e.restores) != 1 { + t.Fatal("policy retry replayed memory restore") + } +} diff --git a/sandboxd/pool/managed_sleep_recovery.go b/sandboxd/pool/managed_sleep_recovery.go new file mode 100644 index 00000000..aa77506a --- /dev/null +++ b/sandboxd/pool/managed_sleep_recovery.go @@ -0,0 +1,142 @@ +package pool + +import ( + "context" + "time" + + "github.com/cocoonstack/sandbox/sandboxd/types" +) + +func (m *Manager) managedSleepSources() map[string]bool { + sources := map[string]bool{} + if m.managed == nil { + return sources + } + m.managed.mu.Lock() + defer m.managed.mu.Unlock() + for _, h := range m.managed.data.Holds { + if h.Sleep != nil { + sources[h.Sleep.Snapshot] = true + } + } + return sources +} + +func (m *Manager) releaseManagedSleepLocked(ctx context.Context, sb *types.Sandbox, expiredAt time.Time) error { + m.mu.Lock() + if m.claimed[sb.ID] != sb { + m.mu.Unlock() + return ErrUnknownSandbox + } + if !expiredAt.IsZero() && (sb.Deadline.IsZero() || !expiredAt.After(sb.Deadline)) { + m.mu.Unlock() + return nil + } + if err := m.retireManaged(sb.ID); err != nil { + m.mu.Unlock() + return err + } + m.dropClaimLocked(sb) + js := m.store.del(sb.ID) + m.mu.Unlock() + if err := m.store.commitDurable(js); err != nil { + m.mu.Lock() + m.addClaimLocked(sb) + m.store.set(sb) + m.mu.Unlock() + m.closeManagedStreams(sb) + return err + } + ctx = context.WithoutCancel(ctx) + h, _ := m.managedHold(sb.ID) + td := volumeTeardown{managedID: sb.ID} + if h.Sleep == nil { + td = m.quiesceVolumes(ctx, sb) + } + removed := m.removeOrRetry(ctx, sb.VMName, sb.ID, td) + m.out.Disarm(sb.ID, removed) + if !removed { + return sleepRecoveryError() + } + m.counters.releases.Add(1) + m.recordUsage(ctx, usageEvent{Event: "release", ID: sb.ID, VMName: sb.VMName}) + return nil +} + +func (m *Manager) removeManagedSleep(ctx context.Context, h volumeHold) bool { + current, present, err := m.observeManagedRemoval(ctx, h) + if err != nil { + return false + } + h = current + if present { + _ = m.eng.Remove(ctx, h.VMName) + } + + if !m.confirmGone(ctx, h.VMName) || !m.managed.processGone(h.Process) || !m.managed.processGone(h.Sleep.BeforeRestore) { + return false + } + _ = m.eng.SnapshotRemove(ctx, h.Sleep.Snapshot) + _, exists, inspectErr := m.eng.InspectHibernateSource(ctx, h.Sleep.Snapshot) + if inspectErr != nil || exists { + return false + } + + return true +} + +func (m *Manager) observeManagedRemoval(ctx context.Context, h volumeHold) (volumeHold, bool, error) { + rec, present, err := m.eng.Inspect(ctx, h.VMName) + if err != nil || !present { + return h, present, err + } + if rec.Config.Name != h.VMName || rec.ID != h.Sleep.VMID { + return h, true, sleepRecoveryError() + } + if rec.State != vmStateRunning { + if !m.managed.processGone(h.Process) { + return h, true, sleepRecoveryError() + } + return h, true, nil + } + restored := h.Sleep.Phase == sleepWaking || h.Sleep.Phase == sleepResuming + if _, err = m.sleepVM(ctx, h, restored); err != nil { + return h, true, err + } + p, err := m.managed.process(rec.PID) + if err != nil || p == h.Process { + return h, true, err + } + r := m.managed + r.mu.Lock() + defer r.mu.Unlock() + h = r.data.Holds[h.SandboxID] + h.Process = p + next := *h.Sleep + next.Restored = p + h.Sleep = &next + r.data.Holds[h.SandboxID] = h + return h, true, r.commit() +} + +func (m *Manager) retryManagedSleepRemoval(ctx context.Context, name string, pending pendingRemoval) bool { + h, held := m.managedHold(pending.volumes.managedID) + if !held || h.Sleep == nil { + return false + } + m.mu.Lock() + sb := m.claimed[h.SandboxID] + m.mu.Unlock() + if sb != nil { + sb.Transition.Lock() + defer sb.Transition.Unlock() + } + if m.removeVM(ctx, name) { + m.finishRemoval(ctx, pending) + if _, held = m.managedHold(h.SandboxID); !held { + return true + } + } + m.queueRemoval(name, pending.sandboxID, pending.tap, pending.volumes, time.Time{}) + return true +} diff --git a/sandboxd/pool/managed_sleep_registry.go b/sandboxd/pool/managed_sleep_registry.go new file mode 100644 index 00000000..5e9225ba --- /dev/null +++ b/sandboxd/pool/managed_sleep_registry.go @@ -0,0 +1,144 @@ +package pool + +import ( + "errors" + "slices" + "strings" + + "github.com/cocoonstack/sandbox/sandboxd/config" + "github.com/cocoonstack/sandbox/sandboxd/types" +) + +type sleepPhase string + +const ( + sleepPreparing sleepPhase = "preparing" + sleepAborting sleepPhase = "aborting" + sleepCapturing sleepPhase = "capturing" + sleepSleeping sleepPhase = "sleeping" + sleepWaking sleepPhase = "waking" + sleepResuming sleepPhase = "resuming" +) + +type managedSleep struct { + Operation string `json:"operation"` + Phase sleepPhase `json:"phase"` + Snapshot string `json:"snapshot"` + SourceID string `json:"source_id,omitempty"` + Hook config.ManagedVolumeHibernate `json:"hook"` + VMID string `json:"vm_id"` + Generation uint64 `json:"generation"` + BeforeRestore processIdentity `json:"before_restore"` + Restored processIdentity `json:"restored,omitzero"` +} + +func validateManagedSleep(h volumeHold) error { + s := h.Sleep + if h.State == holdAttaching || h.VMName == "" || len(h.Refs) != 1 || h.Refs[0].ID == "" || !h.Refs[0].Volume.RW() || h.Refs[0].Volume.Mount == "" || s.Operation == "" || !strings.HasPrefix(s.Snapshot, hibernatePrefix) || s.VMID == "" || s.Generation == 0 || s.Generation > ^uint64(0)-2 || s.Hook.Validate() != nil || !validSleepProcess(s.BeforeRestore) { + return errors.New("invalid managed sleep record") + } + switch s.Phase { + case sleepPreparing, sleepAborting, sleepCapturing: + if s.Restored != (processIdentity{}) || h.Process != s.BeforeRestore { + return errors.New("invalid pre-restore identity") + } + case sleepSleeping: + if s.Restored != (processIdentity{}) || h.Process != s.BeforeRestore { + return errors.New("sleeping owner has restored process") + } + fallthrough + case sleepWaking, sleepResuming: + if s.SourceID == "" { + return errors.New("managed sleep lacks snapshot identity") + } + default: + return errors.New("invalid managed sleep phase") + } + if s.Restored != (processIdentity{}) && (s.Restored != h.Process || !validSleepProcess(s.Restored)) { + return errors.New("invalid restored process identity") + } + if s.Phase == sleepResuming && s.Restored == (processIdentity{}) { + return errors.New("resume lacks process identity") + } + return nil +} + +func (r *volumeRegistry) commitManagedSleep(h volumeHold, next *managedSleep) error { + r.mu.Lock() + defer r.mu.Unlock() + current, ok := r.data.Holds[h.SandboxID] + if r.fault != nil || !ok || current.State != holdClaimed || (current.Sleep != nil && (h.Sleep == nil || current.Sleep.Operation != h.Sleep.Operation)) { + return managedStorageError() + } + if r.data.Version == 1 { + r.data.Version = 2 + if err := r.commit(); err != nil { + return err + } + } + h.Sleep = next + if next != nil { + if err := validateManagedSleep(h); err != nil { + return err + } + } + r.data.Holds[h.SandboxID] = h + if err := r.commit(); err != nil { + if next == nil { + h.Sleep = current.Sleep + r.data.Holds[h.SandboxID] = h + } + return err + } + return nil +} + +func sleepMatchesClaim(h volumeHold, sb *types.Sandbox) bool { + return sb != nil && sb.ID == h.SandboxID && sb.VMName == h.VMName && sb.Tenant == h.Tenant && len(h.Refs) == 1 && slices.Equal(sb.Volumes, []types.Volume{h.Refs[0].Volume}) && slices.Equal(sb.VolumesWant, sb.Volumes) && sb.ArchiveCk == "" && sb.ArchivePending == "" && sb.ReplicaCk == "" && sb.ReplicaPending == "" && !sb.Replicated() && sb.Key.Net == types.NetNone +} + +func (m *Manager) verifySleepBacking(h volumeHold) error { + r := m.managed + r.mu.Lock() + defer r.mu.Unlock() + if r.fault != nil { + return managedStorageError() + } + ref := h.Refs[0] + v, ok := r.data.Volumes[ref.Volume.Name] + if !ok || v.ID != ref.ID || v.State != volumeReady || ref.Path != r.path(v.ID) || ref.Backing != v.Backing || (h.Tenant != "" && v.Tenant != h.Tenant) || r.verify(v) != nil { + return managedStorageError() + } + return nil +} + +func (m *Manager) managedLifecycle(sb *types.Sandbox) bool { + h, ok := m.managedHold(sb.ID) + return ok && (h.Sleep != nil || (m.managedSleepHook != nil && len(h.Refs) == 1 && h.Refs[0].ID != "" && h.Refs[0].Volume.RW() && h.Refs[0].Volume.Mount != "")) +} + +func validSleepProcess(p processIdentity) bool { return p.PID > 0 && p.Start > 0 && p.BootID != "" } + +func (m *Manager) managedPaused(sb *types.Sandbox) bool { + h, held := m.managedHold(sb.ID) + return held && (h.Sleep != nil || h.State == holdRetiring) +} + +func (r *volumeRegistry) sleepOwnsVolume(id volumeID) bool { + for _, h := range r.data.Holds { + if h.Sleep != nil && h.Refs[0].ID == id { + return true + } + } + return false +} + +func projectSleepState(sb *types.Sandbox, s *managedSleep) { + sb.ReaderSafe = false + switch s.Phase { + case sleepPreparing, sleepAborting, sleepCapturing: + sb.PendingSnap, sb.HibernateSnap = s.Snapshot, "" + default: + sb.HibernateSnap, sb.PendingSnap = s.Snapshot, "" + } +} diff --git a/sandboxd/pool/managed_sleep_test.go b/sandboxd/pool/managed_sleep_test.go new file mode 100644 index 00000000..d1bbd3c3 --- /dev/null +++ b/sandboxd/pool/managed_sleep_test.go @@ -0,0 +1,851 @@ +package pool + +import ( + "context" + "encoding/json/v2" + "errors" + "os" + "path/filepath" + "slices" + "strings" + "sync" + "testing" + "time" + + "github.com/cocoonstack/sandbox/sandboxd/config" + "github.com/cocoonstack/sandbox/sandboxd/engine" + "github.com/cocoonstack/sandbox/sandboxd/tenants" + "github.com/cocoonstack/sandbox/sandboxd/types" +) + +type sleepTestEngine struct { + *managedTestEngine + sleepMu sync.Mutex + generation uint64 + reason string + vmID string + hooks []engine.VolumeHookCall + hookErr string + hookEntered, hookContinue chan struct{} + restoreAfterErr bool + mounted bool +} + +func newSleepTest(t *testing.T) (*Manager, *sleepTestEngine, *types.Sandbox, *config.Config) { + t.Helper() + cfg := managedConfig(t) + cfg.ManagedVolumeHibernate = &config.ManagedVolumeHibernate{HookArgv: []string{"/usr/local/bin/volume-hook"}, HookTimeoutSeconds: 1} + e := &sleepTestEngine{managedTestEngine: newManagedEngine(), generation: 2, reason: "boot", vmID: "immutable-vm", mounted: true} + m, err := NewManager(t.Context(), cfg, e, testSecrets(t), testBackends(t, cfg)) + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { _ = m.Close() }) + m.managed.process = e.identity + if err = m.PutTenant(t.Context(), tenants.TenantSpec{Name: "t1", Token: "t1-token"}); err != nil { + t.Fatal(err) + } + createManagedTest(t, m, "data") + opts := managedOptions("data") + opts.Volumes[0].AttachOnly = false + sb, err := m.ClaimProvision(t.Context(), testKey, opts) + if err != nil { + t.Fatal(err) + } + return m, e, sb, cfg +} + +func (e *sleepTestEngine) identity(pid int) (processIdentity, error) { + e.mu.Lock() + defer e.mu.Unlock() + for name, p := range e.pids { + if p == pid && !e.stopped[name] { + if _, exists := e.vms[name]; exists { + return processIdentity{PID: pid, Start: uint64(pid), BootID: "test-boot"}, nil + } + } + } + return processIdentity{}, os.ErrNotExist +} + +func (e *sleepTestEngine) Inspect(ctx context.Context, name string) (types.VMRecord, bool, error) { + rec, found, err := e.managedTestEngine.Inspect(ctx, name) + e.sleepMu.Lock() + defer e.sleepMu.Unlock() + rec.ID, rec.TransitionGeneration, rec.LastTransitionReason = e.vmID, e.generation, e.reason + return rec, found, err +} + +func (e *sleepTestEngine) DiskAttach(ctx context.Context, name string, spec engine.VolumeSpec) error { + if err := e.managedTestEngine.DiskAttach(ctx, name, spec); err != nil { + return err + } + e.mu.Lock() + if e.disks == nil { + e.disks = map[string][]types.VMDisk{} + } + e.disks[name] = []types.VMDisk{{Name: spec.Name, Path: spec.Path, ReadOnly: !spec.RW}} + e.mu.Unlock() + return nil +} + +func (e *sleepTestEngine) DetachVolume(ctx context.Context, name, sock, serial, mount string) error { + if err := e.fakeEngine.DetachVolume(ctx, name, sock, serial, mount); err != nil { + return err + } + e.mu.Lock() + e.disks[name] = nil + e.mu.Unlock() + e.sleepMu.Lock() + e.mounted = false + e.sleepMu.Unlock() + return nil +} + +func (e *sleepTestEngine) MountLiveVolume(ctx context.Context, sock, name, mount string, rw bool) error { + if err := e.fakeEngine.MountLiveVolume(ctx, sock, name, mount, rw); err != nil { + return err + } + e.sleepMu.Lock() + e.mounted = true + e.sleepMu.Unlock() + return nil +} + +func (e *sleepTestEngine) VerifyMountedVolume(context.Context, string, string, string) error { + e.sleepMu.Lock() + defer e.sleepMu.Unlock() + if !e.mounted { + return errors.New("mount absent") + } + return nil +} + +func (e *sleepTestEngine) Hibernate(ctx context.Context, name, snap string) error { + err := e.fakeEngine.Hibernate(ctx, name, snap) + e.mu.Lock() + stopped := e.stopped[name] + e.mu.Unlock() + if stopped { + e.sleepMu.Lock() + e.generation++ + e.reason = "stop-user" + e.sleepMu.Unlock() + } + return err +} + +func (e *sleepTestEngine) Restore(ctx context.Context, name, snap string) (string, error) { + sock, err := e.fakeEngine.Restore(ctx, name, snap) + if err != nil { + return sock, err + } + e.mu.Lock() + e.pids[name]++ + e.mu.Unlock() + e.sleepMu.Lock() + e.generation++ + e.reason = "restore" + postErr := e.restoreAfterErr + e.sleepMu.Unlock() + if postErr { + return "", errors.New("restore reply lost") + } + return sock, nil +} + +func (e *sleepTestEngine) RunVolumeHook(_ context.Context, _ string, call engine.VolumeHookCall) error { + e.sleepMu.Lock() + e.hooks = append(e.hooks, call) + fail := e.hookErr == call.Action + entered, resume := e.hookEntered, e.hookContinue + e.sleepMu.Unlock() + if entered != nil { + entered <- struct{}{} + <-resume + } + if fail { + return errors.New("hook failed") + } + return nil +} + +func requireSleep(t *testing.T, m *Manager, sb *types.Sandbox) { + t.Helper() + if err := m.Hibernate(t.Context(), sb.ID, Cred{Token: sb.Token}); err != nil { + t.Fatal(err) + } + h, _ := m.managedHold(sb.ID) + if h.Sleep == nil || h.Sleep.Phase != sleepSleeping { + t.Fatalf("not sleeping: %+v", h) + } +} + +func TestManagedSleepRoundTripAndExclusiveHold(t *testing.T) { + m, e, sb, _ := newSleepTest(t) + _, fenced, release, err := m.WakeAgentSocket(t.Context(), sb.ID, sb.Token) + if err != nil { + t.Fatal(err) + } + defer release() + requireSleep(t, m, sb) + select { + case <-fenced: + default: + t.Fatal("existing stream not fenced") + } + _, err = m.ClaimProvision(t.Context(), testKey, managedOptions("data")) + requireVolumeStatus(t, err, 409) + _, err = m.AgentSocket(sb.ID, sb.Token) + if !errors.Is(err, ErrPaused) { + t.Fatalf("passive socket = %v", err) + } + if err = m.Wake(t.Context(), sb.ID, Cred{Token: sb.Token}); err != nil { + t.Fatal(err) + } + _, fresh, unhold, err := m.WakeAgentSocket(t.Context(), sb.ID, sb.Token) + if err != nil { + t.Fatal(err) + } + defer unhold() + select { + case <-fresh: + t.Fatal("new stream inherited old fence") + default: + } + h, _ := m.managedHold(sb.ID) + if h.Sleep != nil || h.Process.PID == 1001 { + t.Fatalf("wake not bound: %+v", h) + } + if len(e.restores) != 1 || len(e.hooks) != 2 || e.hooks[0].Operation != e.hooks[1].Operation { + t.Fatalf("effects restores=%v hooks=%v", e.restores, e.hooks) + } +} + +func TestManagedSleepAbortKeepsOriginalVM(t *testing.T) { + for _, failure := range []string{"prepare", "unmount", "detach", "capture"} { + t.Run(failure, func(t *testing.T) { + m, e, sb, _ := newSleepTest(t) + switch failure { + case "prepare": + e.hookErr = "prepare" + case "unmount": + e.unmountVolumeErr = errors.New("busy") + case "detach": + e.detachVolumeErr = errors.New("detach failed") + case "capture": + e.hibernateErr = errors.New("capture refused") + } + if err := m.Hibernate(t.Context(), sb.ID, Cred{Token: sb.Token}); err == nil { + t.Fatal("failure hidden") + } + h, _ := m.managedHold(sb.ID) + if h.Sleep != nil || e.stopped[sb.VMName] || len(e.restores) != 0 { + t.Fatalf("failed preparation damaged VM: %+v", h) + } + if failure != "capture" && len(e.hibernates) != 0 { + t.Fatal("captured after failed preparation") + } + if e.hooks[len(e.hooks)-1].Action != "resume" { + t.Fatal("writers not resumed") + } + }) + } +} + +func TestManagedSleepWakeFailuresKeepRestoredProcess(t *testing.T) { + for _, failure := range []string{"restore reply", "attach", "mount", "resume"} { + t.Run(failure, func(t *testing.T) { + m, e, sb, _ := newSleepTest(t) + requireSleep(t, m, sb) + switch failure { + case "restore reply": + e.restoreAfterErr = true + case "attach": + e.diskAttachErr = errors.New("attach unavailable") + case "mount": + e.mountVolumeErr = errors.New("mount failed") + case "resume": + e.hookErr = "resume" + } + err := m.Wake(t.Context(), sb.ID, Cred{Token: sb.Token}) + if failure != "restore reply" && err == nil { + t.Fatal("wake failure hidden") + } + if e.stopped[sb.VMName] || len(e.restores) != 1 { + t.Fatal("restored VM was discarded") + } + e.diskAttachErr, e.mountVolumeErr = nil, nil + e.hookErr = "" + if err := m.Wake(t.Context(), sb.ID, Cred{Token: sb.Token}); err != nil { + t.Fatal(err) + } + if len(e.restores) != 1 { + t.Fatal("restore replayed over live VM") + } + }) + } +} + +func TestManagedSleepBackingRefusesBeforeRestore(t *testing.T) { + for _, replace := range []bool{false, true} { + t.Run(map[bool]string{false: "missing", true: "replaced"}[replace], func(t *testing.T) { + m, e, sb, _ := newSleepTest(t) + requireSleep(t, m, sb) + h, _ := m.managedHold(sb.ID) + path := h.Refs[0].Path + saved := filepath.Join(filepath.Dir(filepath.Dir(path)), "saved.raw") + if err := os.Rename(path, saved); err != nil { + t.Fatal(err) + } + if replace { + if err := os.WriteFile(path, nil, 0o600); err != nil { + t.Fatal(err) + } + } + if err := m.Wake(t.Context(), sb.ID, Cred{Token: sb.Token}); err == nil || len(e.restores) != 0 { + t.Fatal("restore ran without exact backing") + } + info, err := m.GetVolume("data", "t1") + if err != nil || info.State != string(volumeRecovery) || info.Available { + t.Fatalf("missing backing detail: %+v %v", info, err) + } + infos := m.Volumes("t1", nil) + if len(infos) != 1 || infos[0].Available { + t.Fatalf("missing backing list: %+v", infos) + } + if _, _, err = m.CreateVolume(t.Context(), "data", types.VolumeCreate{SizeGiB: 1}, "t1"); err == nil { + t.Fatal("missing backing create replay succeeded") + } + if _, err = m.ClaimProvision(t.Context(), testKey, managedOptions("data")); err == nil { + t.Fatal("missing backing admitted second writer") + } + if replace { + if err := os.Remove(path); err != nil { + t.Fatal(err) + } + } + if err := os.Rename(saved, path); err != nil { + t.Fatal(err) + } + if err := m.Wake(t.Context(), sb.ID, Cred{Token: sb.Token}); err != nil { + t.Fatal(err) + } + }) + } +} + +func TestManagedSleepProvenanceRejectsReplacement(t *testing.T) { + for _, mismatch := range []string{"id", "generation", "process", "disk path", "disk name"} { + t.Run(mismatch, func(t *testing.T) { + m, e, sb, _ := newSleepTest(t) + requireSleep(t, m, sb) + e.hookErr = "resume" + if err := m.Wake(t.Context(), sb.ID, Cred{Token: sb.Token}); err == nil { + t.Fatal("expected suspended resume") + } + switch mismatch { + case "id": + e.vmID = "replacement" + case "generation": + e.generation++ + case "process": + e.pids[sb.VMName]++ + case "disk path": + e.disks[sb.VMName][0].Path = "/wrong" + case "disk name": + e.disks[sb.VMName][0].Name = "other" + } + e.hookErr = "" + if err := m.Wake(t.Context(), sb.ID, Cred{Token: sb.Token}); err == nil { + t.Fatal("adopted replacement") + } + if len(e.restores) != 1 { + t.Fatal("replayed restore") + } + }) + } +} + +func TestManagedSleepReleasePreservesBacking(t *testing.T) { + m, e, sb, _ := newSleepTest(t) + requireSleep(t, m, sb) + h, _ := m.managedHold(sb.ID) + before := slices.Clone(e.volumeOps) + if err := m.Release(t.Context(), sb.ID, Cred{Token: sb.Token}); err != nil { + t.Fatal(err) + } + if _, held := m.managedHold(sb.ID); held { + t.Fatal("hold not released") + } + if _, err := os.Stat(h.Refs[0].Path); err != nil { + t.Fatal(err) + } + if len(e.snapshots) != 0 { + t.Fatal("sleep snapshot leaked") + } + if !slices.Equal(before, e.volumeOps) { + t.Fatal("asleep release touched guest") + } +} + +func TestManagedSleepRestartEveryPhase(t *testing.T) { + for _, phase := range []sleepPhase{sleepPreparing, sleepAborting, sleepCapturing, sleepSleeping, sleepWaking, sleepResuming} { + t.Run(string(phase), func(t *testing.T) { + m, e, sb, cfg := newSleepTest(t) + original, _ := m.managedHold(sb.ID) + requireSleep(t, m, sb) + h, _ := m.managedHold(sb.ID) + next := *h.Sleep + next.Phase = phase + if phase == sleepPreparing || phase == sleepAborting || phase == sleepCapturing { + e.mu.Lock() + e.stopped[sb.VMName] = false + if phase != sleepCapturing { + e.disks[sb.VMName] = []types.VMDisk{{Name: "data", Path: h.Refs[0].Path}} + } + e.snapshots = nil + e.mu.Unlock() + e.sleepMu.Lock() + e.generation = 2 + e.reason = "boot" + e.mounted = phase != sleepCapturing + e.sleepMu.Unlock() + next.SourceID = "" + } + if phase == sleepResuming { + if _, err := e.Restore(t.Context(), sb.VMName, next.Snapshot); err != nil { + t.Fatal(err) + } + rec, _, _ := e.Inspect(t.Context(), sb.VMName) + p, err := e.identity(rec.PID) + if err != nil { + t.Fatal(err) + } + next.Restored = p + h.Process = p + } + if err := m.managed.commitManagedSleep(h, &next); err != nil { + t.Fatal(err) + } + if err := m.Close(); err != nil { + t.Fatal(err) + } + cfg.ManagedVolumeHibernate = nil + recovered, err := NewManager(t.Context(), cfg, e, testSecrets(t), testBackends(t, cfg)) + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { _ = recovered.Close() }) + recovered.managed.process = e.identity + if err = recovered.Reconcile(t.Context()); err != nil { + t.Fatal(err) + } + if _, err = recovered.AgentSocket(sb.ID, sb.Token); !errors.Is(err, ErrPaused) { + t.Fatalf("restart bypassed journal: %v", err) + } + if _, held := recovered.managedHold(sb.ID); !held { + t.Fatal("restart freed sleep owner") + } + if err = recovered.Wake(t.Context(), sb.ID, Cred{Token: sb.Token}); err != nil { + t.Fatal(err) + } + got, _ := recovered.managedHold(sb.ID) + if got.Sleep != nil || got.Refs[0] != original.Refs[0] { + t.Fatalf("recovery changed backing: %+v", got) + } + if !slices.Equal(e.hooks[len(e.hooks)-1].Argv, []string{"/usr/local/bin/volume-hook"}) { + t.Fatal("recovery lost pinned hook") + } + }) + } +} + +func TestManagedSleepPostCaptureErrorAdoptsAndPersists(t *testing.T) { + m, e, sb, _ := newSleepTest(t) + e.hibernateErr = errors.New("reply lost") + e.hibernateErrCompletes = true + requireSleep(t, m, sb) + if err := m.Wake(t.Context(), sb.ID, Cred{Token: sb.Token}); err != nil { + t.Fatal(err) + } +} + +func TestManagedSleepJournalFailureStopsEffects(t *testing.T) { + for _, phase := range []sleepPhase{sleepPreparing, sleepCapturing, sleepSleeping, sleepWaking, sleepResuming} { + t.Run(string(phase), func(t *testing.T) { + m, e, sb, _ := newSleepTest(t) + if phase == sleepWaking || phase == sleepResuming { + requireSleep(t, m, sb) + } + commit := m.managed.commitFile + m.managed.commitFile = func(path string, raw []byte) error { + if strings.Contains(string(raw), `"phase":"`+string(phase)+`"`) { + return errors.New("injected journal write failure") + } + return commit(path, raw) + } + var err error + if phase == sleepWaking || phase == sleepResuming { + err = m.Wake(t.Context(), sb.ID, Cred{Token: sb.Token}) + } else { + err = m.Hibernate(t.Context(), sb.ID, Cred{Token: sb.Token}) + } + if err == nil { + t.Fatal("persistence failure hidden") + } + beforeHooks, beforeRestores, beforeCaptures := len(e.hooks), len(e.restores), len(e.hibernates) + if err := m.Wake(t.Context(), sb.ID, Cred{Token: sb.Token}); err == nil { + t.Fatal("poisoned registry served guest") + } + if len(e.hooks) != beforeHooks || len(e.restores) != beforeRestores || len(e.hibernates) != beforeCaptures { + t.Fatal("effect after poisoned registry") + } + if phase == sleepPreparing && len(e.hooks) != 0 { + t.Fatal("hook preceded durable intent") + } + if phase == sleepCapturing && len(e.hibernates) != 0 { + t.Fatal("capture preceded durable intent") + } + if phase == sleepWaking && len(e.restores) != 0 { + t.Fatal("restore preceded durable intent") + } + if phase == sleepResuming && e.hooks[len(e.hooks)-1].Action == "resume" { + t.Fatal("resume preceded durable intent") + } + }) + } +} + +func TestManagedSleepUnknownSourceAndDeadResumingRefuse(t *testing.T) { + for _, failure := range []string{"missing source", "source identity", "dead resuming"} { + t.Run(failure, func(t *testing.T) { + m, e, sb, _ := newSleepTest(t) + requireSleep(t, m, sb) + switch failure { + case "missing source": + e.snapshots = nil + case "source identity": + h, _ := m.managedHold(sb.ID) + next := *h.Sleep + next.SourceID = "other" + if err := m.managed.commitManagedSleep(h, &next); err != nil { + t.Fatal(err) + } + case "dead resuming": + e.hookErr = "resume" + if err := m.Wake(t.Context(), sb.ID, Cred{Token: sb.Token}); err == nil { + t.Fatal("expected failed resume") + } + e.stopped[sb.VMName] = true + } + restores := len(e.restores) + if err := m.Wake(t.Context(), sb.ID, Cred{Token: sb.Token}); err == nil { + t.Fatal("uncertain source served guest") + } + if len(e.restores) != restores { + t.Fatal("unsafe memory replay") + } + }) + } +} + +func TestManagedSleepReleaseWaitsForPrepare(t *testing.T) { + m, e, sb, _ := newSleepTest(t) + _, fenced, release, err := m.WakeAgentSocket(t.Context(), sb.ID, sb.Token) + if err != nil { + t.Fatal(err) + } + defer release() + e.hookEntered = make(chan struct{}, 2) + e.hookContinue = make(chan struct{}) + hibernate := make(chan error, 1) + released := make(chan error, 1) + go func() { hibernate <- m.Hibernate(t.Context(), sb.ID, Cred{Token: sb.Token}) }() + <-e.hookEntered + select { + case <-fenced: + default: + t.Fatal("prepare started before existing streams were fenced") + } + + if info, ok := m.Sandbox(sb.ID); !ok || info.Hibernated { + t.Fatal("preparing was reported as successfully hibernated") + } + if _, err := m.AgentSocket(sb.ID, sb.Token); !errors.Is(err, ErrPaused) { + t.Fatalf("prepare exposed guest: %v", err) + } + go func() { released <- m.Release(t.Context(), sb.ID, Cred{Token: sb.Token}) }() + select { + case err := <-released: + t.Fatalf("release overtook prepare: %v", err) + case <-time.After(20 * time.Millisecond): + } + close(e.hookContinue) + if err := <-hibernate; err != nil { + t.Fatal(err) + } + if err := <-released; err != nil { + t.Fatal(err) + } + if _, held := m.managedHold(sb.ID); held { + t.Fatal("release retained completed hold") + } +} + +func TestManagedSleepRegistryUpgradeAndCorruption(t *testing.T) { + for _, corruption := range []string{"phase", "version downgrade", "missing VM ID", "invalid hook", "sleeping restored process"} { + t.Run(corruption, func(t *testing.T) { + m, _, sb, cfg := newSleepTest(t) + if m.managed.data.Version != 1 { + t.Fatal("new store unexpectedly upgraded") + } + requireSleep(t, m, sb) + if m.managed.data.Version != 2 { + t.Fatal("sleep did not fence old registry readers") + } + if err := m.Close(); err != nil { + t.Fatal(err) + } + path := filepath.Join(cfg.ManagedVolumeDir, "registry.json") + raw, err := os.ReadFile(path) + if err != nil { + t.Fatal(err) + } + var data volumeRegistryData + if err = json.Unmarshal(raw, &data); err != nil { + t.Fatal(err) + } + h := data.Holds[sb.ID] + switch corruption { + case "phase": + h.Sleep.Phase = "unknown" + case "version downgrade": + data.Version = 1 + case "missing VM ID": + h.Sleep.VMID = "" + case "invalid hook": + h.Sleep.Hook.HookArgv = nil + case "sleeping restored process": + h.Sleep.Restored = h.Process + } + data.Holds[sb.ID] = h + raw, err = json.Marshal(data) + if err != nil { + t.Fatal(err) + } + if err = os.WriteFile(path, raw, 0o600); err != nil { + t.Fatal(err) + } + r, err := openVolumeRegistry(cfg.ManagedVolumeDir, "test-node") + if err != nil { + t.Fatal(err) + } + defer r.close() + if r.fault == nil { + t.Fatal("corrupt sleep journal was accepted") + } + }) + } +} + +func TestManagedSleepMissingClaimRetainsOwnership(t *testing.T) { + m, e, sb, cfg := newSleepTest(t) + requireSleep(t, m, sb) + h, _ := m.managedHold(sb.ID) + if err := m.store.commitDurable(m.store.del(sb.ID)); err != nil { + t.Fatal(err) + } + if err := m.Close(); err != nil { + t.Fatal(err) + } + recovered, err := NewManager(t.Context(), cfg, e, testSecrets(t), testBackends(t, cfg)) + if err != nil { + t.Fatal(err) + } + defer recovered.Close() + recovered.managed.process = e.identity + if err = recovered.Reconcile(t.Context()); err == nil { + t.Fatal("missing credentials were reconstructed") + } + if _, held := recovered.managedHold(sb.ID); !held { + t.Fatal("missing claim freed sleep owner") + } + if !slices.Contains(e.snapshots, h.Sleep.Snapshot) { + t.Fatal("missing claim lost recovery source") + } +} + +func TestManagedSleepExpiryIgnoresArchivePool(t *testing.T) { + m, e, sb, _ := newSleepTest(t) + requireSleep(t, m, sb) + m.mu.Lock() + sb.Deadline = time.Now().Add(-time.Minute) + m.pools[sb.Key].archiveAfter = time.Second + m.mu.Unlock() + m.reapOnce(t.Context()) + deadline := time.Now().Add(time.Second) + for { + if _, held := m.managedHold(sb.ID); !held { + break + } + if time.Now().After(deadline) { + t.Fatal("expired managed sleep stayed held") + } + time.Sleep(time.Millisecond) + } + e.mu.Lock() + defer e.mu.Unlock() + if len(e.snapshots) != 0 { + t.Fatal("expiry retained sleep source") + } +} + +func TestManagedSleepUnsupportedPreflightHasNoEffects(t *testing.T) { + for _, bad := range []string{"disabled", "desired mismatch", "extra disk", "missing mount", "egress"} { + t.Run(bad, func(t *testing.T) { + m, e, sb, _ := newSleepTest(t) + switch bad { + case "disabled": + m.managedSleepHook = nil + case "desired mismatch": + sb.VolumesWant = nil + case "extra disk": + e.disks[sb.VMName] = append(e.disks[sb.VMName], types.VMDisk{Name: "extra", Path: "/extra"}) + case "missing mount": + e.mounted = false + case "egress": + sb.Key.Net = types.NetEgress + } + if err := m.Hibernate(t.Context(), sb.ID, Cred{Token: sb.Token}); err == nil { + t.Fatal("unsupported sleep accepted") + } + if len(e.hooks) != 0 || len(e.hibernates) != 0 || slices.Contains(e.volumeOps, "detach:data") { + t.Fatal("unsupported preflight changed guest") + } + }) + } +} + +func TestManagedSleepReadyJournalFailureCannotBypassGate(t *testing.T) { + m, e, sb, _ := newSleepTest(t) + requireSleep(t, m, sb) + commit := m.managed.commitFile + m.managed.commitFile = func(path string, raw []byte) error { + if !strings.Contains(string(raw), `"sleep":`) { + return errors.New("ready journal failed") + } + return commit(path, raw) + } + m.managedSleepHook = nil + if err := m.Wake(t.Context(), sb.ID, Cred{Token: sb.Token}); err == nil { + t.Fatal("ready journal failure hidden") + } + if _, err := m.AgentSocket(sb.ID, sb.Token); !errors.Is(err, ErrPaused) { + t.Fatalf("passive gate bypassed: %v", err) + } + hooks, restores := len(e.hooks), len(e.restores) + if err := m.Wake(t.Context(), sb.ID, Cred{Token: sb.Token}); err == nil { + t.Fatal("poisoned completed journal served guest") + } + if len(e.hooks) != hooks || len(e.restores) != restores { + t.Fatal("effect after ready commit failure") + } +} + +func TestManagedSleepRestoreBeforeEffectRetriesSameSource(t *testing.T) { + m, e, sb, _ := newSleepTest(t) + requireSleep(t, m, sb) + e.restoreErr = errors.New("restore preflight refused") + if err := m.Wake(t.Context(), sb.ID, Cred{Token: sb.Token}); err == nil { + t.Fatal("restore failure hidden") + } + if !e.stopped[sb.VMName] { + t.Fatal("failed restore changed VM") + } + e.restoreErr = nil + if err := m.Wake(t.Context(), sb.ID, Cred{Token: sb.Token}); err != nil { + t.Fatal(err) + } + if len(e.restores) != 2 { + t.Fatal("safe stopped restore was not retried") + } +} + +func TestManagedSleepAdoptsRestoredProcessAcrossCrash(t *testing.T) { + m, e, sb, cfg := newSleepTest(t) + requireSleep(t, m, sb) + h, _ := m.managedHold(sb.ID) + if err := m.setSleepPhase(h, sleepWaking); err != nil { + t.Fatal(err) + } + if _, err := e.Restore(t.Context(), sb.VMName, h.Sleep.Snapshot); err != nil { + t.Fatal(err) + } + e.snapshots = nil + if err := m.Close(); err != nil { + t.Fatal(err) + } + recovered, err := NewManager(t.Context(), cfg, e, testSecrets(t), testBackends(t, cfg)) + if err != nil { + t.Fatal(err) + } + defer recovered.Close() + recovered.managed.process = e.identity + if err = recovered.Reconcile(t.Context()); err != nil { + t.Fatal(err) + } + if err = recovered.Wake(t.Context(), sb.ID, Cred{Token: sb.Token}); err != nil { + t.Fatal(err) + } + if len(e.restores) != 1 { + t.Fatal("crashed wake replayed restore over live process") + } +} + +func TestManagedSleepCaptureCrashAdoptsStoppedSnapshot(t *testing.T) { + m, e, sb, cfg := newSleepTest(t) + commit := m.managed.commitFile + m.managed.commitFile = func(path string, raw []byte) error { + if strings.Contains(string(raw), `"phase":"sleeping"`) { + return errors.New("sleep commit lost") + } + return commit(path, raw) + } + if err := m.Hibernate(t.Context(), sb.ID, Cred{Token: sb.Token}); err == nil { + t.Fatal("lost commit hidden") + } + if !e.stopped[sb.VMName] { + t.Fatal("capture did not take effect") + } + if err := m.Close(); err != nil { + t.Fatal(err) + } + recovered, err := NewManager(t.Context(), cfg, e, testSecrets(t), testBackends(t, cfg)) + if err != nil { + t.Fatal(err) + } + defer recovered.Close() + recovered.managed.process = e.identity + if err = recovered.Reconcile(t.Context()); err != nil { + t.Fatal(err) + } + if err = recovered.Wake(t.Context(), sb.ID, Cred{Token: sb.Token}); err != nil { + t.Fatal(err) + } + if len(e.hibernates) != 1 || len(e.restores) != 1 { + t.Fatal("recovery repeated capture") + } +} + +func TestManagedSleepInitialClaimBarrierPrecedesIntent(t *testing.T) { + m, e, sb, _ := newSleepTest(t) + m.store.path = filepath.Join(t.TempDir(), "missing", "claims.json") + if err := m.Hibernate(t.Context(), sb.ID, Cred{Token: sb.Token}); err == nil { + t.Fatal("claim barrier failed silently") + } + h, _ := m.managedHold(sb.ID) + if h.Sleep != nil || len(e.hooks) != 0 || len(e.hibernates) != 0 { + t.Fatal("effect preceded durable credentials") + } +} diff --git a/sandboxd/pool/managed_sleep_vm.go b/sandboxd/pool/managed_sleep_vm.go new file mode 100644 index 00000000..4a874526 --- /dev/null +++ b/sandboxd/pool/managed_sleep_vm.go @@ -0,0 +1,110 @@ +package pool + +import ( + "context" + "errors" + + "github.com/cocoonstack/sandbox/sandboxd/engine" + "github.com/cocoonstack/sandbox/sandboxd/types" +) + +const vmReasonStopUser = "stop-user" + +func exactSleepDisk(rec types.VMRecord, ref volumeRef) bool { + return len(rec.AttachedDevices.Disks) == 1 && rec.AttachedDevices.Disks[0].Name == ref.Volume.Name && rec.AttachedDevices.Disks[0].Path == ref.Path && !rec.AttachedDevices.Disks[0].ReadOnly +} + +func (m *Manager) sleepVM(ctx context.Context, h volumeHold, restored bool) (types.VMRecord, error) { + rec, found, err := m.eng.Inspect(ctx, h.VMName) + if err != nil { + return rec, err + } + s := h.Sleep + generation := s.Generation + if restored { + generation += 2 + } + if !found || rec.Config.Name != h.VMName || rec.ID != s.VMID || rec.State != vmStateRunning || rec.TransitionGeneration != generation || rec.LastTransitionReason == "" || (restored && rec.LastTransitionReason != "restore") { + return rec, sleepRecoveryError() + } + p, err := m.managed.process(rec.PID) + if err != nil || (s.Restored != (processIdentity{}) && p != s.Restored) || (!restored && p != s.BeforeRestore) { + return rec, sleepRecoveryError() + } + return rec, nil +} + +func (m *Manager) sleepHook(ctx context.Context, h volumeHold, sock, action string) error { + s, ref := h.Sleep, h.Refs[0] + return m.eng.RunVolumeHook(ctx, sock, engine.VolumeHookCall{Argv: s.Hook.HookArgv, Action: action, Operation: s.Operation, VolumeID: string(ref.ID), VolumeName: ref.Volume.Name, Mount: ref.Volume.Mount, Timeout: types.Seconds(s.Hook.HookTimeoutSeconds)}) +} + +func (m *Manager) restoreManagedSleep(ctx context.Context, h volumeHold) (types.VMRecord, error) { + rec, found, err := m.eng.Inspect(ctx, h.VMName) + if err != nil { + return rec, err + } + if !found || rec.ID != h.Sleep.VMID || rec.Config.Name != h.VMName { + return rec, sleepRecoveryError() + } + if rec.State == vmStateRunning { + return m.sleepVM(ctx, h, true) + } + if h.Sleep.Restored != (processIdentity{}) || rec.State != vmStateStopped || rec.TransitionGeneration != h.Sleep.Generation+1 || rec.LastTransitionReason != vmReasonStopUser || !m.managed.processGone(h.Sleep.BeforeRestore) { + return rec, sleepRecoveryError() + } + source, exists, err := m.eng.InspectHibernateSource(ctx, h.Sleep.Snapshot) + if err != nil { + return rec, err + } + if !exists || source.ID != h.Sleep.SourceID { + return rec, sleepRecoveryError() + } + if err = m.diskErr(); err != nil { + return rec, err + } + _, effectErr := m.eng.Restore(ctx, h.VMName, h.Sleep.Snapshot) + rec, err = m.sleepVM(ctx, h, true) + if err != nil { + return rec, errors.Join(effectErr, err) + } + return rec, nil +} + +func (m *Manager) attachManagedSleep(ctx context.Context, h volumeHold, sock string, restored bool) error { + if err := m.verifySleepBacking(h); err != nil { + return err + } + rec, err := m.sleepVM(ctx, h, restored) + if err != nil { + return err + } + ref := h.Refs[0] + if len(rec.AttachedDevices.Disks) == 0 { + effectErr := m.eng.DiskAttach(ctx, h.VMName, engine.VolumeSpec{Name: ref.Volume.Name, Path: ref.Path, RW: true}) + rec, err = m.sleepVM(ctx, h, restored) + if err != nil { + return errors.Join(effectErr, err) + } + if !exactSleepDisk(rec, ref) { + return errors.Join(effectErr, sleepRecoveryError()) + } + } else if !exactSleepDisk(rec, ref) { + return sleepRecoveryError() + } + if err := m.eng.MountLiveVolume(ctx, sock, ref.Volume.Name, ref.Volume.Mount, true); err != nil { + return err + } + return m.eng.VerifyMountedVolume(ctx, sock, ref.Volume.Name, ref.Volume.Mount) +} + +func (m *Manager) verifySleepingVM(ctx context.Context, h volumeHold) error { + rec, found, err := m.eng.Inspect(ctx, h.VMName) + if err != nil { + return err + } + if !found || rec.Config.Name != h.VMName || rec.ID != h.Sleep.VMID || rec.State != vmStateStopped || rec.TransitionGeneration != h.Sleep.Generation+1 || rec.LastTransitionReason != vmReasonStopUser || !m.managed.processGone(h.Sleep.BeforeRestore) { + return sleepRecoveryError() + } + return nil +} diff --git a/sandboxd/pool/managed_volume.go b/sandboxd/pool/managed_volume.go index 01d4fe8f..6ec127d9 100644 --- a/sandboxd/pool/managed_volume.go +++ b/sandboxd/pool/managed_volume.go @@ -136,6 +136,9 @@ func (r *volumeRegistry) completeCreate(v managedVolume) error { } func (r *volumeRegistry) markRecovery(v managedVolume) error { + if r.sleepOwnsVolume(v.ID) { + return managedStorageError() + } v.State = volumeRecovery r.data.Volumes[v.Name] = v if err := r.commit(); err != nil { @@ -162,7 +165,7 @@ func (m *Manager) GetVolume(name, tenant string) (types.ManagedVolumeInfo, error if err := r.markRecovery(v); r.fault != nil { return types.ManagedVolumeInfo{}, err } - v = r.data.Volumes[name] + v.State = volumeRecovery } return r.info(v), nil } diff --git a/sandboxd/pool/operator.go b/sandboxd/pool/operator.go index 45e7d781..a54fb82d 100644 --- a/sandboxd/pool/operator.go +++ b/sandboxd/pool/operator.go @@ -53,7 +53,7 @@ func (m *Manager) SetInstanceMetadata(ctx context.Context, id string, doc []byte return ErrArchived case sb.Failed != "": return ErrFailed - case sb.HibernateSnap != "" || sb.PendingSnap != "": + case m.managedPaused(sb) || sb.HibernateSnap != "" || sb.PendingSnap != "": return ErrPaused } return m.eng.WriteInstanceMetadata(ctx, sb.VsockSocket, doc) diff --git a/sandboxd/pool/pool.go b/sandboxd/pool/pool.go index 3d49c116..2ee5ea7d 100644 --- a/sandboxd/pool/pool.go +++ b/sandboxd/pool/pool.go @@ -72,6 +72,7 @@ const ( hibernatePrefix = "sbx-hib-" forkPrefix = "sbx-fork-" vmStateRunning = "running" + vmStateStopped = "stopped" vmStateCreating = "creating" goldenStampSuffix = ".stamp" @@ -144,6 +145,9 @@ type Engine interface { Warmup(ctx context.Context, vsockSocket string, argv []string) error TrimCow(ctx context.Context, vsockSocket string) error CheckLiveVolumeSupport(ctx context.Context) error + RunVolumeHook(ctx context.Context, socket string, call engine.VolumeHookCall) error + VerifyMountedVolume(ctx context.Context, socket, name, mount string) error + InspectHibernateSource(ctx context.Context, name string) (engine.HibernateSource, bool, error) DiskAttach(ctx context.Context, vmName string, spec engine.VolumeSpec) error DetachVolume(ctx context.Context, vmName, sock, name, mount string) error MountVolume(ctx context.Context, vsockSocket, name, mount string, rw bool) error @@ -328,10 +332,12 @@ type Manager struct { store *claimStore volumes map[string]catalogVolume // volumeAdmission counts the live holders of each volume name; guarded by m.mu. - volumeAdmission map[string]volumeHolders - managed *volumeRegistry - volumeRuns map[*types.Sandbox]*volumeRun - volumeErrors map[*types.Sandbox]map[string]string + volumeAdmission map[string]volumeHolders + managed *volumeRegistry + managedSleepHook *config.ManagedVolumeHibernate + managedStreams map[*types.Sandbox]chan struct{} + volumeRuns map[*types.Sandbox]*volumeRun + volumeErrors map[*types.Sandbox]map[string]string poolStore *poolStore // poolShared is the cell's pool set with a meta store; poolVersion, under m.mu, is its newest version applied. @@ -444,42 +450,44 @@ func NewManager(ctx context.Context, cfg *config.Config, eng Engine, secrets *eg } refill := cmp.Or(cfg.RefillConcurrency, defaultRefill) m := &Manager{ - eng: eng, - dataDir: cfg.DataDir, - egress: cfg.HasEgress(), - releaseDelay: time.Duration(cfg.ReleaseDelaySeconds) * time.Second, - vmmRestart: cfg.VMMRestart, - cocoondSocket: cfg.CocoondSocket, - maxFork: cmp.Or(cfg.MaxForkCount, defaultMaxFork), - store: newClaimStore(cfg.DataDir, cfg.SyncClaims), - volumes: make(map[string]catalogVolume, len(cfg.Volumes)), - volumeAdmission: map[string]volumeHolders{}, - volumeRuns: map[*types.Sandbox]*volumeRun{}, - volumeErrors: map[*types.Sandbox]map[string]string{}, - poolStore: newPoolStore(cfg.DataDir), - poolShared: b.CellPools, - archives: b.Archives, - lease: leaseTiming{ttl: leaseTTL, every: leaseRenew, margin: leaseMargin}, - tenants: b.Tenants, - ckpts: b.Checkpoints, - ckptsShared: b.SharedCheckpoints, - tpls: b.Templates, - pools: make(map[types.PoolKey]*pool, len(cfg.Pools)), - claimed: map[string]*types.Sandbox{}, - pendingRemovals: map[string]pendingRemoval{}, - tenantLive: map[string]int{}, - readers: &readerSet{live: map[string]*reader{}}, - archiving: map[string]struct{}{}, - pendingCks: map[string]struct{}{}, - recLocks: map[string]*sync.RWMutex{}, - recRefs: map[string]int{}, - healPending: map[string]struct{}{}, - healAbort: map[string]struct{}{}, - refillSem: make(chan struct{}, refill), - maintSem: make(chan struct{}, max(1, refill-max(1, refill/4))), - probeSem: make(chan struct{}, refill), - refillKick: make(chan struct{}, 1), - healSem: make(chan struct{}, maxConcurrentHeals), + eng: eng, + dataDir: cfg.DataDir, + egress: cfg.HasEgress(), + releaseDelay: time.Duration(cfg.ReleaseDelaySeconds) * time.Second, + vmmRestart: cfg.VMMRestart, + cocoondSocket: cfg.CocoondSocket, + maxFork: cmp.Or(cfg.MaxForkCount, defaultMaxFork), + store: newClaimStore(cfg.DataDir, cfg.SyncClaims), + volumes: make(map[string]catalogVolume, len(cfg.Volumes)), + volumeAdmission: map[string]volumeHolders{}, + volumeRuns: map[*types.Sandbox]*volumeRun{}, + managedSleepHook: cfg.ManagedVolumeHibernate, + managedStreams: map[*types.Sandbox]chan struct{}{}, + volumeErrors: map[*types.Sandbox]map[string]string{}, + poolStore: newPoolStore(cfg.DataDir), + poolShared: b.CellPools, + archives: b.Archives, + lease: leaseTiming{ttl: leaseTTL, every: leaseRenew, margin: leaseMargin}, + tenants: b.Tenants, + ckpts: b.Checkpoints, + ckptsShared: b.SharedCheckpoints, + tpls: b.Templates, + pools: make(map[types.PoolKey]*pool, len(cfg.Pools)), + claimed: map[string]*types.Sandbox{}, + pendingRemovals: map[string]pendingRemoval{}, + tenantLive: map[string]int{}, + readers: &readerSet{live: map[string]*reader{}}, + archiving: map[string]struct{}{}, + pendingCks: map[string]struct{}{}, + recLocks: map[string]*sync.RWMutex{}, + recRefs: map[string]int{}, + healPending: map[string]struct{}{}, + healAbort: map[string]struct{}{}, + refillSem: make(chan struct{}, refill), + maintSem: make(chan struct{}, max(1, refill-max(1, refill/4))), + probeSem: make(chan struct{}, refill), + refillKick: make(chan struct{}, 1), + healSem: make(chan struct{}, maxConcurrentHeals), } for _, volume := range cfg.Volumes { m.volumes[volume.Name] = catalogVolume{ diff --git a/sandboxd/pool/pool_test.go b/sandboxd/pool/pool_test.go index 136cd118..179dc956 100644 --- a/sandboxd/pool/pool_test.go +++ b/sandboxd/pool/pool_test.go @@ -2009,3 +2009,14 @@ func seedGolden(t *testing.T, m *Manager, imageID string) string { } return g } + +func (f *fakeEngine) RunVolumeHook(_ context.Context, _ string, _ engine.VolumeHookCall) error { + return nil +} + +func (f *fakeEngine) InspectHibernateSource(ctx context.Context, name string) (engine.HibernateSource, bool, error) { + names, err := f.SnapshotList(ctx) + return engine.HibernateSource{ID: name, Name: name}, slices.Contains(names, name), err +} + +func (f *fakeEngine) VerifyMountedVolume(_ context.Context, _, _, _ string) error { return nil } diff --git a/sandboxd/pool/reconcile.go b/sandboxd/pool/reconcile.go index 95564b48..424a99eb 100644 --- a/sandboxd/pool/reconcile.go +++ b/sandboxd/pool/reconcile.go @@ -46,13 +46,16 @@ func (m *Manager) Reconcile(ctx context.Context) error { if err := m.recoverManagedOrphans(ctx, claims, live, owned); err != nil { return err } - referenced := map[string]bool{} + referenced := m.managedSleepSources() var adopted []*types.Sandbox m.mu.Lock() for _, sb := range claims { sb.SetEnv(sb.Env) rec, ok := live[sb.VMName] + h, managed := m.managedHold(sb.ID) switch { + case managed && h.Sleep != nil: + projectSleepState(sb, h.Sleep) case sb.ArchiveCk != "": // archived: no local VM by design, so adopt the stub and let the first exec wake it m.addClaimLocked(sb) @@ -206,7 +209,7 @@ func (m *Manager) resyncEgress(ctx context.Context, live map[string]types.VMReco for _, sb := range m.claimed { sb.TouchAt(now) // a hibernated or archived claim has no guest to serve; its wake arms the door - if sb.HibernateSnap != "" || sb.ArchiveCk != "" || sb.Failed != "" { + if sb.HibernateSnap != "" || sb.PendingSnap != "" || sb.ArchiveCk != "" || sb.Failed != "" { continue } if m.out.LocksNIC(sb.Key) { diff --git a/sandboxd/pool/remove.go b/sandboxd/pool/remove.go index 46ec818e..3951f015 100644 --- a/sandboxd/pool/remove.go +++ b/sandboxd/pool/remove.go @@ -20,10 +20,16 @@ func (m *Manager) removeVM(ctx context.Context, name string) bool { hold, managed := m.managedHoldByVM(name) if managed { if err := m.retireManaged(hold.SandboxID); err != nil { + if hold.Sleep != nil { + return false + } _ = m.eng.Remove(ctx, name) return false } } + if managed && hold.Sleep != nil { + return m.removeManagedSleep(ctx, hold) + } err := m.eng.Remove(ctx, name) if managed { return m.confirmGone(ctx, name) && m.managed.processGone(hold.Process) @@ -102,6 +108,9 @@ func (m *Manager) retryRemovals(ctx context.Context) *sync.WaitGroup { } func (m *Manager) retryRemoval(ctx context.Context, name string, pending pendingRemoval) { + if m.retryManagedSleepRemoval(ctx, name, pending) { + return + } if pending.staleCreate { switch outcome, err := m.eng.ReconcileStaleCreate(ctx, name); { case err != nil: diff --git a/sandboxd/pool/volume_registry.go b/sandboxd/pool/volume_registry.go index 5fc25a32..eebdd017 100644 --- a/sandboxd/pool/volume_registry.go +++ b/sandboxd/pool/volume_registry.go @@ -62,6 +62,7 @@ type processIdentity struct { } type volumeHold struct { + Sleep *managedSleep `json:"sleep,omitempty"` SandboxID string `json:"sandbox_id"` Tenant string `json:"tenant"` VMName string `json:"vm_name,omitempty"` @@ -243,7 +244,7 @@ func (r *volumeRegistry) verify(v managedVolume) error { func (r *volumeRegistry) validate() error { d := r.data - if d.Version != 1 || d.Volumes == nil || d.Holds == nil || d.Deleted == nil { + if (d.Version != 1 && d.Version != 2) || d.Volumes == nil || d.Holds == nil || d.Deleted == nil { return errors.New("invalid volume registry format") } ids := map[volumeID]bool{} @@ -316,6 +317,14 @@ func (r *volumeRegistry) validateHoldRecord(id string, h volumeHold, vms map[str } else if h.State == holdClaimed { return errors.New("claimed volume lacks VM") } + if h.Sleep != nil { + if r.data.Version != 2 { + return errors.New("sleep requires registry version 2") + } + if err := validateManagedSleep(h); err != nil { + return err + } + } names := map[string]bool{} managed := false for _, ref := range h.Refs { diff --git a/sandboxd/types/types.go b/sandboxd/types/types.go index 4cc8d307..8cbe0b78 100644 --- a/sandboxd/types/types.go +++ b/sandboxd/types/types.go @@ -721,6 +721,7 @@ type VMConfig struct { // VMDisk is one hot-attached disk that `cocoon vm inspect` reports. type VMDisk struct { Name string `json:"name"` + Path string `json:"path"` ReadOnly bool `json:"readonly,omitzero"` } @@ -731,12 +732,15 @@ type VMDevices struct { // VMRecord is the subset of cocoon's VM records the control plane reads. type VMRecord struct { - State string `json:"state"` - PID int `json:"pid"` - VsockSocket string `json:"vsock_socket"` - NetworkConfigs []VMNetConfig `json:"network_configs,omitempty"` - Config VMConfig `json:"config"` - AttachedDevices VMDevices `json:"attached_devices,omitzero"` + ID string `json:"id"` + TransitionGeneration uint64 `json:"transition_generation"` + LastTransitionReason string `json:"last_transition_reason"` + State string `json:"state"` + PID int `json:"pid"` + VsockSocket string `json:"vsock_socket"` + NetworkConfigs []VMNetConfig `json:"network_configs,omitempty"` + Config VMConfig `json:"config"` + AttachedDevices VMDevices `json:"attached_devices,omitzero"` } // TapDevice returns the first NIC's host tap; empty when the record carries none. From 6127f6e3c17d5664848b3585c5e83cde8cdf1ff1 Mon Sep 17 00:00:00 2001 From: minatoaquaMK2 Date: Mon, 19 Oct 2026 13:38:47 +0000 Subject: [PATCH 04/75] fix(sandboxd): consume sleep source before disk reattach --- sandboxd/pool/managed_sleep.go | 13 +- sandboxd/pool/managed_sleep_test.go | 277 ++++++++++++++++++++++++++++ sandboxd/pool/managed_sleep_vm.go | 25 +++ 3 files changed, 313 insertions(+), 2 deletions(-) diff --git a/sandboxd/pool/managed_sleep.go b/sandboxd/pool/managed_sleep.go index d7fb1992..3eda2ebc 100644 --- a/sandboxd/pool/managed_sleep.go +++ b/sandboxd/pool/managed_sleep.go @@ -282,6 +282,10 @@ func (m *Manager) wakeManagedSleep(ctx context.Context, sb *types.Sandbox, h vol } h.Sleep = &next } + rec, err = m.consumeManagedSleepSource(ctx, h) + if err != nil { + return err + } sock, err := m.probeReady(ctx, h.VMName, rec.VsockSocket, claimProbeTimeout) if err != nil { return err @@ -296,7 +300,13 @@ func (m *Manager) wakeManagedSleep(ctx context.Context, sb *types.Sandbox, h vol } func (m *Manager) resumeManagedSleep(ctx context.Context, sb *types.Sandbox, h volumeHold, restored bool) error { - rec, err := m.sleepVM(ctx, h, restored) + var rec types.VMRecord + var err error + if restored { + rec, err = m.consumeManagedSleepSource(ctx, h) + } else { + rec, err = m.sleepVM(ctx, h, false) + } if err != nil { return err } @@ -344,7 +354,6 @@ func (m *Manager) resumeManagedSleep(ctx context.Context, sb *types.Sandbox, h v m.counters.wakes.Add(1) m.recordUsage(ctx, usageEvent{Event: "wake", ID: sb.ID, VMName: sb.VMName}) } - m.dropSnap(ctx, h.Sleep.Snapshot) return nil } diff --git a/sandboxd/pool/managed_sleep_test.go b/sandboxd/pool/managed_sleep_test.go index d1bbd3c3..e1f4ce58 100644 --- a/sandboxd/pool/managed_sleep_test.go +++ b/sandboxd/pool/managed_sleep_test.go @@ -24,10 +24,15 @@ type sleepTestEngine struct { generation uint64 reason string vmID string + sourceID string hooks []engine.VolumeHookCall hookErr string hookEntered, hookContinue chan struct{} restoreAfterErr bool + replaceSourceOnRestore bool + snapshotRemoveErr error + snapshotRemoveAfterEffect bool + snapshotRemoveMutation string mounted bool } @@ -146,6 +151,9 @@ func (e *sleepTestEngine) Restore(ctx context.Context, name, snap string) (strin e.sleepMu.Lock() e.generation++ e.reason = "restore" + if e.replaceSourceOnRestore { + e.sourceID = "replacement-source" + } postErr := e.restoreAfterErr e.sleepMu.Unlock() if postErr { @@ -154,6 +162,47 @@ func (e *sleepTestEngine) Restore(ctx context.Context, name, snap string) (strin return sock, nil } +func (e *sleepTestEngine) InspectHibernateSource(ctx context.Context, name string) (engine.HibernateSource, bool, error) { + source, found, err := e.fakeEngine.InspectHibernateSource(ctx, name) + e.sleepMu.Lock() + defer e.sleepMu.Unlock() + if found && e.sourceID != "" { + source.ID = e.sourceID + } + return source, found, err +} + +func (e *sleepTestEngine) SnapshotRemove(ctx context.Context, name string) error { + e.sleepMu.Lock() + err := e.snapshotRemoveErr + afterEffect := e.snapshotRemoveAfterEffect + mutation := e.snapshotRemoveMutation + e.sleepMu.Unlock() + if err != nil && !afterEffect { + return err + } + if removeErr := e.fakeEngine.SnapshotRemove(ctx, name); removeErr != nil { + return removeErr + } + switch mutation { + case "vm id": + e.sleepMu.Lock() + e.vmID = "replacement-vm" + e.sleepMu.Unlock() + case "generation": + e.sleepMu.Lock() + e.generation++ + e.sleepMu.Unlock() + case "process": + e.mu.Lock() + for vmName := range e.pids { + e.pids[vmName]++ + } + e.mu.Unlock() + } + return err +} + func (e *sleepTestEngine) RunVolumeHook(_ context.Context, _ string, call engine.VolumeHookCall) error { e.sleepMu.Lock() e.hooks = append(e.hooks, call) @@ -249,6 +298,9 @@ func TestManagedSleepAbortKeepsOriginalVM(t *testing.T) { if e.hooks[len(e.hooks)-1].Action != "resume" { t.Fatal("writers not resumed") } + if len(e.snapRemoves) != 0 { + t.Fatalf("abort removed nonexistent source: %v", e.snapRemoves) + } }) } } @@ -550,6 +602,38 @@ func TestManagedSleepUnknownSourceAndDeadResumingRefuse(t *testing.T) { } } +func TestManagedSleepDeadBoundVMIsReleaseOnly(t *testing.T) { + m, e, sb, _ := newSleepTest(t) + requireSleep(t, m, sb) + e.hookErr = "resume" + if err := m.Wake(t.Context(), sb.ID, Cred{Token: sb.Token}); err == nil { + t.Fatal("resume failure hidden") + } + h, _ := m.managedHold(sb.ID) + if h.Sleep == nil || h.Sleep.Restored == (processIdentity{}) || len(e.snapshots) != 0 { + t.Fatalf("wake did not bind before consuming source: %+v snapshots=%v", h, e.snapshots) + } + e.mu.Lock() + e.stopped[sb.VMName] = true + e.mu.Unlock() + restores := len(e.restores) + if err := m.Wake(t.Context(), sb.ID, Cred{Token: sb.Token}); err == nil { + t.Fatal("dead bound VM resumed") + } + if len(e.restores) != restores { + t.Fatal("dead bound VM replayed restore") + } + if err := m.Release(t.Context(), sb.ID, Cred{Token: sb.Token}); err != nil { + t.Fatal(err) + } + if _, held := m.managedHold(sb.ID); held { + t.Fatal("release retained dead bound owner") + } + if _, err := os.Stat(h.Refs[0].Path); err != nil { + t.Fatal(err) + } +} + func TestManagedSleepReleaseWaitsForPrepare(t *testing.T) { m, e, sb, _ := newSleepTest(t) _, fenced, release, err := m.WakeAgentSocket(t.Context(), sb.ID, sb.Token) @@ -772,6 +856,199 @@ func TestManagedSleepRestoreBeforeEffectRetriesSameSource(t *testing.T) { } } +func TestManagedSleepWakeWaitsForSourceConsumption(t *testing.T) { + for _, tc := range []struct { + name string + err error + }{ + {"active restore lease", errors.New("snapshot is in use")}, + {"remove failure", errors.New("snapshot delete failed")}, + } { + t.Run(tc.name, func(t *testing.T) { + m, e, sb, _ := newSleepTest(t) + requireSleep(t, m, sb) + e.mu.Lock() + beforeOps := slices.Clone(e.volumeOps) + e.mu.Unlock() + e.sleepMu.Lock() + beforeHooks := len(e.hooks) + e.snapshotRemoveErr = tc.err + e.sleepMu.Unlock() + + if err := m.Wake(t.Context(), sb.ID, Cred{Token: sb.Token}); err == nil { + t.Fatal("wake passed unavailable source-consumption barrier") + } + h, held := m.managedHold(sb.ID) + if !held || h.Sleep == nil || h.Sleep.Phase != sleepWaking || h.Sleep.Restored == (processIdentity{}) { + t.Fatalf("failed barrier lost waking owner: %+v", h) + } + if _, err := m.AgentSocket(sb.ID, sb.Token); !errors.Is(err, ErrPaused) { + t.Fatalf("failed barrier published guest: %v", err) + } + e.mu.Lock() + afterOps := slices.Clone(e.volumeOps) + sourcePresent := slices.Contains(e.snapshots, h.Sleep.Snapshot) + e.mu.Unlock() + e.sleepMu.Lock() + afterHooks := len(e.hooks) + e.snapshotRemoveErr = nil + e.sleepMu.Unlock() + if !slices.Equal(afterOps, beforeOps) || afterHooks != beforeHooks || !sourcePresent { + t.Fatalf("barrier failure reached guest: ops=%v hooks=%d source=%v", afterOps, afterHooks, sourcePresent) + } + generation := e.generation + if err := m.Wake(t.Context(), sb.ID, Cred{Token: sb.Token}); err != nil { + t.Fatal(err) + } + if len(e.restores) != 1 || e.generation != generation { + t.Fatalf("barrier retry replayed restore: restores=%d generation=%d want=%d", len(e.restores), e.generation, generation) + } + }) + } +} + +func TestManagedSleepWakeAcceptsRemoveErrorAfterSourceGone(t *testing.T) { + m, e, sb, _ := newSleepTest(t) + requireSleep(t, m, sb) + e.snapshotRemoveErr = errors.New("snapshot remove reply lost") + e.snapshotRemoveAfterEffect = true + if err := m.Wake(t.Context(), sb.ID, Cred{Token: sb.Token}); err != nil { + t.Fatal(err) + } + if len(e.snapRemoves) != 1 || len(e.snapshots) != 0 { + t.Fatalf("remove after-effect was not accepted: removes=%v snapshots=%v", e.snapRemoves, e.snapshots) + } +} + +func TestManagedSleepWakeRetriesAfterConsumedSource(t *testing.T) { + m, e, sb, cfg := newSleepTest(t) + requireSleep(t, m, sb) + e.diskAttachErr = errors.New("attach unavailable") + if err := m.Wake(t.Context(), sb.ID, Cred{Token: sb.Token}); err == nil { + t.Fatal("attach failure hidden") + } + h, held := m.managedHold(sb.ID) + if !held || h.Sleep == nil || h.Sleep.Restored == (processIdentity{}) { + t.Fatalf("failed attach lost bound waking owner: %+v", h) + } + if len(e.snapshots) != 0 || len(e.snapRemoves) != 1 { + t.Fatalf("source was not consumed before attach: removes=%v snapshots=%v", e.snapRemoves, e.snapshots) + } + generation := e.generation + if err := m.Close(); err != nil { + t.Fatal(err) + } + e.diskAttachErr = nil + recovered, err := NewManager(t.Context(), cfg, e, testSecrets(t), testBackends(t, cfg)) + if err != nil { + t.Fatal(err) + } + defer recovered.Close() + recovered.managed.process = e.identity + if err = recovered.Reconcile(t.Context()); err != nil { + t.Fatal(err) + } + if err = recovered.Wake(t.Context(), sb.ID, Cred{Token: sb.Token}); err != nil { + t.Fatal(err) + } + if len(e.restores) != 1 || len(e.snapRemoves) != 1 || e.generation != generation { + t.Fatalf("consumed-source retry repeated effect: restores=%d removes=%d generation=%d want=%d", len(e.restores), len(e.snapRemoves), e.generation, generation) + } +} + +func TestManagedSleepWakeRefusesReplacementSourceAfterRestore(t *testing.T) { + m, e, sb, _ := newSleepTest(t) + requireSleep(t, m, sb) + e.replaceSourceOnRestore = true + if err := m.Wake(t.Context(), sb.ID, Cred{Token: sb.Token}); err == nil { + t.Fatal("replacement source crossed restore barrier") + } + h, held := m.managedHold(sb.ID) + if !held || h.Sleep == nil || h.Sleep.Restored == (processIdentity{}) || len(e.snapRemoves) != 0 { + t.Fatalf("replacement source changed ownership: %+v removes=%v", h, e.snapRemoves) + } + e.mu.Lock() + defer e.mu.Unlock() + if len(e.disks[sb.VMName]) != 0 { + t.Fatalf("replacement source reached attach: %+v", e.disks[sb.VMName]) + } +} + +func TestManagedSleepWakeRevalidatesAfterSourceConsumption(t *testing.T) { + for _, mutation := range []string{"vm id", "generation", "process"} { + t.Run(mutation, func(t *testing.T) { + m, e, sb, _ := newSleepTest(t) + requireSleep(t, m, sb) + e.mu.Lock() + beforeOps := slices.Clone(e.volumeOps) + e.mu.Unlock() + e.snapshotRemoveMutation = mutation + if err := m.Wake(t.Context(), sb.ID, Cred{Token: sb.Token}); err == nil { + t.Fatal("changed restored VM crossed source barrier") + } + h, held := m.managedHold(sb.ID) + if !held || h.Sleep == nil || h.Sleep.Restored == (processIdentity{}) || len(e.snapshots) != 0 { + t.Fatalf("revalidation failure lost owner: %+v snapshots=%v", h, e.snapshots) + } + e.mu.Lock() + defer e.mu.Unlock() + if !slices.Equal(e.volumeOps, beforeOps) { + t.Fatalf("revalidation failure reached guest: before=%v after=%v", beforeOps, e.volumeOps) + } + }) + } +} + +func TestManagedSleepResumingWaitsForSourceConsumption(t *testing.T) { + m, e, sb, _ := newSleepTest(t) + requireSleep(t, m, sb) + h, _ := m.managedHold(sb.ID) + if _, err := e.Restore(t.Context(), sb.VMName, h.Sleep.Snapshot); err != nil { + t.Fatal(err) + } + rec, _, err := e.Inspect(t.Context(), sb.VMName) + if err != nil { + t.Fatal(err) + } + p, err := e.identity(rec.PID) + if err != nil { + t.Fatal(err) + } + next := *h.Sleep + next.Phase = sleepResuming + next.Restored = p + h.Process = p + if err = m.managed.commitManagedSleep(h, &next); err != nil { + t.Fatal(err) + } + e.mu.Lock() + beforeOps := slices.Clone(e.volumeOps) + e.mu.Unlock() + e.sleepMu.Lock() + beforeHooks := len(e.hooks) + e.snapshotRemoveErr = errors.New("snapshot is in use") + e.sleepMu.Unlock() + if err = m.Wake(t.Context(), sb.ID, Cred{Token: sb.Token}); err == nil { + t.Fatal("resuming guest passed unavailable source-consumption barrier") + } + e.mu.Lock() + afterOps := slices.Clone(e.volumeOps) + e.mu.Unlock() + e.sleepMu.Lock() + afterHooks := len(e.hooks) + e.snapshotRemoveErr = nil + e.sleepMu.Unlock() + if !slices.Equal(afterOps, beforeOps) || afterHooks != beforeHooks { + t.Fatalf("resuming barrier failure reached guest: ops=%v hooks=%d", afterOps, afterHooks) + } + if err = m.Wake(t.Context(), sb.ID, Cred{Token: sb.Token}); err != nil { + t.Fatal(err) + } + if len(e.restores) != 1 { + t.Fatalf("resuming barrier retry replayed restore: %v", e.restores) + } +} + func TestManagedSleepAdoptsRestoredProcessAcrossCrash(t *testing.T) { m, e, sb, cfg := newSleepTest(t) requireSleep(t, m, sb) diff --git a/sandboxd/pool/managed_sleep_vm.go b/sandboxd/pool/managed_sleep_vm.go index 4a874526..95b087eb 100644 --- a/sandboxd/pool/managed_sleep_vm.go +++ b/sandboxd/pool/managed_sleep_vm.go @@ -71,6 +71,31 @@ func (m *Manager) restoreManagedSleep(ctx context.Context, h volumeHold) (types. return rec, nil } +func (m *Manager) consumeManagedSleepSource(ctx context.Context, h volumeHold) (types.VMRecord, error) { + if h.Sleep.Restored == (processIdentity{}) || h.Process != h.Sleep.Restored { + return types.VMRecord{}, sleepRecoveryError() + } + source, exists, err := m.eng.InspectHibernateSource(ctx, h.Sleep.Snapshot) + if err != nil { + return types.VMRecord{}, err + } + if exists && source.ID != h.Sleep.SourceID { + return types.VMRecord{}, sleepRecoveryError() + } + var effectErr error + if exists { + effectErr = m.eng.SnapshotRemove(ctx, h.Sleep.Snapshot) + } + source, exists, err = m.eng.InspectHibernateSource(ctx, h.Sleep.Snapshot) + if err != nil { + return types.VMRecord{}, errors.Join(effectErr, err) + } + if exists { + return types.VMRecord{}, errors.Join(effectErr, sleepRecoveryError()) + } + return m.sleepVM(ctx, h, true) +} + func (m *Manager) attachManagedSleep(ctx context.Context, h volumeHold, sock string, restored bool) error { if err := m.verifySleepBacking(h); err != nil { return err From 0391ec292afce37e972f2c8b0e8ad0827f7f44fe Mon Sep 17 00:00:00 2001 From: minatoaquaMK2 Date: Mon, 19 Oct 2026 14:00:53 +0000 Subject: [PATCH 05/75] test(e2e): implement managed sleep engine methods --- e2e/fakeengine_test.go | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/e2e/fakeengine_test.go b/e2e/fakeengine_test.go index 9bbc9113..74c767d7 100644 --- a/e2e/fakeengine_test.go +++ b/e2e/fakeengine_test.go @@ -47,6 +47,18 @@ func newFakeEngine(dir string) *fakeEngine { func (f *fakeEngine) CheckLiveVolumeSupport(context.Context) error { return nil } +func (f *fakeEngine) RunVolumeHook(context.Context, string, engine.VolumeHookCall) error { + return errors.New("fakeEngine has no volume hook") +} + +func (f *fakeEngine) VerifyMountedVolume(context.Context, string, string, string) error { + return errors.New("fakeEngine cannot verify mounted volumes") +} + +func (f *fakeEngine) InspectHibernateSource(context.Context, string) (engine.HibernateSource, bool, error) { + return engine.HibernateSource{}, false, errors.New("fakeEngine cannot inspect hibernate sources") +} + func (f *fakeEngine) Clone(_ context.Context, _, name string, _ types.PoolKey) (types.VMRecord, error) { return f.createRecord(name) } From 68e78befc782ce588e390800aa403a57dbca8959 Mon Sep 17 00:00:00 2001 From: minatoaquaMK2 Date: Mon, 19 Oct 2026 14:55:25 +0000 Subject: [PATCH 06/75] fix(sandboxd): bound managed sleep snapshot names --- sandboxd/pool/managed_sleep.go | 2 +- sandboxd/pool/managed_sleep_test.go | 21 +++++++++++++++++++++ 2 files changed, 22 insertions(+), 1 deletion(-) diff --git a/sandboxd/pool/managed_sleep.go b/sandboxd/pool/managed_sleep.go index 3eda2ebc..7d9c03de 100644 --- a/sandboxd/pool/managed_sleep.go +++ b/sandboxd/pool/managed_sleep.go @@ -153,7 +153,7 @@ func (m *Manager) beginManagedSleep(ctx context.Context, sb *types.Sandbox) erro } hook := *m.managedSleepHook hook.HookArgv = slices.Clone(hook.HookArgv) - next := &managedSleep{Operation: randHex(16), Phase: sleepPreparing, Snapshot: hibernatePrefix + strings.TrimPrefix(sb.VMName, vmPrefix) + "-" + randHex(8), Hook: hook, VMID: rec.ID, Generation: rec.TransitionGeneration, BeforeRestore: h.Process} + next := &managedSleep{Operation: randHex(16), Phase: sleepPreparing, Snapshot: hibernatePrefix + strings.TrimPrefix(sb.VMName, vmPrefix) + "-" + randHex(3), Hook: hook, VMID: rec.ID, Generation: rec.TransitionGeneration, BeforeRestore: h.Process} if err = m.managed.commitManagedSleep(h, next); err != nil { return err } diff --git a/sandboxd/pool/managed_sleep_test.go b/sandboxd/pool/managed_sleep_test.go index e1f4ce58..62a13f36 100644 --- a/sandboxd/pool/managed_sleep_test.go +++ b/sandboxd/pool/managed_sleep_test.go @@ -6,6 +6,7 @@ import ( "errors" "os" "path/filepath" + "regexp" "slices" "strings" "sync" @@ -271,6 +272,26 @@ func TestManagedSleepRoundTripAndExclusiveHold(t *testing.T) { } } +func TestManagedSleepSnapshotNameFitsEngineContract(t *testing.T) { + m, e, sb, _ := newSleepTest(t) + if err := m.Hibernate(t.Context(), sb.ID, Cred{Token: sb.Token}); err != nil { + t.Fatal(err) + } + if len(e.hibernates) != 1 { + t.Fatalf("hibernate calls = %v, want one", e.hibernates) + } + snapshot := e.hibernates[0] + if len(snapshot) > 63 { + t.Fatalf("snapshot name %q has length %d, exceeds Cocoon's 63-character limit", snapshot, len(snapshot)) + } + if !regexp.MustCompile(`^[a-zA-Z0-9][a-zA-Z0-9._:/-]{0,62}$`).MatchString(snapshot) { + t.Fatalf("snapshot name %q violates Cocoon's name syntax", snapshot) + } + if err := m.Wake(t.Context(), sb.ID, Cred{Token: sb.Token}); err != nil { + t.Fatal(err) + } +} + func TestManagedSleepAbortKeepsOriginalVM(t *testing.T) { for _, failure := range []string{"prepare", "unmount", "detach", "capture"} { t.Run(failure, func(t *testing.T) { From 97cab4bd2ce685aafc6a821b2260ef9a9ee17c80 Mon Sep 17 00:00:00 2001 From: minatoaquaMK2 Date: Tue, 20 Oct 2026 06:33:14 +0000 Subject: [PATCH 07/75] Add durable online growth for mounted managed volumes --- sandboxd/engine/cocoond_test.go | 2 +- sandboxd/engine/testdata/disk-grow.json | 76 +++ sandboxd/engine/testdata/disk-inspect.json | 41 ++ sandboxd/engine/volume_grow.go | 240 +++++++++ sandboxd/engine/volume_grow_test.go | 113 +++++ sandboxd/engine/volume_live.go | 3 +- sandboxd/pool/hibernate.go | 3 + sandboxd/pool/liveness.go | 6 + sandboxd/pool/managed_attachment.go | 13 +- sandboxd/pool/managed_grow.go | 437 ++++++++++++++++ sandboxd/pool/managed_grow_test.go | 556 +++++++++++++++++++++ sandboxd/pool/managed_recovery.go | 5 +- sandboxd/pool/managed_sleep.go | 6 + sandboxd/pool/managed_sleep_registry.go | 2 +- sandboxd/pool/managed_volume.go | 32 +- sandboxd/pool/reconcile.go | 6 + sandboxd/pool/remove.go | 9 +- sandboxd/pool/volume_mutation.go | 4 +- sandboxd/pool/volume_registry.go | 33 +- sandboxd/types/api.go | 40 +- sandboxd/types/types.go | 1 + 21 files changed, 1592 insertions(+), 36 deletions(-) create mode 100644 sandboxd/engine/testdata/disk-grow.json create mode 100644 sandboxd/engine/testdata/disk-inspect.json create mode 100644 sandboxd/engine/volume_grow.go create mode 100644 sandboxd/engine/volume_grow_test.go create mode 100644 sandboxd/pool/managed_grow.go create mode 100644 sandboxd/pool/managed_grow_test.go diff --git a/sandboxd/engine/cocoond_test.go b/sandboxd/engine/cocoond_test.go index 77044695..86843c64 100644 --- a/sandboxd/engine/cocoond_test.go +++ b/sandboxd/engine/cocoond_test.go @@ -48,7 +48,7 @@ func TestInspectReadsOneVMAndReportsAMissingOne(t *testing.T) { "echo \"Error: inspect: vm $3: vm not found\" >&2; exit 1\n") e := New("cocoon", nil, nil, false, false, "") rec, ok, err := e.Inspect(t.Context(), "sbx-1") - wantDisks := []types.VMDisk{{Name: "imagenet", Path: "/v/a.img", ReadOnly: true}, {Name: "scratch", Path: "/v/b.img"}} + wantDisks := []types.VMDisk{{ID: "disk-a", Name: "imagenet", Path: "/v/a.img", ReadOnly: true}, {ID: "disk-b", Name: "scratch", Path: "/v/b.img"}} if err != nil || !ok || rec.State != "running" || rec.Config.Name != "sbx-1" || !slices.Equal(rec.AttachedDevices.Disks, wantDisks) { t.Errorf("Inspect sbx-1 = %+v, %v, %v", rec, ok, err) } diff --git a/sandboxd/engine/testdata/disk-grow.json b/sandboxd/engine/testdata/disk-grow.json new file mode 100644 index 00000000..26011abf --- /dev/null +++ b/sandboxd/engine/testdata/disk-grow.json @@ -0,0 +1,76 @@ +{ + "guest_block_status": "unobserved", + "live": { + "binding": { + "backing": { + "device": 64512, + "fds": [ + { + "access": "read_write", + "number": 250 + }, + { + "access": "read_write", + "number": 261 + } + ], + "inode": 8520047 + }, + "format": "raw", + "mode": "read_write", + "observed_bytes": "42949672960", + "path": "/root/cg-owned-9578808062cc309c3b577001/managed/disks/vol_df3c2dd415fb32004a5f54a57aa2552f.raw", + "ref": { + "disk_id": "cocoon-disk-g09578808062cc", + "vm_id": "J426Z5BTLBPWO4WND6FCTMBUVD" + }, + "revision": 1, + "serial": "g09578808062cc", + "vmm": { + "boot_id": "31594e64-377c-4047-bd49-f9ee9c611e92", + "pid": 1665882, + "start": 35979927 + } + }, + "state": "observed" + }, + "receipt": { + "outcome": "acknowledged", + "request": { + "expected": { + "backing": { + "device": 64512, + "fds": [ + { + "access": "read_write", + "number": 250 + }, + { + "access": "read_write", + "number": 261 + } + ], + "inode": 8520047 + }, + "format": "raw", + "mode": "read_write", + "observed_bytes": "21474836480", + "path": "/root/cg-owned-9578808062cc309c3b577001/managed/disks/vol_df3c2dd415fb32004a5f54a57aa2552f.raw", + "ref": { + "disk_id": "cocoon-disk-g09578808062cc", + "vm_id": "J426Z5BTLBPWO4WND6FCTMBUVD" + }, + "revision": 0, + "serial": "g09578808062cc", + "vmm": { + "boot_id": "31594e64-377c-4047-bd49-f9ee9c611e92", + "pid": 1665882, + "start": 35979927 + } + }, + "id": "b51b3d2b-4ccc-41d5-b430-3046340c30b0", + "target_bytes": "42949672960" + }, + "revision": 1 + } +} diff --git a/sandboxd/engine/testdata/disk-inspect.json b/sandboxd/engine/testdata/disk-inspect.json new file mode 100644 index 00000000..26a301d3 --- /dev/null +++ b/sandboxd/engine/testdata/disk-inspect.json @@ -0,0 +1,41 @@ +{ + "guest_block_status": "unobserved", + "live": { + "binding": { + "backing": { + "device": 64512, + "fds": [ + { + "access": "read_write", + "number": 250 + }, + { + "access": "read_write", + "number": 261 + } + ], + "inode": 8520047 + }, + "format": "raw", + "mode": "read_write", + "observed_bytes": "21474836480", + "path": "/root/cg-owned-9578808062cc309c3b577001/managed/disks/vol_df3c2dd415fb32004a5f54a57aa2552f.raw", + "ref": { + "disk_id": "cocoon-disk-g09578808062cc", + "vm_id": "J426Z5BTLBPWO4WND6FCTMBUVD" + }, + "revision": 0, + "serial": "g09578808062cc", + "vmm": { + "boot_id": "31594e64-377c-4047-bd49-f9ee9c611e92", + "pid": 1665882, + "start": 35979927 + } + }, + "state": "observed" + }, + "requested": { + "disk_id": "cocoon-disk-g09578808062cc", + "vm_id": "J426Z5BTLBPWO4WND6FCTMBUVD" + } +} diff --git a/sandboxd/engine/volume_grow.go b/sandboxd/engine/volume_grow.go new file mode 100644 index 00000000..2797efb6 --- /dev/null +++ b/sandboxd/engine/volume_grow.go @@ -0,0 +1,240 @@ +package engine + +import ( + "bytes" + "context" + "encoding/json/v2" + "errors" + "fmt" + "os" + "os/exec" + "path/filepath" + "regexp" + "slices" + "strconv" + "strings" +) + +type DiskRef struct { + VMID string `json:"vm_id"` + DiskID string `json:"disk_id"` +} +type DiskProcess struct { + BootID string `json:"boot_id"` + PID int `json:"pid"` + Start uint64 `json:"start"` +} +type DiskFD struct { + Number int `json:"number"` + Access string `json:"access"` +} +type DiskBacking struct { + Device uint64 `json:"device"` + Inode uint64 `json:"inode"` + FDs []DiskFD `json:"fds"` +} +type DiskBinding struct { + Ref DiskRef `json:"ref"` + VMM DiskProcess `json:"vmm"` + Serial string `json:"serial"` + Path string `json:"path"` + Mode string `json:"mode"` + Format string `json:"format"` + Backing DiskBacking `json:"backing"` + ObservedBytes int64 `json:"observed_bytes,string"` + Revision uint64 `json:"revision"` +} +type DiskGrowRequest struct { + Expected DiskBinding `json:"expected"` + ID string `json:"id"` + TargetBytes int64 `json:"target_bytes,string"` +} +type DiskReceipt struct { + Request DiskGrowRequest `json:"request"` + Revision uint64 `json:"revision"` + Outcome string `json:"outcome"` +} +type DiskObservation struct { + State string `json:"state"` + Binding *DiskBinding `json:"binding,omitempty"` + Reason string `json:"reason,omitempty"` +} +type DiskError struct { + Code string `json:"code"` + Message string `json:"message"` +} +type DiskInspection struct { + Requested DiskRef `json:"requested"` + Live DiskObservation `json:"live"` + LatestReceipt *DiskReceipt `json:"latest_receipt,omitempty"` + GuestBlockStatus string `json:"guest_block_status"` + Error *DiskError `json:"error,omitempty"` +} +type DiskGrowResult struct { + Receipt *DiskReceipt `json:"receipt,omitempty"` + Live DiskObservation `json:"live"` + GuestBlockStatus string `json:"guest_block_status"` + Error *DiskError `json:"error,omitempty"` +} + +var growUUID = regexp.MustCompile(`^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$`) + +func ValidGrowID(id string) bool { + return growUUID.MatchString(id) && id != "00000000-0000-0000-0000-000000000000" +} + +func (b DiskBinding) Validate() error { + if b.Ref.VMID == "" || b.Ref.DiskID == "" || b.VMM.BootID == "" || b.VMM.PID <= 0 || b.VMM.Start == 0 || b.Serial == "" || !filepath.IsAbs(b.Path) || filepath.Clean(b.Path) != b.Path || b.Mode != "read_write" || b.Format != "raw" || b.Backing.Inode == 0 || len(b.Backing.FDs) == 0 || b.ObservedBytes <= 0 { + return errors.New("invalid disk binding") + } + for i, fd := range b.Backing.FDs { + if fd.Number < 0 || fd.Access != "read_write" || i > 0 && b.Backing.FDs[i-1].Number >= fd.Number { + return errors.New("invalid disk FD witnesses") + } + } + return nil +} +func (b DiskBinding) SameIdentity(other DiskBinding) bool { + return b.Ref == other.Ref && b.VMM == other.VMM && b.Serial == other.Serial && b.Path == other.Path && b.Mode == other.Mode && b.Format == other.Format && b.Backing.Device == other.Backing.Device && b.Backing.Inode == other.Backing.Inode && slices.Equal(b.Backing.FDs, other.Backing.FDs) +} +func (b DiskBinding) Equal(other DiskBinding) bool { + return b.SameIdentity(other) && b.ObservedBytes == other.ObservedBytes && b.Revision == other.Revision +} +func (r DiskGrowRequest) Equal(other DiskGrowRequest) bool { + return r.ID == other.ID && r.TargetBytes == other.TargetBytes && r.Expected.Equal(other.Expected) +} + +func (e *Engine) diskCommand(ctx context.Context, args ...string) ([]byte, error) { + ctx, cancel := context.WithTimeout(ctx, cmdTimeout) + defer cancel() + cmd := exec.CommandContext(ctx, e.bin, args...) + var stdout, stderr bytes.Buffer + cmd.Stdout, cmd.Stderr = &stdout, &stderr + runErr := cmd.Run() + return stdout.Bytes(), runErr +} + +func (e *Engine) InspectDisk(ctx context.Context, ref DiskRef) (DiskInspection, error) { + var result DiskInspection + raw, err := e.diskCommand(ctx, "vm", "disk", "inspect", ref.VMID, "--disk-id", ref.DiskID, "--output", "json") + if parseErr := json.Unmarshal(raw, &result, json.RejectUnknownMembers(true)); parseErr != nil { + return DiskInspection{}, errors.Join(err, parseErr) + } + if result.Requested != ref || result.GuestBlockStatus != "unobserved" { + return DiskInspection{}, errors.New("unsupported disk inspection") + } + if result.Live.State != "observed" && result.Live.State != "unavailable" { + return DiskInspection{}, errors.New("invalid disk observation") + } + if result.Live.State == "observed" && (result.Live.Binding == nil || result.Live.Binding.Validate() != nil || result.Live.Binding.Ref != ref) { + return DiskInspection{}, errors.New("invalid observed binding") + } + return result, err +} + +func (e *Engine) GrowDisk(ctx context.Context, request DiskGrowRequest) (DiskGrowResult, error) { + if request.Expected.Validate() != nil || !ValidGrowID(request.ID) || request.TargetBytes <= request.Expected.ObservedBytes || request.TargetBytes%4096 != 0 { + return DiskGrowResult{}, errors.New("invalid disk grow request") + } + expected := DiskInspection{Requested: request.Expected.Ref, Live: DiskObservation{State: "observed", Binding: &request.Expected}, GuestBlockStatus: "unobserved"} + raw, err := json.Marshal(expected) + if err != nil { + return DiskGrowResult{}, err + } + f, err := os.CreateTemp("", "sandbox-grow-") + if err != nil { + return DiskGrowResult{}, err + } + defer func() { _ = os.Remove(f.Name()) }() + _, writeErr := f.Write(raw) + if err = errors.Join(writeErr, f.Close()); err != nil { + return DiskGrowResult{}, err + } + var result DiskGrowResult + raw, err = e.diskCommand(ctx, "vm", "disk", "grow", "--expect", f.Name(), "--request-id", request.ID, "--target-bytes", strconv.FormatInt(request.TargetBytes, 10), "--output", "json") + if parseErr := json.Unmarshal(raw, &result, json.RejectUnknownMembers(true)); parseErr != nil { + return DiskGrowResult{}, errors.Join(err, parseErr) + } + if result.GuestBlockStatus != "unobserved" { + return DiskGrowResult{}, errors.New("unsupported disk grow response") + } + return result, err +} + +type GuestVolumeObservation struct { + BootID string `json:"boot_id"` + Serial string `json:"serial"` + Device string `json:"device"` + DevNum string `json:"dev_num"` + MountID string `json:"mount_id"` + Mount string `json:"mount"` + Root string `json:"root"` + Mode string `json:"mode"` + SizeBytes int64 `json:"size_bytes"` +} + +func (g GuestVolumeObservation) SameIdentity(other GuestVolumeObservation) bool { + g.SizeBytes = other.SizeBytes + return g == other +} +func (g GuestVolumeObservation) Valid() bool { + return g.BootID != "" && g.Serial != "" && strings.HasPrefix(g.Device, "/dev/") && g.DevNum != "" && g.MountID != "" && filepath.IsAbs(g.Mount) && g.Root == "/" && g.Mode == "rw" && g.SizeBytes > 0 +} + +// ObserveVolume reads block and mount identity without changing the filesystem. +func (e *Engine) ObserveVolume(ctx context.Context, socket, serial, mount string) (GuestVolumeObservation, error) { + first, err := e.observeVolume(ctx, socket, serial, mount) + if err != nil { + return GuestVolumeObservation{}, err + } + second, err := e.observeVolume(ctx, socket, serial, mount) + if err != nil { + return GuestVolumeObservation{}, err + } + if !first.SameIdentity(second) { + return GuestVolumeObservation{}, errors.New("guest identity changed during block observation") + } + return second, nil +} + +func (e *Engine) observeVolume(ctx context.Context, socket, serial, mount string) (GuestVolumeObservation, error) { + ctx, cancel := context.WithTimeout(ctx, volumeSetupTimeout) + defer cancel() + device, found, err := e.findVolumeDevice(ctx, socket, serial) + if err != nil || !found { + return GuestVolumeObservation{}, errors.New("guest disk unavailable") + } + boot, err := e.silkdReadFile(ctx, socket, "/proc/sys/kernel/random/boot_id", silkdReadMax) + if err != nil { + return GuestVolumeObservation{}, err + } + devNum, mounts, err := e.volumeMounts(ctx, socket, device) + if err != nil { + return GuestVolumeObservation{}, err + } + target := e.resolveMount(ctx, socket, mount, mounts) + g := GuestVolumeObservation{BootID: strings.TrimSpace(string(boot)), Serial: serial, Device: device, DevNum: devNum} + for _, m := range mounts { + if m.devNum != devNum && m.path != target { + continue + } + mode, _, _ := strings.Cut(m.options, ",") + if g.MountID != "" || m.devNum != devNum || m.path != target || m.root != "/" || mode != "rw" { + return GuestVolumeObservation{}, errors.New("guest mount identity mismatch") + } + g.MountID, g.Mount, g.Root, g.Mode = m.id, m.path, m.root, mode + } + size, err := e.silkdReadFile(ctx, socket, "/sys/block/"+filepath.Base(device)+"/size", silkdReadMax) + if err != nil { + return GuestVolumeObservation{}, err + } + sectors, err := strconv.ParseInt(strings.TrimSpace(string(size)), 10, 64) + if err != nil || sectors <= 0 || sectors > (1<<63-1)/512 { + return GuestVolumeObservation{}, fmt.Errorf("invalid guest block size") + } + g.SizeBytes = sectors * 512 + if !g.Valid() { + return GuestVolumeObservation{}, errors.New("incomplete guest volume observation") + } + return g, nil +} diff --git a/sandboxd/engine/volume_grow_test.go b/sandboxd/engine/volume_grow_test.go new file mode 100644 index 00000000..094efc17 --- /dev/null +++ b/sandboxd/engine/volume_grow_test.go @@ -0,0 +1,113 @@ +package engine + +import ( + "encoding/json/v2" + "os" + "path/filepath" + "strconv" + "strings" + "testing" +) + +func TestGrowthQualifiedWireAndPrivateExpectation(t *testing.T) { + // Captured from qualified Cocoon efe8897 v5 runtime evidence, without normalization. + inspect, err := os.ReadFile("testdata/disk-inspect.json") + if err != nil { + t.Fatal(err) + } + grow, err := os.ReadFile("testdata/disk-grow.json") + if err != nil { + t.Fatal(err) + } + var expected DiskInspection + var result DiskGrowResult + if err := json.Unmarshal(inspect, &expected, json.RejectUnknownMembers(true)); err != nil { + t.Fatal(err) + } + if err := json.Unmarshal(grow, &result, json.RejectUnknownMembers(true)); err != nil { + t.Fatal(err) + } + if expected.Live.Binding.ObservedBytes != 20<<30 || result.Receipt.Request.TargetBytes != 40<<30 || !result.Receipt.Request.Expected.Equal(*expected.Live.Binding) || len(expected.Live.Binding.Backing.FDs) != 2 { + t.Fatal("qualified schema changed") + } + dir := t.TempDir() + t.Setenv("GROW_CAPTURE", dir) + t.Setenv("GROW_INSPECT", string(inspect)) + t.Setenv("GROW_RESULT", string(grow)) + fakeCocoon(t, `#!/bin/sh +if [ "$1 $2 $3" = "vm disk inspect" ]; then printf '%s' "$GROW_INSPECT"; exit 0; fi +[ "$1 $2 $3 $4" = "vm disk grow --expect" ] || exit 72 +cp "$5" "$GROW_CAPTURE/expected" +stat -c '%a' "$5" > "$GROW_CAPTURE/mode" +printf '%s' "$5" > "$GROW_CAPTURE/temp-path" +printf '%s\n' "$@" > "$GROW_CAPTURE/args" +printf '%s' "$GROW_RESULT" +exit 1 +`) + e := New("cocoon", nil, nil, false, false, "") + got, err := e.InspectDisk(t.Context(), expected.Requested) + if err != nil || !got.Live.Binding.Equal(*expected.Live.Binding) { + t.Fatalf("inspection %+v %v", got, err) + } + receipt, err := e.GrowDisk(t.Context(), result.Receipt.Request) + if err == nil || receipt.Receipt == nil || receipt.Receipt.Outcome != "acknowledged" { + t.Fatalf("nonzero stdout lost: %+v %v", receipt, err) + } + raw, _ := os.ReadFile(filepath.Join(dir, "expected")) + var transported DiskInspection + if err := json.Unmarshal(raw, &transported, json.RejectUnknownMembers(true)); err != nil || !transported.Live.Binding.Equal(*expected.Live.Binding) || transported.LatestReceipt != nil || transported.Error != nil { + t.Fatalf("expected envelope: %s %v", raw, err) + } + mode, _ := os.ReadFile(filepath.Join(dir, "mode")) + if strings.TrimSpace(string(mode)) != "600" { + t.Fatalf("temp mode: %s", mode) + } + path, _ := os.ReadFile(filepath.Join(dir, "temp-path")) + if _, err := os.Stat(string(path)); !os.IsNotExist(err) { + t.Fatal("expect tempfile leaked") + } + args, _ := os.ReadFile(filepath.Join(dir, "args")) + if !strings.Contains(string(args), "--target-bytes\n42949672960\n--output\njson\n") || strings.Contains(string(args), expected.Requested.VMID) { + t.Fatalf("wrong grow args: %s", args) + } +} + +func TestGrowthGuestBlockWitnessDoesNotResizeFilesystem(t *testing.T) { + sock := sockPath(t) + fake := serveFakeSilkd(t, sock) + configureVolumeDevices(fake) + fake.mu.Lock() + fake.serial["/sys/block/vdc/dev"] = []byte("8:32\n") + fake.serial["/sys/block/vdc/size"] = []byte(strconv.FormatInt(40<<30/512, 10) + "\n") + fake.serial["/proc/sys/kernel/random/boot_id"] = []byte("guest-boot\n") + fake.serial["/proc/self/mountinfo"] = []byte(rootMountInfo + "27 1 8:32 / /data rw - ext4 /dev/vdc rw\n") + fake.mu.Unlock() + e := New("cocoon", nil, nil, false, false, "") + got, err := e.ObserveVolume(t.Context(), sock, "imagenet", "/data") + if err != nil || got.MountID != "27" || got.SizeBytes != 40<<30 || got.BootID != "guest-boot" || got.Device != "/dev/vdc" { + t.Fatalf("guest witness %+v: %v", got, err) + } + fake.mu.Lock() + defer fake.mu.Unlock() + if len(fake.execCalls) != 0 { + t.Fatalf("observation executed guest tools: %v", fake.execCalls) + } +} + +func TestGrowthRejectsMalformedWireEvenWithAValidReceiptPrefix(t *testing.T) { + raw, err := os.ReadFile("testdata/disk-grow.json") + if err != nil { + t.Fatal(err) + } + var qualified DiskGrowResult + if err := json.Unmarshal(raw, &qualified); err != nil { + t.Fatal(err) + } + t.Setenv("GROW_BAD_JSON", strings.TrimSpace(string(raw))+" invalid trailing bytes") + fakeCocoon(t, "#!/bin/sh\nprintf '%s' \"$GROW_BAD_JSON\"\nexit 1\n") + e := New("cocoon", nil, nil, false, false, "") + got, err := e.GrowDisk(t.Context(), qualified.Receipt.Request) + if err == nil || got.Receipt != nil { + t.Fatalf("partial malformed stdout accepted: %+v %v", got, err) + } +} diff --git a/sandboxd/engine/volume_live.go b/sandboxd/engine/volume_live.go index e1033a84..3fc7b822 100644 --- a/sandboxd/engine/volume_live.go +++ b/sandboxd/engine/volume_live.go @@ -32,6 +32,7 @@ func (e *LiveVolumeUnsupported) Error() string { type volumeMount struct { devNum, root, path, options string + id string } // CheckLiveVolumeSupport rejects runtimes without idempotent live disk attach and detach. @@ -112,7 +113,7 @@ func (e *Engine) volumeMounts(ctx context.Context, vsockSocket, device string) ( if len(fields) < 10 || !slices.Contains(fields[6:], "-") { return "", nil, errors.New("invalid guest mountinfo") } - mounts = append(mounts, volumeMount{fields[2], mountUnescaper.Replace(fields[3]), mountUnescaper.Replace(fields[4]), fields[5]}) + mounts = append(mounts, volumeMount{fields[2], mountUnescaper.Replace(fields[3]), mountUnescaper.Replace(fields[4]), fields[5], fields[0]}) } return strings.TrimSpace(string(devNum)), mounts, nil } diff --git a/sandboxd/pool/hibernate.go b/sandboxd/pool/hibernate.go index 6b26e76b..9d37a040 100644 --- a/sandboxd/pool/hibernate.go +++ b/sandboxd/pool/hibernate.go @@ -247,6 +247,9 @@ func (m *Manager) wakeOrReadLocked(ctx context.Context, sb *types.Sandbox, reade } func (m *Manager) stopVM(ctx context.Context, name string) { + if m.growthOwnsVM(name) { + return + } if err := m.eng.Stop(ctx, name); err != nil { log.WithFunc("pool.stopVM").Warnf(ctx, "stop vm %s: %v", name, err) } diff --git a/sandboxd/pool/liveness.go b/sandboxd/pool/liveness.go index c87398ba..1cb65bf4 100644 --- a/sandboxd/pool/liveness.go +++ b/sandboxd/pool/liveness.go @@ -129,6 +129,9 @@ func (m *Manager) watchedClaims(vmName string) []liveSuspect { func (m *Manager) recoverVMM(ctx context.Context, sb *types.Sandbox) { sb.Transition.Lock() defer sb.Transition.Unlock() + if m.growthOwnsHold(sb.ID) { + return + } now := time.Now() m.mu.Lock() live := m.claimed[sb.ID] == sb && runningShaped(sb) && !m.leaseLapsed(sb, now) @@ -204,6 +207,9 @@ func (m *Manager) restartFailed(ctx context.Context, sb *types.Sandbox) error { // restartLocked cold-boots sb's VM from its own disk and republishes it; the caller holds sb.Transition. func (m *Manager) restartLocked(ctx context.Context, sb *types.Sandbox) error { + if m.growthOwnsHold(sb.ID) { + return growBusy() + } ctx = context.WithoutCancel(ctx) if err := m.eng.Start(ctx, sb.VMName); err != nil { m.stopVM(ctx, sb.VMName) diff --git a/sandboxd/pool/managed_attachment.go b/sandboxd/pool/managed_attachment.go index 03b835df..4e48946f 100644 --- a/sandboxd/pool/managed_attachment.go +++ b/sandboxd/pool/managed_attachment.go @@ -208,7 +208,7 @@ func (m *Manager) finishManaged(id string, clears []string) bool { if !ok { return true } - if r.fault != nil { + if r.fault != nil || r.growthOwnsHold(id) { return false } commit := m.store.commit @@ -264,6 +264,13 @@ func (m *Manager) recoverManaged(ctx context.Context, claims map[string]*types.S } m.mu.Unlock() sb, exists := claims[h.SandboxID] + if m.growthOwnsHold(h.SandboxID) { + owned[h.VMName] = true + if !exists || !sleepMatchesClaim(h, sb) { + return owned, managedStorageError() + } + continue + } if h.Sleep != nil && h.State != holdRetiring { if !exists || !sleepMatchesClaim(h, sb) { return owned, managedStorageError() @@ -334,7 +341,7 @@ func (m *Manager) managedLiveAdd(id string, volumes []resolvedVolume) error { if !ok { return nil } - if r.fault != nil || h.State != holdClaimed { + if r.fault != nil || h.State != holdClaimed || r.growthOwnsHold(id) { return managedStorageError() } for _, v := range volumes { @@ -367,7 +374,7 @@ func (m *Manager) managedLiveRemove(id string, volumes []types.Volume) error { if !ok { return nil } - if r.fault != nil || h.State != holdClaimed { + if r.fault != nil || h.State != holdClaimed || r.growthOwnsHold(id) { return managedStorageError() } remove := map[string]bool{} diff --git a/sandboxd/pool/managed_grow.go b/sandboxd/pool/managed_grow.go new file mode 100644 index 00000000..6b4161de --- /dev/null +++ b/sandboxd/pool/managed_grow.go @@ -0,0 +1,437 @@ +package pool + +import ( + "context" + "errors" + "slices" + + "github.com/cocoonstack/sandbox/sandboxd/engine" + "github.com/cocoonstack/sandbox/sandboxd/types" +) + +type volumeGrowEngine interface { + InspectDisk(context.Context, engine.DiskRef) (engine.DiskInspection, error) + GrowDisk(context.Context, engine.DiskGrowRequest) (engine.DiskGrowResult, error) + ObserveVolume(context.Context, string, string, string) (engine.GuestVolumeObservation, error) +} + +type managedGrowth struct { + Request types.VolumeGrow `json:"request"` + Runtime engine.DiskGrowRequest `json:"runtime"` + SandboxID string `json:"sandbox_id"` + VMName string `json:"vm_name"` + Guest engine.GuestVolumeObservation `json:"guest"` + State string `json:"state"` + Outcome string `json:"outcome"` + GuestSizeBytes int64 `json:"guest_size_bytes"` +} + +func (g *managedGrowth) fenced() bool { + return g != nil && g.State != "completed" && g.Outcome != "old_vmm_gone" +} +func growBusy() error { + return volumeError(409, "volume_grow_busy", "volume growth owns the sandbox lifecycle") +} +func growRecovery() error { + return volumeError(409, "volume_grow_recovery_required", "volume growth requires recovery; retain the operation ID and retry after recovery") +} +func (r *volumeRegistry) growthOwnsHold(id string) bool { + for _, v := range r.data.Volumes { + if v.Growth != nil && (v.Growth.fenced() || r.fault != nil) && v.Growth.SandboxID == id { + return true + } + } + return false +} +func (m *Manager) growthOwnsHold(id string) bool { + if m.managed == nil { + return false + } + m.managed.mu.Lock() + defer m.managed.mu.Unlock() + return m.managed.growthOwnsHold(id) +} +func (m *Manager) growthOwnsVM(name string) bool { + if m.managed == nil { + return false + } + m.managed.mu.Lock() + defer m.managed.mu.Unlock() + for _, v := range m.managed.data.Volumes { + if v.Growth != nil && (v.Growth.fenced() || m.managed.fault != nil) && v.Growth.VMName == name { + return true + } + } + return false +} + +func (r *volumeRegistry) growVolume(name, expectedID, tenant string, req types.VolumeGrow) (managedVolume, error) { + if r.fault != nil { + return managedVolume{}, managedStorageError() + } + v, ok := r.data.Volumes[name] + if !ok || tenant != "" && tenant != v.Tenant { + return managedVolume{}, managedNotFound() + } + if expectedID == "" { + return v, volumeError(428, "volume_precondition_required", "If-Match must identify the volume") + } + if string(v.ID) != expectedID { + return v, volumeError(412, "volume_precondition_failed", "volume identity does not match") + } + if v.Growth != nil && req.OperationID == v.Growth.Request.OperationID { + if req != v.Growth.Request { + return v, volumeError(409, "volume_grow_conflict", "operation ID has different parameters") + } + return v, nil + } + if req.ExpectedGeneration != v.CapacityGeneration { + return v, volumeError(412, "volume_precondition_failed", "capacity generation does not match") + } + if v.Growth.fenced() { + return v, growBusy() + } + if v.State != volumeReady { + return v, growRecovery() + } + if req.SizeGiB<<30 <= v.Size { + return v, volumeError(400, "invalid_volume_grow", "target must exceed committed capacity") + } + if v.CapacityGeneration == ^uint64(0) { + return v, volumeError(409, "volume_generation_exhausted", "capacity generation is exhausted") + } + return v, nil +} + +func (m *Manager) GrowVolume(ctx context.Context, name, expectedID string, req types.VolumeGrow, tenant string) (types.ManagedVolumeInfo, error) { + if !types.ValidVolumeName(name) || req.SizeGiB < 1 || req.SizeGiB > 1024 || !engine.ValidGrowID(req.OperationID) { + return types.ManagedVolumeInfo{}, volumeError(400, "invalid_volume_grow", "invalid volume name, size_gib (1 through 1024), or canonical operation UUID") + } + r := m.managed + if r == nil { + return types.ManagedVolumeInfo{}, managedNotFound() + } + r.mu.Lock() + v, err := r.growVolume(name, expectedID, tenant, req) + if err != nil { + r.mu.Unlock() + return types.ManagedVolumeInfo{}, err + } + if v.Growth != nil && v.Growth.Request == req && !v.Growth.fenced() { + info := r.info(v) + r.mu.Unlock() + if v.Growth.State != "completed" { + return info, growRecovery() + } + return info, nil + } + var owner string + if v.Growth != nil && v.Growth.Request == req { + owner = v.Growth.SandboxID + } else { + holders := r.holders(v.ID) + if len(holders) == 1 { + owner = holders[0].SandboxID + } + } + r.mu.Unlock() + ge, supported := m.eng.(volumeGrowEngine) + if !supported { + return types.ManagedVolumeInfo{}, volumeError(503, "volume_grow_unsupported", "runtime does not support managed volume growth") + } + m.mu.Lock() + sb := m.claimed[owner] + m.mu.Unlock() + if sb == nil { + return types.ManagedVolumeInfo{}, growRecovery() + } + sb.Transition.Lock() + defer sb.Transition.Unlock() + r.mu.Lock() + v, err = r.growVolume(name, expectedID, tenant, req) + r.mu.Unlock() + if err != nil { + return types.ManagedVolumeInfo{}, err + } + if v.Growth == nil || v.Growth.Request != req { + if err := m.admitGrowth(ctx, sb, v, req, ge); err != nil { + return types.ManagedVolumeInfo{}, err + } + } + if err := m.driveGrowth(ctx, name, sb, ge); err != nil { + return types.ManagedVolumeInfo{}, err + } + r.mu.Lock() + defer r.mu.Unlock() + v = r.data.Volumes[name] + info := r.info(v) + if v.Growth.State == "recovery_required" { + return info, growRecovery() + } + return info, nil +} + +func (m *Manager) growthHolderLocked(sb *types.Sandbox, v managedVolume) (volumeHold, error) { + r := m.managed + h, ok := r.data.Holds[sb.ID] + _, outbound := m.view.Load().out.Resolve(sb, m.claimPooledLocked) + _, removing := m.pendingRemovals[h.VMName] + if !ok || m.claimed[sb.ID] != sb || h.State != holdClaimed || h.Sleep != nil || !sleepMatchesClaim(h, sb) || sb.Failed != "" || sb.HibernateSnap != "" || sb.PendingSnap != "" || m.volumeRuns[sb] != nil || outbound || removing || h.Refs[0].ID != v.ID || !h.Refs[0].Volume.RW() || h.Refs[0].Volume.Mount == "" || len(r.holders(v.ID)) != 1 || r.growthOwnsHold(sb.ID) { + return h, growBusy() + } + if r.verify(v) != nil { + return h, managedStorageError() + } + return h, nil +} + +func (m *Manager) admitGrowth(ctx context.Context, sb *types.Sandbox, v managedVolume, req types.VolumeGrow, ge volumeGrowEngine) error { + r := m.managed + m.mu.Lock() + r.mu.Lock() + h, err := m.growthHolderLocked(sb, v) + r.mu.Unlock() + m.mu.Unlock() + if err != nil { + return err + } + rec, present, err := m.eng.Inspect(ctx, h.VMName) + if err != nil || !present || rec.ID == "" || rec.Config.Name != h.VMName || rec.State != vmStateRunning || len(rec.AttachedDevices.Disks) != 1 { + return growRecovery() + } + disk := rec.AttachedDevices.Disks[0] + if disk.ID == "" || disk.Name != v.Name || disk.Path != r.path(v.ID) || disk.ReadOnly { + return growRecovery() + } + inspection, err := ge.InspectDisk(ctx, engine.DiskRef{VMID: rec.ID, DiskID: disk.ID}) + if err != nil || inspection.Error != nil || inspection.GuestBlockStatus != "unobserved" || inspection.Live.State != "observed" || inspection.Live.Binding == nil { + return volumeError(503, "volume_grow_unsupported", "runtime cannot provide a qualified disk binding") + } + binding := *inspection.Live.Binding + p, err := r.process(rec.PID) + if err != nil || p != h.Process || binding.Validate() != nil || binding.Ref != inspection.Requested || binding.Ref.VMID != rec.ID || binding.Ref.DiskID != disk.ID || binding.VMM != (engine.DiskProcess{BootID: p.BootID, PID: p.PID, Start: p.Start}) || binding.Serial != v.Name || binding.Path != r.path(v.ID) || binding.Backing.Device != v.Backing.Device || binding.Backing.Inode != v.Backing.Inode || binding.ObservedBytes != v.Size || binding.Revision == ^uint64(0) { + return growRecovery() + } + if inspection.LatestReceipt != nil && inspection.LatestReceipt.Outcome == "outcome_unknown" { + return growRecovery() + } + guest, err := ge.ObserveVolume(ctx, rec.VsockSocket, v.Name, h.Refs[0].Volume.Mount) + if err != nil || !guest.Valid() || guest.Serial != v.Name || guest.SizeBytes != v.Size { + return growRecovery() + } + m.mu.Lock() + defer m.mu.Unlock() + r.mu.Lock() + defer r.mu.Unlock() + current, err := r.growVolume(v.Name, string(v.ID), v.Tenant, req) + if err != nil { + return err + } + if current.Growth != nil && current.Growth.Request == req { + return nil + } + if _, err := m.growthHolderLocked(sb, current); err != nil { + return err + } + if current.Size != v.Size || current.Backing != v.Backing { + return growRecovery() + } + if r.data.Version != 3 { + r.data.Version = 3 + if err := r.commit(); err != nil { + return err + } + } + current.State = volumeGrowing + current.Growth = &managedGrowth{Request: req, Runtime: engine.DiskGrowRequest{Expected: binding, ID: req.OperationID, TargetBytes: req.SizeGiB << 30}, SandboxID: sb.ID, VMName: h.VMName, Guest: guest, State: "pending", Outcome: "unobserved"} + r.data.Volumes[v.Name] = current + return r.commit() +} + +func (m *Manager) driveGrowth(ctx context.Context, name string, sb *types.Sandbox, ge volumeGrowEngine) error { + r := m.managed + r.mu.Lock() + v := r.data.Volumes[name] + g := *v.Growth + r.mu.Unlock() + if !g.fenced() { + return nil + } + inspection, inspectErr := ge.InspectDisk(ctx, g.Runtime.Expected.Ref) + if inspection.Requested != g.Runtime.Expected.Ref || inspection.GuestBlockStatus != "unobserved" { + return nil + } + receipt, live := inspection.LatestReceipt, inspection.Live + if receipt == nil || !receipt.Request.Equal(g.Runtime) { + // Only the original source binding permits a first dispatch. A known receipt is query-only. + if sb == nil || g.Outcome != "unobserved" || g.State != "pending" || inspectErr != nil || inspection.Error != nil || live.State != "observed" || live.Binding == nil || !live.Binding.Equal(g.Runtime.Expected) { + return m.saveGrowth(v, g, "recovery_required", g.Outcome, 0) + } + if receipt != nil && receipt.Revision > g.Runtime.Expected.Revision { + return m.saveGrowth(v, g, "recovery_required", g.Outcome, 0) + } + result, _ := ge.GrowDisk(ctx, g.Runtime) + receipt, live = result.Receipt, result.Live + if receipt == nil { + return nil + } + } + if !receipt.Request.Equal(g.Runtime) || receipt.Revision != g.Runtime.Expected.Revision+1 { + return m.saveGrowth(v, g, "recovery_required", g.Outcome, 0) + } + switch receipt.Outcome { + case "outcome_unknown": + return m.saveGrowth(v, g, "recovery_required", "outcome_unknown", 0) + case "old_vmm_gone": + p := g.Runtime.Expected.VMM + if !r.processGone(processIdentity{BootID: p.BootID, PID: p.PID, Start: p.Start}) { + return m.saveGrowth(v, g, "recovery_required", g.Outcome, 0) + } + return m.saveGrowth(v, g, "recovery_required", "old_vmm_gone", 0) + case "acknowledged": + if live.State != "observed" || live.Binding == nil || live.Binding.Validate() != nil || !live.Binding.SameIdentity(g.Runtime.Expected) || live.Binding.ObservedBytes != g.Runtime.TargetBytes || live.Binding.Revision != receipt.Revision { + return m.saveGrowth(v, g, "recovery_required", "acknowledged", 0) + } + default: + return m.saveGrowth(v, g, "recovery_required", g.Outcome, 0) + } + if err := m.saveGrowth(v, g, "pending", "acknowledged", 0); err != nil { + return err + } + if sb == nil { + return nil + } + guest, err := ge.ObserveVolume(ctx, sb.VsockSocket, v.Name, g.Guest.Mount) + if err != nil { + return nil + } + if !guest.Valid() || !guest.SameIdentity(g.Guest) { + return m.saveGrowth(v, g, "recovery_required", "acknowledged", 0) + } + if guest.SizeBytes == g.Guest.SizeBytes { + return nil + } + if guest.SizeBytes != g.Runtime.TargetBytes { + return m.saveGrowth(v, g, "recovery_required", "acknowledged", guest.SizeBytes) + } + closing, closeErr := ge.InspectDisk(ctx, g.Runtime.Expected.Ref) + if closeErr != nil || closing.Error != nil || closing.Requested != g.Runtime.Expected.Ref || closing.LatestReceipt == nil || !closing.LatestReceipt.Request.Equal(g.Runtime) || closing.LatestReceipt.Revision != receipt.Revision || closing.LatestReceipt.Outcome != "acknowledged" || closing.Live.State != "observed" || closing.Live.Binding == nil || !closing.Live.Binding.Equal(*live.Binding) { + return m.saveGrowth(v, g, "recovery_required", "acknowledged", guest.SizeBytes) + } + p, processErr := r.process(g.Runtime.Expected.VMM.PID) + if processErr != nil || p != (processIdentity{BootID: g.Runtime.Expected.VMM.BootID, PID: g.Runtime.Expected.VMM.PID, Start: g.Runtime.Expected.VMM.Start}) { + return m.saveGrowth(v, g, "recovery_required", "acknowledged", guest.SizeBytes) + } + return m.saveGrowth(v, g, "completed", "acknowledged", guest.SizeBytes) +} + +func (m *Manager) saveGrowth(v managedVolume, g managedGrowth, state, outcome string, guestSize int64) error { + r := m.managed + r.mu.Lock() + defer r.mu.Unlock() + current := r.data.Volumes[v.Name] + if r.fault != nil || current.ID != v.ID || current.Growth == nil || current.Growth.Request != g.Request { + return managedStorageError() + } + g.State, g.Outcome, g.GuestSizeBytes = state, outcome, guestSize + current.Growth = &g + if state == "completed" { + if !current.Growth.fenced() && current.CapacityGeneration != g.Request.ExpectedGeneration { + return managedStorageError() + } + actual, err := inspectBacking(r.path(v.ID)) + if err != nil || actual.Device != v.Backing.Device || actual.Inode != v.Backing.Inode || actual.Size != g.Runtime.TargetBytes { + return managedStorageError() + } + current.Size, current.Backing, current.State = actual.Size, actual, volumeReady + current.CapacityGeneration++ + for id, h := range r.data.Holds { + h.Refs = slices.Clone(h.Refs) + for i := range h.Refs { + if h.Refs[i].ID == v.ID { + h.Refs[i].Backing = actual + } + } + r.data.Holds[id] = h + } + } else if outcome == "old_vmm_gone" { + current.State = volumeRecovery + } + r.data.Volumes[v.Name] = current + return r.commit() +} + +func (r *volumeRegistry) validateGrowth(v managedVolume) error { + g := v.Growth + if g == nil { + if v.State == volumeGrowing { + return errors.New("growing volume lacks intent") + } + return nil + } + b := g.Runtime.Expected + if r.data.Version != 3 || !engine.ValidGrowID(g.Request.OperationID) || g.Runtime.ID != g.Request.OperationID || g.Request.SizeGiB < 1 || g.Request.SizeGiB > 1024 || g.Runtime.TargetBytes != g.Request.SizeGiB<<30 || b.Validate() != nil || b.Revision == ^uint64(0) || b.Path != r.path(v.ID) || b.Serial != v.Name || b.Backing.Inode != v.Backing.Inode || b.Backing.Device != v.Backing.Device || b.ObservedBytes < 1<<30 || b.ObservedBytes%(1<<30) != 0 || b.ObservedBytes >= g.Runtime.TargetBytes || g.Request.ExpectedGeneration == ^uint64(0) || !g.Guest.Valid() || g.Guest.Serial != v.Name || g.Guest.SizeBytes != b.ObservedBytes || g.SandboxID == "" || g.VMName == "" { + return errors.New("invalid volume growth intent") + } + switch g.Outcome { + case "unobserved", "outcome_unknown", "acknowledged", "old_vmm_gone": + default: + return errors.New("invalid growth outcome") + } + switch g.State { + case "completed": + if (v.State != volumeReady && v.State != volumeDeleting && v.State != volumeRecovery) || g.Outcome != "acknowledged" || g.GuestSizeBytes != g.Runtime.TargetBytes || v.Size != g.Runtime.TargetBytes || g.Request.ExpectedGeneration == ^uint64(0) || v.CapacityGeneration != g.Request.ExpectedGeneration+1 { + return errors.New("inconsistent completed growth") + } + case "pending", "recovery_required": + if g.State == "pending" && g.Outcome != "unobserved" && g.Outcome != "acknowledged" { + return errors.New("pending growth has terminal runtime outcome") + } + if v.Size != b.ObservedBytes || v.CapacityGeneration != g.Request.ExpectedGeneration || g.Outcome == "old_vmm_gone" && ((v.State != volumeRecovery && v.State != volumeDeleting) || g.State != "recovery_required") || g.Outcome != "old_vmm_gone" && v.State != volumeGrowing { + return errors.New("inconsistent unresolved growth") + } + if g.fenced() { + h, ok := r.data.Holds[g.SandboxID] + if !ok || h.VMName != g.VMName || h.State != holdClaimed || h.Sleep != nil || len(h.Refs) != 1 || h.Refs[0].ID != v.ID || !h.Refs[0].Volume.RW() || h.Refs[0].Volume.Mount == "" || h.Process != (processIdentity{BootID: b.VMM.BootID, PID: b.VMM.PID, Start: b.VMM.Start}) { + return errors.New("growth lost owner") + } + } + default: + return errors.New("invalid growth state") + } + return nil +} + +func (m *Manager) recoverGrowth(ctx context.Context, claims map[string]*types.Sandbox) error { + r := m.managed + if r == nil { + return nil + } + r.mu.Lock() + if r.fault != nil { + r.mu.Unlock() + return managedStorageError() + } + var pending []managedVolume + for _, v := range r.data.Volumes { + if v.Growth.fenced() { + pending = append(pending, v) + } + } + r.mu.Unlock() + ge, ok := m.eng.(volumeGrowEngine) + if !ok && len(pending) > 0 { + return managedStorageError() + } + for _, v := range pending { + sb := claims[v.Growth.SandboxID] + h, held := m.managedHold(v.Growth.SandboxID) + if sb != nil && (!held || !sleepMatchesClaim(h, sb) || sb.VMName != v.Growth.VMName || sb.Tenant != v.Tenant) { + sb = nil + } + if err := m.driveGrowth(ctx, v.Name, sb, ge); err != nil { + return err + } + } + return nil +} diff --git a/sandboxd/pool/managed_grow_test.go b/sandboxd/pool/managed_grow_test.go new file mode 100644 index 00000000..d2b477ad --- /dev/null +++ b/sandboxd/pool/managed_grow_test.go @@ -0,0 +1,556 @@ +package pool + +import ( + "context" + "encoding/json/v2" + "errors" + "os" + "path/filepath" + "sync" + "testing" + "time" + + "github.com/cocoonstack/sandbox/sandboxd/config" + "github.com/cocoonstack/sandbox/sandboxd/engine" + "github.com/cocoonstack/sandbox/sandboxd/tenants" + "github.com/cocoonstack/sandbox/sandboxd/types" +) + +type growthTestEngine struct { + *managedTestEngine + growMu sync.Mutex + binding engine.DiskBinding + receipt *engine.DiskReceipt + calls int + observations int + lag, unknown, lostResponse, mutateAfterGuest bool + guestChange bool + entered, resume chan struct{} + intentSeen bool +} + +func (e *growthTestEngine) DiskAttach(ctx context.Context, name string, spec engine.VolumeSpec) error { + if err := e.managedTestEngine.DiskAttach(ctx, name, spec); err != nil { + return err + } + e.mu.Lock() + defer e.mu.Unlock() + if e.disks == nil { + e.disks = map[string][]types.VMDisk{} + } + e.disks[name] = []types.VMDisk{{ID: "actual-ch-id", Name: spec.Name, Path: spec.Path, ReadOnly: !spec.RW}} + return nil +} + +func (e *growthTestEngine) Inspect(ctx context.Context, name string) (types.VMRecord, bool, error) { + rec, found, err := e.managedTestEngine.Inspect(ctx, name) + rec.ID = "immutable-grow-vm" + for i := range rec.AttachedDevices.Disks { + rec.AttachedDevices.Disks[i].ID = "actual-ch-id" + } + return rec, found, err +} +func (e *growthTestEngine) InspectDisk(_ context.Context, ref engine.DiskRef) (engine.DiskInspection, error) { + e.growMu.Lock() + defer e.growMu.Unlock() + b := e.binding + if e.mutateAfterGuest && e.observations > 1 { + b.VMM.Start++ + } + return engine.DiskInspection{Requested: ref, Live: engine.DiskObservation{State: "observed", Binding: &b}, LatestReceipt: e.receipt, GuestBlockStatus: "unobserved"}, nil +} +func (e *growthTestEngine) GrowDisk(_ context.Context, req engine.DiskGrowRequest) (engine.DiskGrowResult, error) { + e.growMu.Lock() + e.calls++ + raw, err := os.ReadFile(filepath.Join(filepath.Dir(filepath.Dir(req.Expected.Path)), "registry.json")) + var saved volumeRegistryData + if err == nil && json.Unmarshal(raw, &saved) == nil { + v := saved.Volumes[req.Expected.Serial] + e.intentSeen = saved.Version == 3 && v.Growth != nil && v.Growth.Runtime.Equal(req) && v.State == volumeGrowing + } + if !e.intentSeen { + e.growMu.Unlock() + return engine.DiskGrowResult{}, errors.New("effect before intent") + } + entered, resume := e.entered, e.resume + e.growMu.Unlock() + if entered != nil { + close(entered) + <-resume + } + e.growMu.Lock() + defer e.growMu.Unlock() + if err := os.Truncate(req.Expected.Path, req.TargetBytes); err != nil { + return engine.DiskGrowResult{}, err + } + outcome := "acknowledged" + if e.unknown { + outcome = "outcome_unknown" + } + e.receipt = &engine.DiskReceipt{Request: req, Revision: req.Expected.Revision + 1, Outcome: outcome} + e.binding.ObservedBytes, e.binding.Revision = req.TargetBytes, req.Expected.Revision+1 + b := e.binding + result := engine.DiskGrowResult{Receipt: e.receipt, Live: engine.DiskObservation{State: "observed", Binding: &b}, GuestBlockStatus: "unobserved"} + if e.lostResponse { + return engine.DiskGrowResult{}, context.DeadlineExceeded + } + return result, nil +} +func (e *growthTestEngine) ObserveVolume(_ context.Context, _ string, serial, mount string) (engine.GuestVolumeObservation, error) { + e.growMu.Lock() + defer e.growMu.Unlock() + e.observations++ + size := e.binding.ObservedBytes + if e.lag && e.receipt != nil { + size = e.receipt.Request.Expected.ObservedBytes + } + g := engine.GuestVolumeObservation{BootID: "guest-boot", Serial: serial, Device: "/dev/vdc", DevNum: "8:32", MountID: "27", Mount: mount, Root: "/", Mode: "rw", SizeBytes: size} + if e.guestChange && e.observations > 1 { + g.MountID = "28" + } + return g, nil +} + +func openGrowthTest(t *testing.T, cfg *config.Config, e *growthTestEngine) *Manager { + t.Helper() + m, err := NewManager(t.Context(), cfg, e, testSecrets(t), testBackends(t, cfg)) + if err != nil { + t.Fatal(err) + } + m.managed.process = e.identity + t.Cleanup(func() { _ = m.Close() }) + return m +} +func newGrowthTest(t *testing.T) (*Manager, *growthTestEngine, *types.Sandbox, *config.Config, types.ManagedVolumeInfo) { + t.Helper() + cfg := managedConfig(t) + e := &growthTestEngine{managedTestEngine: newManagedEngine()} + m := openGrowthTest(t, cfg, e) + if err := m.PutTenant(t.Context(), tenants.TenantSpec{Name: "t1", Token: "token"}); err != nil { + t.Fatal(err) + } + v := createManagedTest(t, m, "data") + opts := managedOptions("data") + opts.Volumes[0].AttachOnly = false + sb, err := m.ClaimProvision(t.Context(), testKey, opts) + if err != nil { + t.Fatal(err) + } + h, _ := m.managedHold(sb.ID) + ref := h.Refs[0] + e.binding = engine.DiskBinding{Ref: engine.DiskRef{VMID: "immutable-grow-vm", DiskID: "actual-ch-id"}, VMM: engine.DiskProcess{BootID: h.Process.BootID, PID: h.Process.PID, Start: h.Process.Start}, Serial: "data", Path: ref.Path, Mode: "read_write", Format: "raw", Backing: engine.DiskBacking{Device: ref.Backing.Device, Inode: ref.Backing.Inode, FDs: []engine.DiskFD{{Number: 250, Access: "read_write"}, {Number: 261, Access: "read_write"}}}, ObservedBytes: ref.Backing.Size} + return m, e, sb, cfg, v +} +func growRequest() types.VolumeGrow { + return types.VolumeGrow{SizeGiB: 2, OperationID: "b51b3d2b-4ccc-41d5-b430-3046340c30b0"} +} + +func TestManagedGrowthCommitReplayCASAndRestart(t *testing.T) { + m, e, sb, cfg, v := newGrowthTest(t) + req := growRequest() + got, err := m.GrowVolume(t.Context(), "data", v.ID, req, "t1") + if err != nil || got.SizeBytes != 2<<30 || got.CapacityGeneration != 1 || got.Growth.State != "completed" || got.Growth.GuestSizeBytes != 2<<30 || got.Growth.FilesystemStatus != "not_managed" || !e.intentSeen { + t.Fatalf("grow %+v %v", got, err) + } + h, _ := m.managedHold(sb.ID) + if h.Refs[0].Backing.Size != got.SizeBytes { + t.Fatal("held backing not committed atomically") + } + if _, err = m.GrowVolume(t.Context(), "data", v.ID, req, "t1"); err != nil || e.calls != 1 { + t.Fatalf("replay %v calls %d", err, e.calls) + } + changed := req + changed.SizeGiB++ + _, err = m.GrowVolume(t.Context(), "data", v.ID, changed, "t1") + requireVolumeStatus(t, err, 409) + changed.OperationID = "c51b3d2b-4ccc-41d5-b430-3046340c30b0" + _, err = m.GrowVolume(t.Context(), "data", v.ID, changed, "t1") + requireVolumeStatus(t, err, 412) + changed.ExpectedGeneration = 1 + if _, err = m.GrowVolume(t.Context(), "data", v.ID, changed, "t1"); err != nil { + t.Fatal(err) + } + _, err = m.GrowVolume(t.Context(), "data", v.ID, req, "t1") + requireVolumeStatus(t, err, 412) + if err := m.managed.validate(); err != nil { + t.Fatal(err) + } + if err := m.FlushClaims(); err != nil { + t.Fatal(err) + } + m.Close() + m = openGrowthTest(t, cfg, e) + if err := m.Reconcile(t.Context()); err != nil { + t.Fatal(err) + } + got, err = m.GetVolume("data", "t1") + if err != nil || got.CapacityGeneration != 2 || got.SizeBytes != 3<<30 { + t.Fatalf("reopened %+v %v", got, err) + } +} + +func TestManagedGrowthACKGuestLagAndLostResponseRecoverWithoutDispatch(t *testing.T) { + for _, lost := range []bool{false, true} { + t.Run(map[bool]string{false: "guest lag", true: "lost response"}[lost], func(t *testing.T) { + m, e, sb, cfg, v := newGrowthTest(t) + e.lag = true + e.lostResponse = lost + got, err := m.GrowVolume(t.Context(), "data", v.ID, growRequest(), "t1") + if err != nil || got.Growth.State != "pending" || got.CapacityGeneration != 0 || got.SizeBytes != 1<<30 { + t.Fatalf("pending %+v %v", got, err) + } + if !lost && got.Growth.RuntimeOutcome != "acknowledged" { + t.Fatal("ACK not durable before guest") + } + if m.managedPaused(sb) { + t.Fatal("growth paused data plane") + } + if err := m.FlushClaims(); err != nil { + t.Fatal(err) + } + m.Close() + e.lag = false + e.lostResponse = false + m = openGrowthTest(t, cfg, e) + if err := m.Reconcile(t.Context()); err != nil { + t.Fatal(err) + } + got, err = m.GrowVolume(t.Context(), "data", v.ID, growRequest(), "t1") + if err != nil || got.Growth.State != "completed" || got.CapacityGeneration != 1 || e.calls != 1 { + t.Fatalf("recovery %+v %v calls=%d", got, err, e.calls) + } + }) + } +} + +func TestManagedGrowthUnknownFencesLifecycleUntilExplicitExactStop(t *testing.T) { + m, e, sb, _, v := newGrowthTest(t) + e.unknown = true + _, err := m.GrowVolume(t.Context(), "data", v.ID, growRequest(), "t1") + requireVolumeStatus(t, err, 409) + for range 2 { + _, err = m.GrowVolume(t.Context(), "data", v.ID, growRequest(), "t1") + requireVolumeStatus(t, err, 409) + } + if e.calls != 1 { + t.Fatal("unknown was redispatched") + } + requireVolumeStatus(t, m.Release(t.Context(), sb.ID, Cred{Operator: true}), 409) + m.mu.Lock() + sb.Deadline = time.Now().Add(-time.Second) + m.mu.Unlock() + m.reapOnce(t.Context()) + if _, ok := m.Sandbox(sb.ID); !ok { + t.Fatal("ordinary reap dropped grow owner") + } + if m.removeVM(t.Context(), sb.VMName) || m.finishManaged(sb.ID, nil) { + t.Fatal("destructive fallback escaped guard") + } + if _, _, err := m.beginVolumeRun(t.Context(), sb); err == nil { + t.Fatal("live mutation admitted") + } + if err := m.Hibernate(t.Context(), sb.ID, Cred{Operator: true}); err == nil { + t.Fatal("sleep admitted") + } + if err := m.DeleteVolume(t.Context(), "data", v.ID, "t1"); err == nil { + t.Fatal("delete admitted") + } + e.mu.Lock() + removedBefore := len(e.removes) + e.mu.Unlock() + m.stopVM(t.Context(), sb.VMName) + m.recoverVMM(t.Context(), sb) + e.mu.Lock() + if e.stopped[sb.VMName] || len(e.removes) != removedBefore { + t.Fatal("growth automatically stopped/removed VM") + } + e.mu.Unlock() + e.growMu.Lock() + e.receipt.Outcome = "old_vmm_gone" + e.growMu.Unlock() + _, err = m.GrowVolume(t.Context(), "data", v.ID, growRequest(), "t1") + requireVolumeStatus(t, err, 409) + if !m.growthOwnsHold(sb.ID) { + t.Fatal("receipt alone proved process death") + } + e.processUnknown.Store(true) + _, err = m.GrowVolume(t.Context(), "data", v.ID, growRequest(), "t1") + requireVolumeStatus(t, err, 409) + if !m.growthOwnsHold(sb.ID) { + t.Fatal("process read error proved death") + } + e.processUnknown.Store(false) + m.managed.process = func(int) (processIdentity, error) { return processIdentity{}, os.ErrNotExist } + got, err := m.GrowVolume(t.Context(), "data", v.ID, growRequest(), "t1") + requireVolumeStatus(t, err, 409) + if got.Growth.RuntimeOutcome != "old_vmm_gone" || got.SizeBytes != 1<<30 || got.CapacityGeneration != 0 || m.growthOwnsHold(sb.ID) { + t.Fatalf("quarantine %+v", got) + } + if err := m.Release(t.Context(), sb.ID, Cred{Operator: true}); err != nil { + t.Fatal(err) + } + if err := m.DeleteVolume(t.Context(), "data", v.ID, "t1"); err != nil { + t.Fatal("known target quarantine delete", err) + } +} + +func TestManagedGrowthAdmissionSerializesOrdinaryReleaseAndReap(t *testing.T) { + m, e, sb, _, v := newGrowthTest(t) + e.entered = make(chan struct{}) + e.resume = make(chan struct{}) + done := make(chan error, 1) + go func() { _, err := m.GrowVolume(t.Context(), "data", v.ID, growRequest(), "t1"); done <- err }() + select { + case <-e.entered: + case err := <-done: + t.Fatalf("grow did not dispatch: %v", err) + } + if m.managedLifecycle(sb) { + t.Fatal("test must cover release without Transition") + } + requireVolumeStatus(t, m.Release(t.Context(), sb.ID, Cred{Operator: true}), 409) + m.mu.Lock() + sb.Deadline = time.Now().Add(-time.Second) + m.mu.Unlock() + m.reapOnce(t.Context()) + if _, ok := m.Sandbox(sb.ID); !ok { + t.Fatal("reap dropped grow owner") + } + close(e.resume) + if err := <-done; err != nil { + t.Fatal(err) + } +} + +func TestManagedGrowthRejectsChangedGuestAndClosingRuntime(t *testing.T) { + for _, runtimeChange := range []bool{false, true} { + t.Run(map[bool]string{false: "mount replaced", true: "VMM replaced after guest"}[runtimeChange], func(t *testing.T) { + m, e, _, _, v := newGrowthTest(t) + e.guestChange = !runtimeChange + e.mutateAfterGuest = runtimeChange + _, err := m.GrowVolume(t.Context(), "data", v.ID, growRequest(), "t1") + requireVolumeStatus(t, err, 409) + got, err := m.GetVolume("data", "t1") + if err != nil || got.CapacityGeneration != 0 || got.SizeBytes != 1<<30 || got.Growth.State != "recovery_required" { + t.Fatalf("false commit %+v %v", got, err) + } + }) + } +} + +func TestManagedGrowthSnapshotFailuresCannotDispatchOrRelease(t *testing.T) { + for _, failAt := range []int{1, 2, 3, 4} { + t.Run(string(rune('0'+failAt)), func(t *testing.T) { + m, e, sb, _, v := newGrowthTest(t) + calls := 0 + write := m.managed.commitFile + m.managed.commitFile = func(p string, b []byte) error { + calls++ + if calls == failAt { + return errors.New("injected snapshot failure") + } + return write(p, b) + } + _, err := m.GrowVolume(t.Context(), "data", v.ID, growRequest(), "t1") + if err == nil { + t.Fatal("storage failure hidden") + } + if failAt <= 2 && e.calls != 0 { + t.Fatal("runtime dispatch preceded durable intent") + } + if failAt >= 2 && (m.removeVM(t.Context(), sb.VMName) || m.finishManaged(sb.ID, nil)) { + t.Fatal("failed snapshot permitted teardown") + } + }) + } +} + +func TestManagedGrowthMissingClaimAndPriorOrphanPreserveOwner(t *testing.T) { + m, e, sb, _, v := newGrowthTest(t) + e.unknown = true + _, _ = m.GrowVolume(t.Context(), "data", v.ID, growRequest(), "t1") + h, _ := m.managedHold(sb.ID) + m.managed.mu.Lock() + m.managed.data.Orphans = map[string]processIdentity{sb.VMName: h.Process} + m.managed.mu.Unlock() + rec, _, _ := e.Inspect(t.Context(), sb.VMName) + live := map[string]types.VMRecord{sb.VMName: rec} + if err := m.recoverGrowth(t.Context(), map[string]*types.Sandbox{}); err != nil { + t.Fatal(err) + } + if _, err := m.recoverManaged(t.Context(), map[string]*types.Sandbox{}, live); err == nil { + t.Fatal("missing claim must fail closed") + } + if err := m.recoverManagedOrphans(t.Context(), map[string]*types.Sandbox{}, live, map[string]bool{}); err != nil { + t.Fatal(err) + } + if removed := m.sweepStaleVMs(t.Context(), live, map[string]bool{}); removed[sb.VMName] { + t.Fatal("swept grow owner") + } + if e.calls != 1 { + t.Fatal("startup redispatched unknown") + } + e.mu.Lock() + defer e.mu.Unlock() + if len(e.removes) > 0 { + t.Fatal("prior orphan removed grow owner") + } +} + +func TestManagedGrowthConcurrentReplayAndReadOnlyGET(t *testing.T) { + m, e, _, _, v := newGrowthTest(t) + e.lag = true + var wg sync.WaitGroup + errs := make(chan error, 8) + for range 8 { + wg.Go(func() { _, err := m.GrowVolume(t.Context(), "data", v.ID, growRequest(), "t1"); errs <- err }) + } + wg.Wait() + close(errs) + for err := range errs { + if err != nil { + t.Fatal(err) + } + } + if e.calls != 1 { + t.Fatalf("concurrent replay dispatched %d times", e.calls) + } + observations := e.observations + got, err := m.GetVolume("data", "t1") + if err != nil || got.Growth.State != "pending" || e.observations != observations { + t.Fatalf("GET performed recovery: %+v %v", got, err) + } + if _, err := m.GrowVolume(t.Context(), "data", v.ID, growRequest(), "foreign"); err == nil { + t.Fatal("foreign owner admitted") + } + path := m.managed.path(volumeID(v.ID)) + if err := os.Rename(path, path+".replaced"); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(path, nil, 0600); err != nil { + t.Fatal(err) + } + if _, err := m.GetVolume("data", "t1"); err == nil { + t.Fatal("GET accepted replaced growing backing") + } +} + +func TestManagedGrowthHostOrGuestTargetCannotReplaceACK(t *testing.T) { + m, e, _, _, v := newGrowthTest(t) + e.lostResponse = true + if _, err := m.GrowVolume(t.Context(), "data", v.ID, growRequest(), "t1"); err != nil { + t.Fatal(err) + } + e.receipt = nil + _, err := m.GrowVolume(t.Context(), "data", v.ID, growRequest(), "t1") + requireVolumeStatus(t, err, 409) + got, err := m.GetVolume("data", "t1") + if err != nil || got.CapacityGeneration != 0 || got.Growth.State != "recovery_required" || e.calls != 1 { + t.Fatalf("target inferred ACK: %+v %v calls %d", got, err, e.calls) + } +} + +func TestManagedGrowthRegistryRejectsInconsistentIntent(t *testing.T) { + for _, mutation := range []string{"version", "generation", "request", "owner", "sleep", "state", "backing", "guest", "receipt revision"} { + t.Run(mutation, func(t *testing.T) { + m, e, sb, _, v := newGrowthTest(t) + e.lag = true + if _, err := m.GrowVolume(t.Context(), "data", v.ID, growRequest(), "t1"); err != nil { + t.Fatal(err) + } + r := m.managed + current := r.data.Volumes["data"] + switch mutation { + case "version": + r.data.Version = 2 + case "generation": + current.CapacityGeneration++ + case "request": + current.Growth.Runtime.ID = "wrong" + case "owner": + delete(r.data.Holds, sb.ID) + case "sleep": + h := r.data.Holds[sb.ID] + h.Sleep = &managedSleep{} + r.data.Holds[sb.ID] = h + case "state": + current.State = volumeReady + case "backing": + current.Backing.Size++ + case "guest": + current.Growth.Guest.MountID = "" + case "receipt revision": + current.Growth.Runtime.Expected.Revision = ^uint64(0) + } + r.data.Volumes["data"] = current + if err := r.validate(); err == nil { + t.Fatal("invalid registry accepted") + } + }) + } +} + +func TestManagedGrowthRefusesUnmountedOrUnappliedOwner(t *testing.T) { + for _, mutation := range []string{"unmounted", "unapplied", "read only", "retiring", "live mutation"} { + t.Run(mutation, func(t *testing.T) { + m, e, sb, _, v := newGrowthTest(t) + switch mutation { + case "unmounted": + h := m.managed.data.Holds[sb.ID] + h.Refs[0].Volume.Mount = "" + m.managed.data.Holds[sb.ID] = h + case "unapplied": + sb.VolumesWant = nil + case "read only": + h := m.managed.data.Holds[sb.ID] + h.Refs[0].Volume.Mode = "" + m.managed.data.Holds[sb.ID] = h + case "retiring": + if err := m.retireManaged(sb.ID); err != nil { + t.Fatal(err) + } + case "live mutation": + m.volumeRuns[sb] = &volumeRun{} + } + _, err := m.GrowVolume(t.Context(), "data", v.ID, growRequest(), "t1") + requireVolumeStatus(t, err, 409) + if e.calls != 0 { + t.Fatal("ineligible holder dispatched growth") + } + }) + } +} + +func TestManagedGrowthMigratesVersionTwoBeforeIntentAndKeepsSleepValid(t *testing.T) { + m, _, _, _, v := newGrowthTest(t) + r := m.managed + r.data.Version = 2 + if err := r.commit(); err != nil { + t.Fatal(err) + } + write := r.commitFile + var snapshots []volumeRegistryData + r.commitFile = func(path string, raw []byte) error { + var snapshot volumeRegistryData + if err := json.Unmarshal(raw, &snapshot); err != nil { + return err + } + snapshots = append(snapshots, snapshot) + return write(path, raw) + } + if _, err := m.GrowVolume(t.Context(), "data", v.ID, growRequest(), "t1"); err != nil { + t.Fatal(err) + } + if len(snapshots) < 2 || snapshots[0].Version != 3 || snapshots[0].Volumes["data"].Growth != nil || snapshots[1].Volumes["data"].Growth == nil { + t.Fatal("migration was not durable before intent") + } + sleeping, _, sb, _ := newSleepTest(t) + sleeping.managed.data.Version = 3 + if err := sleeping.managed.commit(); err != nil { + t.Fatal(err) + } + requireSleep(t, sleeping, sb) + if err := sleeping.managed.validate(); err != nil { + t.Fatal("version three rejected existing sleep", err) + } +} diff --git a/sandboxd/pool/managed_recovery.go b/sandboxd/pool/managed_recovery.go index 00f3e6d6..0622791d 100644 --- a/sandboxd/pool/managed_recovery.go +++ b/sandboxd/pool/managed_recovery.go @@ -31,7 +31,7 @@ func (m *Manager) recoverManagedOrphans(ctx context.Context, claims map[string]* known[sb.VMName] = true } for name, rec := range live { - if !strings.HasPrefix(name, vmPrefix) || known[name] { + if !strings.HasPrefix(name, vmPrefix) || known[name] || m.growthOwnsVM(name) { continue } if rec.State == vmStateCreating { @@ -75,6 +75,9 @@ func (m *Manager) recoverManagedOrphans(ctx context.Context, claims map[string]* return err } for name, p := range orphans { + if m.growthOwnsVM(name) { + continue + } _ = m.eng.Remove(context.WithoutCancel(ctx), name) if !m.confirmGone(ctx, name) || !r.processGone(p) { return managedStorageError() diff --git a/sandboxd/pool/managed_sleep.go b/sandboxd/pool/managed_sleep.go index 7d9c03de..8324fdfd 100644 --- a/sandboxd/pool/managed_sleep.go +++ b/sandboxd/pool/managed_sleep.go @@ -28,6 +28,9 @@ func (m *Manager) managedSleepPreflight(ctx context.Context, sb *types.Sandbox) busy := m.volumeRuns[sb] != nil _, outbound := m.view.Load().out.Resolve(sb, m.claimPooledLocked) m.mu.Unlock() + if m.growthOwnsHold(sb.ID) { + return h, types.VMRecord{}, growBusy() + } if !ok || m.managedSleepHook == nil || h.State != holdClaimed || !sleepMatchesClaim(h, sb) || h.Refs[0].Volume.Mount == "" || !h.Refs[0].Volume.RW() || h.Refs[0].ID == "" || busy || outbound { return h, types.VMRecord{}, ErrVolumeCapture } @@ -62,6 +65,9 @@ func (m *Manager) managedSleepPreflight(ctx context.Context, sb *types.Sandbox) func (m *Manager) driveManagedSleepLocked(ctx context.Context, sb *types.Sandbox, target managedSleepTarget) error { h, ok := m.managedHold(sb.ID) + if m.growthOwnsHold(sb.ID) { + return growBusy() + } if !ok { return ErrVolumeCapture } diff --git a/sandboxd/pool/managed_sleep_registry.go b/sandboxd/pool/managed_sleep_registry.go index 5e9225ba..687696f0 100644 --- a/sandboxd/pool/managed_sleep_registry.go +++ b/sandboxd/pool/managed_sleep_registry.go @@ -67,7 +67,7 @@ func (r *volumeRegistry) commitManagedSleep(h volumeHold, next *managedSleep) er r.mu.Lock() defer r.mu.Unlock() current, ok := r.data.Holds[h.SandboxID] - if r.fault != nil || !ok || current.State != holdClaimed || (current.Sleep != nil && (h.Sleep == nil || current.Sleep.Operation != h.Sleep.Operation)) { + if r.fault != nil || r.growthOwnsHold(h.SandboxID) || !ok || current.State != holdClaimed || (current.Sleep != nil && (h.Sleep == nil || current.Sleep.Operation != h.Sleep.Operation)) { return managedStorageError() } if r.data.Version == 1 { diff --git a/sandboxd/pool/managed_volume.go b/sandboxd/pool/managed_volume.go index 6ec127d9..fd428489 100644 --- a/sandboxd/pool/managed_volume.go +++ b/sandboxd/pool/managed_volume.go @@ -93,6 +93,8 @@ func (m *Manager) CreateVolume(ctx context.Context, name string, req types.Volum return types.ManagedVolumeInfo{}, false, err } v = r.data.Volumes[name] + case volumeGrowing: + return r.info(v), false, nil case volumeReady: if err := r.verify(v); err != nil { return types.ManagedVolumeInfo{}, false, r.markRecovery(v) @@ -136,7 +138,7 @@ func (r *volumeRegistry) completeCreate(v managedVolume) error { } func (r *volumeRegistry) markRecovery(v managedVolume) error { - if r.sleepOwnsVolume(v.ID) { + if r.sleepOwnsVolume(v.ID) || v.Growth.fenced() { return managedStorageError() } v.State = volumeRecovery @@ -161,6 +163,12 @@ func (m *Manager) GetVolume(name, tenant string) (types.ManagedVolumeInfo, error if !ok || (tenant != "" && v.Tenant != tenant) { return types.ManagedVolumeInfo{}, managedNotFound() } + if v.State == volumeGrowing { + actual, err := inspectBacking(r.path(v.ID)) + if err != nil || r.verifyDirectories() != nil || actual.Device != v.Backing.Device || actual.Inode != v.Backing.Inode || (actual.Size != v.Size && actual.Size != v.Growth.Runtime.TargetBytes) { + return types.ManagedVolumeInfo{}, managedStorageError() + } + } if v.State == volumeReady && r.verify(v) != nil { if err := r.markRecovery(v); r.fault != nil { return types.ManagedVolumeInfo{}, err @@ -197,6 +205,9 @@ func (m *Manager) DeleteVolume(ctx context.Context, name, expected, tenant strin if v.ID != id { return volumeError(412, "volume_precondition_failed", "volume identity does not match") } + if v.Growth.fenced() { + return growBusy() + } if holders := r.holders(v.ID); len(holders) > 0 { return &VolumeError{Status: 409, Code: volumeBusyCode, Message: "volume is held by a sandbox", Volume: name, Holders: holders} } @@ -209,7 +220,7 @@ func (m *Manager) DeleteVolume(ctx context.Context, name, expected, tenant strin if missing && v.State != volumeCreating && v.State != volumeDeleting { return r.markRecovery(v) } - if !missing && (err != nil || v.Backing.Inode == 0 || actual != v.Backing) { + if !missing && (err != nil || v.Backing.Inode == 0 || !growthBackingMatches(v, actual)) { return volumeError(503, "volume_backing_mismatch", "volume backing requires operator recovery") } if v.State != volumeDeleting { @@ -253,7 +264,12 @@ func (r *volumeRegistry) holders(id volumeID) []types.VolumeHolder { } func (r *volumeRegistry) info(v managedVolume) types.ManagedVolumeInfo { - return types.ManagedVolumeInfo{Name: v.Name, DefaultMount: types.DefaultVolumeMount(v.Name), SizeBytes: v.Size, Available: v.State == volumeReady, Nodes: 1, Writable: true, Kind: "managed", ID: string(v.ID), Tenant: v.Tenant, Node: v.Node, State: string(v.State), CreatedAt: v.CreatedAt, Holders: r.holders(v.ID)} + info := types.ManagedVolumeInfo{Name: v.Name, DefaultMount: types.DefaultVolumeMount(v.Name), SizeBytes: v.Size, Available: v.State == volumeReady, Nodes: 1, Writable: true, Kind: "managed", ID: string(v.ID), Tenant: v.Tenant, Node: v.Node, State: string(v.State), CreatedAt: v.CreatedAt, Holders: r.holders(v.ID)} + info.CapacityGeneration = v.CapacityGeneration + if g := v.Growth; g != nil { + info.Growth = &types.VolumeGrowth{OperationID: g.Request.OperationID, ExpectedGeneration: g.Request.ExpectedGeneration, TargetBytes: g.Runtime.TargetBytes, State: g.State, RuntimeOutcome: g.Outcome, GuestSizeBytes: g.GuestSizeBytes, FilesystemStatus: "not_managed"} + } + return info } // ManagedVolumeStatus lets listing fail clearly rather than presenting a corrupt @@ -360,6 +376,9 @@ func (m *Manager) retireManaged(id string) error { if r.fault != nil { return managedStorageError() } + if r.growthOwnsHold(id) { + return growBusy() + } if h.State == holdRetiring { return nil } @@ -412,3 +431,10 @@ func (m *Manager) managedCatalogAlias(path string) bool { } return false } + +func growthBackingMatches(v managedVolume, actual backingIdentity) bool { + if actual == v.Backing { + return true + } + return v.Growth != nil && v.Growth.Outcome == "old_vmm_gone" && actual.Device == v.Backing.Device && actual.Inode == v.Backing.Inode && actual.Size == v.Growth.Runtime.TargetBytes +} diff --git a/sandboxd/pool/reconcile.go b/sandboxd/pool/reconcile.go index 424a99eb..f6398efe 100644 --- a/sandboxd/pool/reconcile.go +++ b/sandboxd/pool/reconcile.go @@ -24,6 +24,9 @@ func (m *Manager) Reconcile(ctx context.Context) error { if err != nil { return err } + if err := m.recoverGrowth(ctx, claims); err != nil { + return err + } fenced := m.fenceArchived(ctx, claims) vms, err := m.eng.List(ctx) if err != nil { @@ -155,6 +158,9 @@ func (m *Manager) sweepStaleVMs(ctx context.Context, live map[string]types.VMRec // removeStaleVM reclaims one unowned VM, reporting whether it is gone. func (m *Manager) removeStaleVM(ctx context.Context, name string, rec types.VMRecord) bool { + if m.growthOwnsVM(name) { + return false + } logger := log.WithFunc("pool.removeStaleVM") if rec.State == vmStateCreating { // a canceled ctx must not skip the busy check guarding the forced remove diff --git a/sandboxd/pool/remove.go b/sandboxd/pool/remove.go index 3951f015..a9d6bfea 100644 --- a/sandboxd/pool/remove.go +++ b/sandboxd/pool/remove.go @@ -14,13 +14,16 @@ import ( ) func (m *Manager) removeVM(ctx context.Context, name string) bool { + if m.growthOwnsVM(name) { + return false + } m.out.ClosePrebound(name) // cancellation-immune: on a canceled ctx `cocoon vm rm` no-ops and orphans. ctx = context.WithoutCancel(ctx) hold, managed := m.managedHoldByVM(name) if managed { if err := m.retireManaged(hold.SandboxID); err != nil { - if hold.Sleep != nil { + if hold.Sleep != nil || m.growthOwnsHold(hold.SandboxID) { return false } _ = m.eng.Remove(ctx, name) @@ -108,6 +111,10 @@ func (m *Manager) retryRemovals(ctx context.Context) *sync.WaitGroup { } func (m *Manager) retryRemoval(ctx context.Context, name string, pending pendingRemoval) { + if m.growthOwnsVM(name) { + m.queueRemoval(name, pending.sandboxID, pending.tap, pending.volumes, pending.notBefore) + return + } if m.retryManagedSleepRemoval(ctx, name, pending) { return } diff --git a/sandboxd/pool/volume_mutation.go b/sandboxd/pool/volume_mutation.go index ff309432..b47f0a3b 100644 --- a/sandboxd/pool/volume_mutation.go +++ b/sandboxd/pool/volume_mutation.go @@ -190,7 +190,7 @@ func (m *Manager) beginVolumeRun(ctx context.Context, sb *types.Sandbox) (contex m.mu.Unlock() return ctx, nil, managedStorageError() } - if h.State != holdClaimed { + if h.State != holdClaimed || m.growthOwnsHold(sb.ID) { m.mu.Unlock() return ctx, nil, ErrFailed } @@ -251,7 +251,7 @@ func (m *Manager) checkVolumeRunLocked(ctx context.Context, sb *types.Sandbox, r if !m.ownsRunLocked(sb, run) { return ErrUnknownSandbox } - if h, managed := m.managedHold(sb.ID); managed && h.State != holdClaimed { + if h, managed := m.managedHold(sb.ID); managed && (h.State != holdClaimed || m.growthOwnsHold(sb.ID)) { return ErrFailed } switch { diff --git a/sandboxd/pool/volume_registry.go b/sandboxd/pool/volume_registry.go index eebdd017..19ad552a 100644 --- a/sandboxd/pool/volume_registry.go +++ b/sandboxd/pool/volume_registry.go @@ -24,6 +24,7 @@ type ( const ( volumeCreating volumePhase = "creating" volumeReady volumePhase = "ready" + volumeGrowing volumePhase = "growing" volumeDeleting volumePhase = "deleting" volumeRecovery volumePhase = "recovery_required" holdAttaching holdPhase = "attaching" @@ -38,14 +39,16 @@ type backingIdentity struct { } type managedVolume struct { - ID volumeID `json:"id"` - Name string `json:"name"` - Tenant string `json:"tenant"` - Node string `json:"node"` - Size int64 `json:"size"` - Backing backingIdentity `json:"backing"` - State volumePhase `json:"state"` - CreatedAt time.Time `json:"created_at"` + CapacityGeneration uint64 `json:"capacity_generation,omitempty"` + Growth *managedGrowth `json:"growth,omitempty"` + ID volumeID `json:"id"` + Name string `json:"name"` + Tenant string `json:"tenant"` + Node string `json:"node"` + Size int64 `json:"size"` + Backing backingIdentity `json:"backing"` + State volumePhase `json:"state"` + CreatedAt time.Time `json:"created_at"` } type volumeRef struct { @@ -244,11 +247,17 @@ func (r *volumeRegistry) verify(v managedVolume) error { func (r *volumeRegistry) validate() error { d := r.data - if (d.Version != 1 && d.Version != 2) || d.Volumes == nil || d.Holds == nil || d.Deleted == nil { + if (d.Version != 1 && d.Version != 2 && d.Version != 3) || d.Volumes == nil || d.Holds == nil || d.Deleted == nil { return errors.New("invalid volume registry format") } ids := map[volumeID]bool{} for name, v := range d.Volumes { + if err := r.validateGrowth(v); err != nil { + return err + } + if d.Version < 3 && v.CapacityGeneration != 0 { + return errors.New("capacity generation requires version 3") + } if err := validateVolumeRecord(name, v, ids); err != nil { return err } @@ -289,7 +298,7 @@ func validateVolumeRecord(name string, v managedVolume, ids map[volumeID]bool) e return errors.New("invalid managed volume record") } switch v.State { - case volumeCreating, volumeReady, volumeDeleting, volumeRecovery: + case volumeCreating, volumeReady, volumeGrowing, volumeDeleting, volumeRecovery: default: return errors.New("invalid managed volume state") } @@ -318,7 +327,7 @@ func (r *volumeRegistry) validateHoldRecord(id string, h volumeHold, vms map[str return errors.New("claimed volume lacks VM") } if h.Sleep != nil { - if r.data.Version != 2 { + if r.data.Version < 2 { return errors.New("sleep requires registry version 2") } if err := validateManagedSleep(h); err != nil { @@ -359,7 +368,7 @@ func (r *volumeRegistry) validateHoldRef(h volumeHold, ref volumeRef, names map[ return false, nil } v, ok := r.data.Volumes[ref.Volume.Name] - if !ok || (v.State != volumeReady && v.State != volumeRecovery) || ref.ID != v.ID || ref.Path != r.path(v.ID) || ref.Backing != v.Backing || (h.Tenant != "" && h.Tenant != v.Tenant) { + if !ok || (v.State != volumeReady && v.State != volumeRecovery && v.State != volumeGrowing) || ref.ID != v.ID || ref.Path != r.path(v.ID) || ref.Backing != v.Backing || (h.Tenant != "" && h.Tenant != v.Tenant) { return false, errors.New("inconsistent managed volume reference") } return true, nil diff --git a/sandboxd/types/api.go b/sandboxd/types/api.go index 0b0e8b11..91478542 100644 --- a/sandboxd/types/api.go +++ b/sandboxd/types/api.go @@ -13,17 +13,18 @@ const ( // VolumeInfo is the caller-visible, host-path-free catalog projection. type VolumeInfo struct { - Kind string `json:"kind,omitempty"` - ID string `json:"id,omitempty"` - Tenant string `json:"tenant,omitempty"` - Node string `json:"node,omitempty"` - State string `json:"state,omitempty"` - CreatedAt time.Time `json:"created_at,omitzero"` - Name string `json:"name"` - DefaultMount string `json:"default_mount"` - SizeBytes int64 `json:"size_bytes"` - Available bool `json:"available"` - Nodes int `json:"nodes"` + CapacityGeneration uint64 `json:"capacity_generation"` + Kind string `json:"kind,omitempty"` + ID string `json:"id,omitempty"` + Tenant string `json:"tenant,omitempty"` + Node string `json:"node,omitempty"` + State string `json:"state,omitempty"` + CreatedAt time.Time `json:"created_at,omitzero"` + Name string `json:"name"` + DefaultMount string `json:"default_mount"` + SizeBytes int64 `json:"size_bytes"` + Available bool `json:"available"` + Nodes int `json:"nodes"` // Writable reports whether the operator allows rw claims of this name. Writable bool `json:"writable,omitzero"` } @@ -31,6 +32,7 @@ type VolumeInfo struct { type ManagedVolumeInfo struct { VolumeInfo Holders []VolumeHolder `json:"holders,omitempty"` + Growth *VolumeGrowth `json:"growth,omitempty"` } type VolumeHolder struct { @@ -73,3 +75,19 @@ type SandboxVolumesResponse struct { func Seconds(n int) time.Duration { return time.Duration(min(max(int64(n), 0), math.MaxInt64/int64(time.Second))) * time.Second } + +type VolumeGrow struct { + SizeGiB int64 `json:"size_gib"` + ExpectedGeneration uint64 `json:"expected_generation"` + OperationID string `json:"operation_id"` +} + +type VolumeGrowth struct { + OperationID string `json:"operation_id"` + ExpectedGeneration uint64 `json:"expected_generation"` + TargetBytes int64 `json:"target_bytes"` + State string `json:"state"` + RuntimeOutcome string `json:"runtime_outcome"` + GuestSizeBytes int64 `json:"guest_size_bytes"` + FilesystemStatus string `json:"filesystem_status"` +} diff --git a/sandboxd/types/types.go b/sandboxd/types/types.go index 8cbe0b78..43dae335 100644 --- a/sandboxd/types/types.go +++ b/sandboxd/types/types.go @@ -720,6 +720,7 @@ type VMConfig struct { // VMDisk is one hot-attached disk that `cocoon vm inspect` reports. type VMDisk struct { + ID string `json:"id"` Name string `json:"name"` Path string `json:"path"` ReadOnly bool `json:"readonly,omitzero"` From 12b811a959b9a5096bef7d09cd96dafec81af622 Mon Sep 17 00:00:00 2001 From: minatoaquaMK2 Date: Tue, 20 Oct 2026 06:28:21 +0000 Subject: [PATCH 08/75] Expose managed volume growth through HTTP and SDKs --- docs/managed-volumes.md | 91 +++++++++- docs/sandboxd-api.md | 39 ++++- sandboxd/server/managed_volumes.go | 75 +++++++++ sandboxd/server/server.go | 1 + sandboxd/server/volume_grow_test.go | 204 +++++++++++++++++++++++ sdk/go/managed_volume.go | 70 +++++++- sdk/go/options.go | 25 +-- sdk/go/volume_grow_test.go | 165 ++++++++++++++++++ sdk/python/cocoonsandbox/client.py | 48 ++++++ sdk/python/tests/test_managed_volumes.py | 107 ++++++++++++ 10 files changed, 802 insertions(+), 23 deletions(-) create mode 100644 sandboxd/server/volume_grow_test.go create mode 100644 sdk/go/volume_grow_test.go diff --git a/docs/managed-volumes.md b/docs/managed-volumes.md index 2de217d1..c3cf5525 100644 --- a/docs/managed-volumes.md +++ b/docs/managed-volumes.md @@ -8,8 +8,9 @@ single-writer exclusion. It builds on the existing This is a subset of [issue #300](https://github.com/cocoonstack/sandbox/issues/300). Explicit hibernate and wake support one mounted writable managed volume when -the node enables a workload hook. Online growth, placement by volume, and -archive export remain follow-up work. Other capture operations still refuse +the node enables a workload hook. Online block growth supports one running +mounted writable managed volume on a qualified Cloud Hypervisor runtime. +Placement by volume and archive export remain follow-up work. Other capture operations still refuse volume holders. This does not complete all of #300. ## Configure one node @@ -60,6 +61,7 @@ tenant owner when creating one. | `PUT /v1/volumes/work` with `{"size_gib":20}` | `201`, `volume`, identity `ETag` | Identical retry returns `200` and the same identity; changed owner or capacity returns `409`. | | `GET /v1/volumes` | Catalog and managed summaries visible to caller | Managed summaries include identity and owner; use individual GET for holders. | | `GET /v1/volumes/work` | Managed detail with public holder IDs, modes and phases | Foreign and unknown names share `404`; host paths and process IDs stay private. | +| `POST /v1/volumes/work/grow` with identity `If-Match` and a growth request | `200` completed or `202` pending, with `volume` | Repeat the exact request to reconcile; generation or identity mismatch returns `412`. | | `DELETE /v1/volumes/work` with `If-Match: "vol_…"` | `204` | Missing precondition `428`; wrong identity `412`; held resource `409`. | Capacity is an integer from 1 through 1024 GiB. The backing is a raw sparse file; @@ -68,10 +70,10 @@ occur while guests write. Creation does not format or mount a filesystem. Deletion tombstones preserve the outcome of an earlier deletion. Replaying that old identity returns `204` without deleting a newly created same-name resource. -The Go SDK exposes `CreateVolume`, `GetVolume` and `DeleteVolume`. Managed detail +The Go SDK exposes `CreateVolume`, `GetVolume`, `GrowVolume`, and `DeleteVolume`. Managed detail uses `ManagedVolumeInfo`; existing `VolumeInfo` remains comparable. Structured `ManagedVolumeError` unwraps to `APIError`. Python exposes `create_volume`, -`get_volume` and `delete_volume`; structured fields remain on `APIError`. +`get_volume`, `grow_volume`, and `delete_volume`; structured fields remain on `APIError`. ## Format once, reuse across templates @@ -114,6 +116,77 @@ through the existing `GET/PUT/PATCH /v1/sandboxes/{id}/volumes` interface. Attach-only managed entries retain their empty mount across those changes and across daemon restart. +## Grow a running volume + +Growth requires one running claim with exactly one managed writable volume +mounted and fully applied. Read-only disks, attach-only disks, sleeping claims, +additional disks, egress, and unsupported runtimes are refused before admission. +Configure every runtime lifecycle caller to use the same qualified Cocoon binary. +Online block growth is qualified for Cloud Hypervisor, not Firecracker. + +Read the volume's immutable `id` and `capacity_generation`. Generate one UUID +for the operation and retain it with the absolute target and original generation. + +```python +import uuid + +volume = client.get_volume("work") +operation_id = str(uuid.uuid4()) +request = dict( + expected_generation=volume["capacity_generation"], + operation_id=operation_id, +) +result = client.grow_volume("work", volume["id"], 40, **request) +``` + +The equivalent HTTP request is: + +```http +POST /v1/volumes/work/grow +Authorization: Bearer tenant-management-token +If-Match: "vol_resource_identity" +Content-Type: application/json + +{"size_gib":40,"expected_generation":0,"operation_id":"efbf4748-1c72-43ee-8304-b4e1a91322b5"} +``` + +`size_gib` is an absolute target from 1 through 1024 GiB. A new operation must +increase committed capacity. `operation_id` is a lowercase canonical nonzero +UUID. All three JSON fields are required. The identity ETag remains unchanged +after growth. `capacity_generation` increases only after completion. + +`200` means the runtime acknowledged the operation and the same running guest +observed the target block capacity with the same disk and mount identity. +The response commits `size_bytes` and includes `growth.state: "completed"`. +It does not extend a partition or filesystem. `growth.filesystem_status` is +`not_managed`. Extend the guest filesystem separately after checking its layout. + +`202` means the operation remains pending. `size_bytes` and +`capacity_generation` retain their committed values. Read `GET /v1/volumes/work` +for the latest `growth`, then repeat the original POST to reconcile. GET is +read-only. Repeat the same operation ID, original generation, target, and resource +ID after a lost response or client timeout. The SDKs do not retry automatically +or generate replacement operation IDs. Only the latest operation is retained. + +A changed request with the same operation ID returns `409`. A stale generation +or wrong resource ID returns `412`, except an exact latest-operation replay +still returns that operation's result. Missing `If-Match` returns `428`. +Invalid JSON or arguments return `400`. Unsupported runtime capability returns +`503`. Foreign and unknown volumes both return `404`. + +Release, sleep, wake, deletion, and live volume changes cannot discard an +unresolved growth operation. An unknown runtime outcome remains fenced, even +when the backing file or guest reports the target capacity. The daemon never +silently restarts the VM to resolve it. A `409` with +`volume_grow_recovery_required` requires operator recovery. Inspect the volume, +preserve its runtime record, and stop the exact recorded VM generation using +the qualified runtime before repeating the same POST. A confirmed stopped +generation permits ownership cleanup but does not count as online growth +success. Offline data repair is outside this API. + +The first growth durably upgrades the managed registry to version 3. Older +binaries refuse that registry. Keep the upgraded binary when restarting it. + ## Hibernate one mounted managed volume The node can enable explicit Hibernate and Wake for one managed writable disk @@ -181,7 +254,8 @@ disk. The selected hook and operation ID remain in the durable sleep record. Retries use them even after a configuration change or disabling new managed sleeps. -The first sleep durably upgrades the managed registry to version 2. Older +The first sleep durably upgrades a version 1 managed registry to version 2. +Sleep also supports registries upgraded to version 3 by growth. Older binaries refuse that registry. Keep the upgraded binary when restarting it. Claim barriers preserve this lifecycle across daemon crashes with either value of `sync_claims`. They do not add a host-power-loss recovery guarantee. @@ -215,9 +289,8 @@ remain in force. ## Follow-up order -1. Add online growth with explicit block-device and guest-filesystem outcomes. -2. Add owner-node placement and a clear unavailable-owner response. -3. Add archive/export and restore of external data disks. -4. Add Kubernetes lifecycle/status ownership once the daemon contracts are stable. +1. Add owner-node placement and a clear unavailable-owner response. +2. Add archive/export and restore of external data disks. +3. Add Kubernetes lifecycle/status ownership once the daemon contracts are stable. These are proposed next steps. None is implemented or qualified by this phase. diff --git a/docs/sandboxd-api.md b/docs/sandboxd-api.md index ffbbcfe0..4bd71e4a 100644 --- a/docs/sandboxd-api.md +++ b/docs/sandboxd-api.md @@ -258,9 +258,46 @@ entry's response is byte-identical to v1. When `managed_volume_dir` is enabled, this list also includes caller-visible managed summaries and requires an explicit root or tenant token. Individual managed GET responses carry holder details. The node-local create, inspect -and conditional-delete routes are documented in +conditional-delete, and online-growth routes are documented in [Tenant managed persistent volumes](managed-volumes.md#manage-a-resource). +## POST /v1/volumes/{name}/grow + +Auth requires an explicit root or tenant management token. The node owning the +managed volume handles the request. `If-Match` contains its quoted immutable +resource ID. The strict JSON body requires `size_gib`, `expected_generation`, +and `operation_id`. Capacity generation is independent of the identity ETag. + +```json +{"size_gib":40,"expected_generation":0,"operation_id":"efbf4748-1c72-43ee-8304-b4e1a91322b5"} +``` + +The target is absolute GiB, from 1 through 1024, and must increase committed +capacity for a fresh operation. The operation ID is a canonical lowercase +nonzero UUID. `200` returns the `volume` envelope with `growth.state` completed. +`202` returns the same envelope with pending progress. Only completion commits +`size_bytes` and increments `capacity_generation`. The ETag remains the resource ID. + +`growth` reports `operation_id`, `expected_generation`, `target_bytes`, `state`, +`runtime_outcome`, `guest_size_bytes`, and `filesystem_status`. State is pending, +completed, or recovery_required. Runtime outcome is unobserved, outcome_unknown, +acknowledged, or old_vmm_gone. Guest bytes remain zero until observed. +Filesystem status is not_managed. Completion proves guest block capacity and +does not extend the filesystem. + +GET detail exposes the latest operation without driving it. After a pending +result or a lost response, repeat the original POST, including its original +generation. An exact latest-operation replay is checked before the current +generation. No permanent operation history is provided. + +Errors are `400` invalid arguments, `401` missing or invalid management token, +`404` foreign or unknown volume, `409` conflicting request or lifecycle, +`412` wrong identity or stale generation, `428` missing `If-Match`, and `503` +unsupported runtime or unavailable storage. A terminal recovery result returns +`409` with code `volume_grow_recovery_required`; GET retains its progress. +See [Grow a running volume](managed-volumes.md#grow-a-running-volume) for the +supported workload and recovery limits. + ## GET /v1/sandboxes/{id}/volumes Returns the persisted desired set (`volumes_want`), confirmed applied set diff --git a/sandboxd/server/managed_volumes.go b/sandboxd/server/managed_volumes.go index f45b0d9e..95e1839a 100644 --- a/sandboxd/server/managed_volumes.go +++ b/sandboxd/server/managed_volumes.go @@ -5,6 +5,7 @@ import ( "errors" "io" "net/http" + "regexp" "strings" "github.com/cocoonstack/sandbox/sandboxd/pool" @@ -20,6 +21,12 @@ type managedVolumeManager interface { ManagedVolumesEnabled() bool } +type managedVolumeGrower interface { + GrowVolume(context.Context, string, string, types.VolumeGrow, string) (types.ManagedVolumeInfo, error) +} + +var volumeOperationID = regexp.MustCompile(`^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$`) + func volumeHTTPError(w http.ResponseWriter, status int, code, message string) { writeJSON(w, status, &pool.VolumeError{Status: status, Code: code, Message: message}) } @@ -127,3 +134,71 @@ func (s *Server) handleDeleteVolume(w http.ResponseWriter, r *http.Request) { } w.WriteHeader(http.StatusNoContent) } + +func (s *Server) handleGrowVolume(w http.ResponseWriter, r *http.Request) { + mgr, ok := s.volumeManager(w) + if !ok { + return + } + expected := r.Header.Get("If-Match") + if expected == "" { + volumeHTTPError(w, 428, "volume_precondition_required", "If-Match must identify the volume") + return + } + if len(expected) < 3 || expected[0] != '"' || expected[len(expected)-1] != '"' || strings.ContainsAny(expected[1:len(expected)-1], "\",\\") { + volumeHTTPError(w, 412, "volume_precondition_failed", "If-Match must contain one quoted volume identity") + return + } + var body struct { + SizeGiB *int64 `json:"size_gib"` + ExpectedGeneration *uint64 `json:"expected_generation"` + OperationID *string `json:"operation_id"` + } + raw, err := io.ReadAll(http.MaxBytesReader(w, r.Body, maxBodyBytes)) + if err != nil || utils.DecodeStrictJSON(raw, &body) != nil || body.SizeGiB == nil || body.ExpectedGeneration == nil || body.OperationID == nil { + volumeHTTPError(w, 400, "invalid_volume_grow", "size_gib, expected_generation and operation_id are required") + return + } + if *body.SizeGiB < 1 || *body.SizeGiB > 1024 || !volumeOperationID.MatchString(*body.OperationID) || *body.OperationID == "00000000-0000-0000-0000-000000000000" { + volumeHTTPError(w, 400, "invalid_volume_grow", "size_gib must be between 1 and 1024 and operation_id must be a canonical nonzero UUID") + return + } + grower, ok := mgr.(managedVolumeGrower) + if !ok { + if _, err := mgr.GetVolume(r.PathValue("name"), tenantFrom(r.Context())); err != nil { + if !writePoolErr(w, err) { + volumeHTTPError(w, 503, "volume_storage_unavailable", "volume storage unavailable; retry") + } + return + } + volumeHTTPError(w, 503, "volume_grow_unsupported", "volume growth is not supported by this runtime") + return + } + v, err := grower.GrowVolume(r.Context(), r.PathValue("name"), expected[1:len(expected)-1], types.VolumeGrow{ + SizeGiB: *body.SizeGiB, ExpectedGeneration: *body.ExpectedGeneration, OperationID: *body.OperationID, + }, tenantFrom(r.Context())) + if err != nil { + if !writePoolErr(w, err) { + volumeHTTPError(w, 503, "volume_storage_unavailable", "volume storage unavailable; retry") + } + return + } + if v.Growth == nil { + volumeHTTPError(w, 503, "volume_storage_unavailable", "volume growth result unavailable; query the volume before retrying") + return + } + status := http.StatusAccepted + switch v.Growth.State { + case "completed": + status = http.StatusOK + case "pending": + case "recovery_required": + volumeHTTPError(w, 409, "volume_grow_recovery_required", "volume growth requires operator recovery; inspect the volume") + return + default: + volumeHTTPError(w, 503, "volume_storage_unavailable", "volume growth result unavailable; query the volume before retrying") + return + } + w.Header().Set("ETag", `"`+v.ID+`"`) + writeJSON(w, status, VolumeResponse{Volume: v}) +} diff --git a/sandboxd/server/server.go b/sandboxd/server/server.go index 9a4f0669..ea099d0b 100644 --- a/sandboxd/server/server.go +++ b/sandboxd/server/server.go @@ -271,6 +271,7 @@ func (s *Server) Handler() http.Handler { mux.HandleFunc("GET /v1/volumes", s.requireVolumeListToken(s.handleVolumes)) mux.HandleFunc("PUT /v1/volumes/{name}", s.requireVolumeManager(s.handleCreateVolume)) mux.HandleFunc("GET /v1/volumes/{name}", s.requireVolumeManager(s.handleGetVolume)) + mux.HandleFunc("POST /v1/volumes/{name}/grow", s.requireVolumeManager(s.handleGrowVolume)) mux.HandleFunc("DELETE /v1/volumes/{name}", s.requireVolumeManager(s.handleDeleteVolume)) mux.HandleFunc("POST /v1/sandboxes/{id}/release", s.handleSandboxVerb("release", s.mgr.Release)) mux.HandleFunc("POST /v1/sandboxes/{id}/hibernate", s.handleSandboxVerb("hibernate", s.mgr.Hibernate)) diff --git a/sandboxd/server/volume_grow_test.go b/sandboxd/server/volume_grow_test.go new file mode 100644 index 00000000..05df878e --- /dev/null +++ b/sandboxd/server/volume_grow_test.go @@ -0,0 +1,204 @@ +package server + +import ( + "context" + "encoding/json/v2" + "errors" + "io" + "net/http" + "strings" + "testing" + + "github.com/cocoonstack/sandbox/sandboxd/pool" + "github.com/cocoonstack/sandbox/sandboxd/tenants" + "github.com/cocoonstack/sandbox/sandboxd/types" +) + +const ( + growOperation = "efbf4748-1c72-43ee-8304-b4e1a91322b5" + growBody = `{"size_gib":40,"expected_generation":0,"operation_id":"` + growOperation + `"}` +) + +type volumeManagerFake struct { + *fakeManager + get func(string, string) (types.ManagedVolumeInfo, error) +} + +func (f *volumeManagerFake) CreateVolume(context.Context, string, types.VolumeCreate, string) (types.ManagedVolumeInfo, bool, error) { + return types.ManagedVolumeInfo{}, false, nil +} + +func (f *volumeManagerFake) GetVolume(name, tenant string) (types.ManagedVolumeInfo, error) { + return f.get(name, tenant) +} + +func (f *volumeManagerFake) DeleteVolume(context.Context, string, string, string) error { return nil } + +func (f *volumeManagerFake) ManagedVolumeStatus() error { return nil } + +func (f *volumeManagerFake) ManagedVolumesEnabled() bool { return true } + +type volumeGrowFake struct { + *volumeManagerFake + grow func(context.Context, string, string, types.VolumeGrow, string) (types.ManagedVolumeInfo, error) +} + +func (f *volumeGrowFake) GrowVolume(ctx context.Context, name, id string, req types.VolumeGrow, tenant string) (types.ManagedVolumeInfo, error) { + return f.grow(ctx, name, id, req, tenant) +} + +func TestGrowVolumeHTTP(t *testing.T) { + for _, state := range []string{"completed", "pending", "recovery_required"} { + t.Run(state, func(t *testing.T) { + calls := 0 + mgr := &volumeGrowFake{volumeManagerFake: &volumeManagerFake{fakeManager: &fakeManager{ + tenants: []tenants.TenantSpec{{Name: "tenant-1", Token: "tenant-token"}}, + }}, grow: func(ctx context.Context, name, id string, req types.VolumeGrow, tenant string) (types.ManagedVolumeInfo, error) { + calls++ + if ctx.Err() != nil || name != "data" || id != "vol_123" || tenant != "tenant-1" || req != (types.VolumeGrow{SizeGiB: 40, ExpectedGeneration: 0, OperationID: growOperation}) { + t.Errorf("grow args = %q %q %+v %q, context %v", name, id, req, tenant, ctx.Err()) + } + return types.ManagedVolumeInfo{ID: "vol_123", Growth: &types.VolumeGrowth{OperationID: growOperation, State: state}}, nil + }} + ts := newTestServer(t, "root-token", mgr, nil) + req, _ := http.NewRequest(http.MethodPost, ts.URL+"/v1/volumes/data/grow", strings.NewReader(growBody)) + req.Header.Set("Authorization", "Bearer tenant-token") + req.Header.Set("If-Match", `"vol_123"`) + resp, err := ts.Client().Do(req) + if err != nil { + t.Fatal(err) + } + defer resp.Body.Close() + want := map[string]int{"completed": 200, "pending": 202, "recovery_required": 409}[state] + if resp.StatusCode != want || calls != 1 { + t.Fatalf("status %d calls %d", resp.StatusCode, calls) + } + if state == "recovery_required" { + var failure pool.VolumeError + if err := json.UnmarshalRead(resp.Body, &failure); err != nil || failure.Code != "volume_grow_recovery_required" { + t.Fatalf("failure = %+v, %v", failure, err) + } + } else if resp.Header.Get("ETag") != `"vol_123"` { + t.Fatalf("ETag %q", resp.Header.Get("ETag")) + } + }) + } +} + +func TestGrowVolumeHTTPRefusesBeforeAdmission(t *testing.T) { + for _, tc := range []struct { + name, token, match, body string + status int + }{ + {"no token", "", `"vol_123"`, growBody, 401}, + {"sandbox token", "sandbox-token", `"vol_123"`, growBody, 401}, + {"missing identity", "root-token", "", growBody, 428}, + {"wildcard", "root-token", "*", growBody, 412}, + {"weak identity", "root-token", `W/"vol_123"`, growBody, 412}, + {"identity list", "root-token", `"vol_123", "vol_456"`, growBody, 412}, + {"missing generation", "root-token", `"vol_123"`, `{"size_gib":40,"operation_id":"` + growOperation + `"}`, 400}, + {"null generation", "root-token", `"vol_123"`, `{"size_gib":40,"expected_generation":null,"operation_id":"` + growOperation + `"}`, 400}, + {"negative generation", "root-token", `"vol_123"`, strings.Replace(growBody, `"expected_generation":0`, `"expected_generation":-1`, 1), 400}, + {"unknown member", "root-token", `"vol_123"`, strings.Replace(growBody, `"size_gib":40`, `"extra":true,"size_gib":40`, 1), 400}, + {"duplicate member", "root-token", `"vol_123"`, strings.Replace(growBody, `"size_gib":40`, `"size_gib":20,"size_gib":40`, 1), 400}, + {"trailing body", "root-token", `"vol_123"`, growBody + `{}`, 400}, + {"zero target", "root-token", `"vol_123"`, strings.Replace(growBody, `40`, `0`, 1), 400}, + {"fractional target", "root-token", `"vol_123"`, strings.Replace(growBody, `40`, `40.5`, 1), 400}, + {"oversize target", "root-token", `"vol_123"`, strings.Replace(growBody, `40`, `1025`, 1), 400}, + {"uppercase UUID", "root-token", `"vol_123"`, strings.Replace(growBody, growOperation, strings.ToUpper(growOperation), 1), 400}, + {"zero UUID", "root-token", `"vol_123"`, strings.Replace(growBody, growOperation, "00000000-0000-0000-0000-000000000000", 1), 400}, + } { + t.Run(tc.name, func(t *testing.T) { + mgr := &volumeGrowFake{volumeManagerFake: &volumeManagerFake{fakeManager: &fakeManager{}}, grow: func(context.Context, string, string, types.VolumeGrow, string) (types.ManagedVolumeInfo, error) { + t.Error("invalid request reached growth admission") + return types.ManagedVolumeInfo{}, nil + }} + ts := newTestServer(t, "root-token", mgr, nil) + req, _ := http.NewRequest(http.MethodPost, ts.URL+"/v1/volumes/data/grow", strings.NewReader(tc.body)) + if tc.token != "" { + req.Header.Set("Authorization", "Bearer "+tc.token) + } + req.Header.Set("If-Match", tc.match) + resp, err := ts.Client().Do(req) + if err != nil { + t.Fatal(err) + } + defer resp.Body.Close() + if resp.StatusCode != tc.status { + raw, _ := io.ReadAll(resp.Body) + t.Fatalf("status %d want %d, %s", resp.StatusCode, tc.status, raw) + } + }) + } +} + +func TestGrowVolumeHTTPErrorPrivacyAndPreconditions(t *testing.T) { + for _, tc := range []struct { + status int + code string + }{ + {404, "volume_not_found"}, {412, "volume_precondition_failed"}, {409, "volume_grow_conflict"}, {409, "volume_busy"}, {503, "volume_grow_unsupported"}, + } { + t.Run(tc.code, func(t *testing.T) { + mgr := &volumeGrowFake{volumeManagerFake: &volumeManagerFake{fakeManager: &fakeManager{}}, grow: func(context.Context, string, string, types.VolumeGrow, string) (types.ManagedVolumeInfo, error) { + return types.ManagedVolumeInfo{}, &pool.VolumeError{Status: tc.status, Code: tc.code, Message: "volume unavailable"} + }} + ts := newTestServer(t, "root-token", mgr, nil) + req, _ := http.NewRequest(http.MethodPost, ts.URL+"/v1/volumes/data/grow", strings.NewReader(growBody)) + req.Header.Set("Authorization", "Bearer root-token") + req.Header.Set("If-Match", `"vol_123"`) + resp, err := ts.Client().Do(req) + if err != nil { + t.Fatal(err) + } + defer resp.Body.Close() + var failure pool.VolumeError + if err := json.UnmarshalRead(resp.Body, &failure); err != nil || resp.StatusCode != tc.status || failure.Code != tc.code { + t.Fatalf("status %d failure %+v err %v", resp.StatusCode, failure, err) + } + }) + } + for _, foreign := range []bool{false, true} { + mgr := &volumeManagerFake{fakeManager: &fakeManager{}, get: func(string, string) (types.ManagedVolumeInfo, error) { + if foreign { + return types.ManagedVolumeInfo{}, &pool.VolumeError{Status: 404, Code: "volume_not_found", Message: "unknown volume"} + } + return types.ManagedVolumeInfo{}, nil + }} + ts := newTestServer(t, "root-token", mgr, nil) + req, _ := http.NewRequest(http.MethodPost, ts.URL+"/v1/volumes/data/grow", strings.NewReader(growBody)) + req.Header.Set("Authorization", "Bearer root-token") + req.Header.Set("If-Match", `"vol_123"`) + resp, err := ts.Client().Do(req) + if err != nil { + t.Fatal(err) + } + _ = resp.Body.Close() + want := 503 + if foreign { + want = 404 + } + if resp.StatusCode != want { + t.Fatalf("optional capability status %d want %d", resp.StatusCode, want) + } + } +} + +func TestGrowVolumeHTTPDoesNotExposeInternalErrors(t *testing.T) { + mgr := &volumeGrowFake{volumeManagerFake: &volumeManagerFake{fakeManager: &fakeManager{}}, grow: func(context.Context, string, string, types.VolumeGrow, string) (types.ManagedVolumeInfo, error) { + return types.ManagedVolumeInfo{}, errors.New("runtime /secret/backing pid 123 failed") + }} + ts := newTestServer(t, "root-token", mgr, nil) + req, _ := http.NewRequest(http.MethodPost, ts.URL+"/v1/volumes/data/grow", strings.NewReader(growBody)) + req.Header.Set("Authorization", "Bearer root-token") + req.Header.Set("If-Match", `"vol_123"`) + resp, err := ts.Client().Do(req) + if err != nil { + t.Fatal(err) + } + defer resp.Body.Close() + raw, _ := io.ReadAll(resp.Body) + if resp.StatusCode != 503 || strings.Contains(string(raw), "/secret") || strings.Contains(string(raw), "pid 123") { + t.Fatalf("unsafe response %d %s", resp.StatusCode, raw) + } +} diff --git a/sdk/go/managed_volume.go b/sdk/go/managed_volume.go index 376667f4..f897ebd4 100644 --- a/sdk/go/managed_volume.go +++ b/sdk/go/managed_volume.go @@ -15,6 +15,7 @@ import ( type ManagedVolumeInfo struct { VolumeInfo Holders []VolumeHolder `json:"holders,omitempty"` + Growth *VolumeGrowth `json:"growth,omitempty"` } type ManagedVolumeError struct { @@ -26,7 +27,28 @@ type ManagedVolumeError struct { func (e *ManagedVolumeError) Unwrap() error { return e.APIError } -var managedVolumeName = regexp.MustCompile(`^[a-z][a-z0-9_-]{0,19}$`) +var ( + managedVolumeName = regexp.MustCompile(`^[a-z][a-z0-9_-]{0,19}$`) + volumeOperationID = regexp.MustCompile(`^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$`) +) + +// VolumeGrow requests an absolute block capacity. Reuse the complete request when retrying. +type VolumeGrow struct { + SizeGiB int64 `json:"size_gib"` + ExpectedGeneration uint64 `json:"expected_generation"` + OperationID string `json:"operation_id"` +} + +// VolumeGrowth reports the latest operation. Filesystem capacity is not managed. +type VolumeGrowth struct { + OperationID string `json:"operation_id"` + ExpectedGeneration uint64 `json:"expected_generation"` + TargetBytes int64 `json:"target_bytes"` + State string `json:"state"` + RuntimeOutcome string `json:"runtime_outcome"` + GuestSizeBytes int64 `json:"guest_size_bytes"` + FilesystemStatus string `json:"filesystem_status"` +} // VolumeCreate requests a raw managed disk owned by Tenant. A tenant token may leave Tenant empty. type VolumeCreate struct { @@ -77,6 +99,52 @@ func (c *Client) GetVolume(ctx context.Context, name string) (*ManagedVolumeInfo return decodeVolumeResponse("get volume", resp.Body) } +// GrowVolume grows one running managed disk. A pending result requires the same request to reconcile. +func (c *Client) GrowVolume(ctx context.Context, name, id string, grow VolumeGrow) (*ManagedVolumeInfo, error) { + if err := validManagedVolumeName(name); err != nil { + return nil, err + } + if id == "" || strings.ContainsAny(id, "\"\r\n") { + return nil, fmt.Errorf("volume id must be a nonempty unquoted resource id") + } + if grow.SizeGiB < 1 || grow.SizeGiB > 1024 { + return nil, fmt.Errorf("volume size_gib must be between 1 and 1024") + } + if !volumeOperationID.MatchString(grow.OperationID) || grow.OperationID == "00000000-0000-0000-0000-000000000000" { + return nil, fmt.Errorf("volume operation_id must be a canonical nonzero UUID") + } + body, err := encodeBody("grow volume", grow) + if err != nil { + return nil, err + } + req, err := http.NewRequestWithContext(ctx, http.MethodPost, c.requestURL(c.addr, volumePath(name)+"/grow"), bytes.NewReader(body)) + if err != nil { + return nil, err + } + req.Header.Set("Content-Type", "application/json") + req.Header.Set("If-Match", `"`+id+`"`) + if c.apiToken != "" { + req.Header.Set("Authorization", "Bearer "+c.apiToken) + } + resp, err := c.hc.Do(req) + if err != nil { + return nil, err + } + defer func() { _ = resp.Body.Close() }() + if resp.StatusCode != http.StatusOK && resp.StatusCode != http.StatusAccepted { + return nil, apiError("grow volume", resp) + } + volume, err := decodeVolumeResponse("grow volume", resp.Body) + if err != nil { + return nil, err + } + if volume.Growth == nil || volume.Growth.OperationID != grow.OperationID || volume.Growth.ExpectedGeneration != grow.ExpectedGeneration || volume.Growth.TargetBytes != grow.SizeGiB<<30 || + (resp.StatusCode == http.StatusOK && volume.Growth.State != "completed") || (resp.StatusCode == http.StatusAccepted && volume.Growth.State != "pending") { + return nil, fmt.Errorf("decode grow volume response: missing or inconsistent growth result") + } + return volume, nil +} + // DeleteVolume deletes exactly id, including when an earlier response was lost. func (c *Client) DeleteVolume(ctx context.Context, name, id string) error { if err := validManagedVolumeName(name); err != nil { diff --git a/sdk/go/options.go b/sdk/go/options.go index 0d04873a..b40a2986 100644 --- a/sdk/go/options.go +++ b/sdk/go/options.go @@ -39,18 +39,19 @@ type Volume struct { // VolumeInfo describes one visible fleet catalog entry. type VolumeInfo struct { - Kind string `json:"kind,omitempty"` - ID string `json:"id,omitempty"` - Tenant string `json:"tenant,omitempty"` - Node string `json:"node,omitempty"` - State string `json:"state,omitempty"` - CreatedAt time.Time `json:"created_at,omitzero"` - Name string `json:"name"` - DefaultMount string `json:"default_mount"` - SizeBytes int64 `json:"size_bytes"` - Available bool `json:"available"` - Nodes int `json:"nodes"` - Writable bool `json:"writable,omitzero"` + Kind string `json:"kind,omitempty"` + ID string `json:"id,omitempty"` + Tenant string `json:"tenant,omitempty"` + Node string `json:"node,omitempty"` + State string `json:"state,omitempty"` + CreatedAt time.Time `json:"created_at,omitzero"` + Name string `json:"name"` + DefaultMount string `json:"default_mount"` + SizeBytes int64 `json:"size_bytes"` + CapacityGeneration uint64 `json:"capacity_generation"` + Available bool `json:"available"` + Nodes int `json:"nodes"` + Writable bool `json:"writable,omitzero"` } type VolumeHolder struct { diff --git a/sdk/go/volume_grow_test.go b/sdk/go/volume_grow_test.go new file mode 100644 index 00000000..20e893d6 --- /dev/null +++ b/sdk/go/volume_grow_test.go @@ -0,0 +1,165 @@ +package sandbox + +import ( + "context" + "encoding/json/v2" + "errors" + "net/http" + "net/http/httptest" + "reflect" + "strings" + "sync/atomic" + "testing" + "time" +) + +const growOperation = "efbf4748-1c72-43ee-8304-b4e1a91322b5" + +func growVolumeFixture(state string) ManagedVolumeInfo { + v := ManagedVolumeInfo{Name: "data", Kind: "managed", ID: "vol_123", Tenant: "t1", Node: "node-1", State: "growing", CreatedAt: time.Date(2026, 10, 19, 0, 0, 0, 0, time.UTC), SizeBytes: 20 << 30} + v.Growth = &VolumeGrowth{OperationID: growOperation, TargetBytes: 40 << 30, State: state, RuntimeOutcome: "acknowledged", FilesystemStatus: "not_managed"} + if state == "completed" { + v.State, v.SizeBytes, v.CapacityGeneration, v.Growth.GuestSizeBytes = "ready", 40<<30, 1, 40<<30 + } + return v +} + +func TestGrowVolumeRequestAndReplay(t *testing.T) { + for _, status := range []int{http.StatusOK, http.StatusAccepted} { + t.Run(http.StatusText(status), func(t *testing.T) { + state := "completed" + if status == http.StatusAccepted { + state = "pending" + } + want := growVolumeFixture(state) + requests := 0 + grow := VolumeGrow{SizeGiB: 40, ExpectedGeneration: 0, OperationID: growOperation} + ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + requests++ + if r.URL.Path != "/v1/volumes/data/grow" || r.Method != http.MethodPost || r.Header.Get("If-Match") != `"vol_123"` || r.Header.Get("Authorization") != "Bearer token" || r.Header.Get("Content-Type") != "application/json" { + t.Errorf("request = %s %s %+v", r.Method, r.URL.Path, r.Header) + } + var got VolumeGrow + if err := json.UnmarshalRead(r.Body, &got); err != nil || got != grow { + t.Errorf("body = %+v, %v", got, err) + } + w.WriteHeader(status) + _ = json.MarshalWrite(w, volumeResponse{Volume: &want}) + })) + t.Cleanup(ts.Close) + c := testClient(t, ts, WithAPIToken("token")) + for range 2 { + got, err := c.GrowVolume(t.Context(), "data", "vol_123", grow) + if err != nil || !reflect.DeepEqual(got, &want) { + t.Fatalf("grow = %+v, %v", got, err) + } + } + if requests != 2 { + t.Fatalf("requests = %d", requests) + } + }) + } +} + +func TestGrowVolumeValidatesBeforeSending(t *testing.T) { + requests := 0 + ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { requests++; w.WriteHeader(503) })) + t.Cleanup(ts.Close) + c := testClient(t, ts) + valid := VolumeGrow{SizeGiB: 40, OperationID: growOperation} + for _, tc := range []struct { + name, id string + grow VolumeGrow + }{ + {"bad/name", "vol_123", valid}, + {"data", "", valid}, + {"data", `"vol_123"`, valid}, + {"data", "bad\r\nid", valid}, + {"data", "vol_123", VolumeGrow{SizeGiB: 0, OperationID: growOperation}}, + {"data", "vol_123", VolumeGrow{SizeGiB: 1025, OperationID: growOperation}}, + {"data", "vol_123", VolumeGrow{SizeGiB: 40}}, + {"data", "vol_123", VolumeGrow{SizeGiB: 40, OperationID: strings.ToUpper(growOperation)}}, + {"data", "vol_123", VolumeGrow{SizeGiB: 40, OperationID: "00000000-0000-0000-0000-000000000000"}}, + } { + if _, err := c.GrowVolume(t.Context(), tc.name, tc.id, tc.grow); err == nil { + t.Errorf("accepted %+v", tc) + } + } + ctx, cancel := context.WithCancel(t.Context()) + cancel() + if _, err := c.GrowVolume(ctx, "data", "vol_123", valid); !errors.Is(err, context.Canceled) { + t.Fatalf("cancellation = %v", err) + } + if requests != 0 { + t.Fatalf("invalid requests sent = %d", requests) + } +} + +func TestGrowVolumeErrorsAndInvalidSuccess(t *testing.T) { + for _, status := range []int{400, 401, 404, 409, 412, 428, 503} { + t.Run(http.StatusText(status), func(t *testing.T) { + requests := 0 + ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + requests++ + w.WriteHeader(status) + _ = json.MarshalWrite(w, map[string]string{"error": "growth unavailable", "code": "volume_grow_recovery_required"}) + })) + t.Cleanup(ts.Close) + _, err := testClient(t, ts).GrowVolume(t.Context(), "data", "vol_123", VolumeGrow{SizeGiB: 40, OperationID: growOperation}) + var failure *ManagedVolumeError + if !errors.As(err, &failure) || failure.Status != status || failure.Code != "volume_grow_recovery_required" || requests != 1 { + t.Fatalf("error = %#v, requests %d", err, requests) + } + }) + } + for _, state := range []string{"", "pending", "recovery_required"} { + want := growVolumeFixture(state) + ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { _ = json.MarshalWrite(w, volumeResponse{Volume: &want}) })) + t.Cleanup(ts.Close) + if _, err := testClient(t, ts).GrowVolume(t.Context(), "data", "vol_123", VolumeGrow{SizeGiB: 40, OperationID: growOperation}); err == nil { + t.Errorf("accepted 200 with state %q", state) + } + } +} + +func TestGetVolumeRetainsGrowthProgress(t *testing.T) { + want := growVolumeFixture("recovery_required") + ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { _ = json.MarshalWrite(w, volumeResponse{Volume: &want}) })) + t.Cleanup(ts.Close) + got, err := testClient(t, ts).GetVolume(t.Context(), "data") + if err != nil || !reflect.DeepEqual(got, &want) { + t.Fatalf("get = %+v, %v", got, err) + } +} + +func TestGrowVolumeLostResponseReplay(t *testing.T) { + var requests atomic.Int32 + want := growVolumeFixture("completed") + grow := VolumeGrow{SizeGiB: 40, OperationID: growOperation} + ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + request := requests.Add(1) + var got VolumeGrow + if err := json.UnmarshalRead(r.Body, &got); err != nil || got != grow || r.Header.Get("If-Match") != `"vol_123"` { + t.Errorf("replayed request = %+v, %v", got, err) + } + if request == 1 { + conn, _, err := w.(http.Hijacker).Hijack() + if err != nil { + t.Error(err) + return + } + _ = conn.Close() + return + } + _ = json.MarshalWrite(w, volumeResponse{Volume: &want}) + })) + t.Cleanup(ts.Close) + c := testClient(t, ts) + if _, err := c.GrowVolume(t.Context(), "data", "vol_123", grow); err == nil || requests.Load() != 1 { + t.Fatalf("lost response = %v, requests %d", err, requests.Load()) + } + got, err := c.GrowVolume(t.Context(), "data", "vol_123", grow) + if err != nil || !reflect.DeepEqual(got, &want) || requests.Load() != 2 { + t.Fatalf("replay = %+v, %v, requests %d", got, err, requests.Load()) + } +} diff --git a/sdk/python/cocoonsandbox/client.py b/sdk/python/cocoonsandbox/client.py index 80904c67..8e0c31e5 100644 --- a/sdk/python/cocoonsandbox/client.py +++ b/sdk/python/cocoonsandbox/client.py @@ -168,6 +168,54 @@ def delete_volume(self, name: str, resource_id: str) -> None: expected_status=(204,), ) + def grow_volume( + self, + name: str, + resource_id: str, + size_gib: int, + *, + expected_generation: int, + operation_id: str, + ) -> dict[str, Any]: + """Grows block capacity. Reuse all arguments to reconcile a pending or lost response.""" + path = _volume_path(name) + "/grow" + if not resource_id or '"' in resource_id or "\r" in resource_id or "\n" in resource_id: + raise ValueError("volume resource_id must be a nonempty unquoted id") + if isinstance(size_gib, bool) or not isinstance(size_gib, int) or not 1 <= size_gib <= 1024: + raise ValueError("volume size_gib must be an integer between 1 and 1024") + if ( + isinstance(expected_generation, bool) + or not isinstance(expected_generation, int) + or not 0 <= expected_generation <= (1 << 64) - 1 + ): + raise ValueError("volume expected_generation must be an unsigned 64-bit integer") + if ( + not isinstance(operation_id, str) + or not re.fullmatch(r"[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}", operation_id) + or operation_id == "00000000-0000-0000-0000-000000000000" + ): + raise ValueError("volume operation_id must be a canonical nonzero UUID") + reply = self._request( + self.addr, + "POST", + path, + {"size_gib": size_gib, "expected_generation": expected_generation, "operation_id": operation_id}, + "grow volume", + headers={"If-Match": f'"{resource_id}"'}, + expected_status=(200, 202), + ) + volume = _managed_volume(reply, "grow volume") + growth = volume.get("growth") + if ( + not isinstance(growth, dict) + or growth.get("operation_id") != operation_id + or growth.get("expected_generation") != expected_generation + or growth.get("target_bytes") != size_gib << 30 + or growth.get("state") not in ("pending", "completed") + ): + raise APIError("grow volume", 0, "reply with missing or inconsistent growth result") + return volume + def info(self) -> dict[str, Any]: """The node's pool/claim counters, as served by GET /v1/info.""" return self._request(self.addr, "GET", "/v1/info", None, "info") diff --git a/sdk/python/tests/test_managed_volumes.py b/sdk/python/tests/test_managed_volumes.py index 198d367e..453caa37 100644 --- a/sdk/python/tests/test_managed_volumes.py +++ b/sdk/python/tests/test_managed_volumes.py @@ -183,3 +183,110 @@ def test_malformed_holder_container_preserves_api_error(node, holders): assert raised.value.status == 409 and raised.value.message == "busy" assert raised.value.holders == [] assert raised.value.reply == {"error": "busy", "holders": holders} + + +GROW_OPERATION = "efbf4748-1c72-43ee-8304-b4e1a91322b5" + + +def growth_volume(state): + return { + "name": "data", + "kind": "managed", + "id": "vol_123", + "tenant": "t1", + "node": "node-1", + "state": "ready" if state == "completed" else "growing", + "created_at": "2026-10-19T00:00:00Z", + "size_bytes": (40 if state == "completed" else 20) << 30, + "capacity_generation": 1 if state == "completed" else 0, + "growth": { + "operation_id": GROW_OPERATION, + "expected_generation": 0, + "target_bytes": 40 << 30, + "state": state, + "runtime_outcome": "acknowledged", + "guest_size_bytes": (40 << 30) if state == "completed" else 0, + "filesystem_status": "not_managed", + }, + } + + +@pytest.mark.parametrize("status,state", [(200, "completed"), (202, "pending")]) +def test_grow_volume_sends_same_request_for_reconciliation(node, status, state): + volume = growth_volume(state) + seen = [] + + def grow(body, path): + seen.append((body, dict(FakeNode.last_headers))) + return status, {"volume": volume} + + FakeNode.routes[("POST", "/v1/volumes/data/grow")] = grow + FakeNode.routes[("GET", "/v1/volumes/data")] = lambda body, path: (200, {"volume": volume}) + client = Client(node, api_token="token") + for _ in range(2): + assert client.grow_volume("data", "vol_123", 40, expected_generation=0, operation_id=GROW_OPERATION) == volume + assert client.get_volume("data") == volume + assert len(seen) == 2 + assert all( + request[0] == {"size_gib": 40, "expected_generation": 0, "operation_id": GROW_OPERATION} for request in seen + ) + assert all(request[1]["If-Match"] == '"vol_123"' for request in seen) + assert all(request[1]["Authorization"] == "Bearer token" for request in seen) + + +@pytest.mark.parametrize( + "field,value", + [ + ("name", "bad/name"), + ("resource_id", ""), + ("resource_id", '"vol_123"'), + ("size_gib", 0), + ("size_gib", 1025), + ("size_gib", True), + ("size_gib", 40.5), + ("expected_generation", -1), + ("expected_generation", 1 << 64), + ("expected_generation", True), + ("expected_generation", 0.5), + ("operation_id", ""), + ("operation_id", GROW_OPERATION.upper()), + ("operation_id", "00000000-0000-0000-0000-000000000000"), + ], +) +def test_grow_volume_validates_before_sending(node, field, value): + calls = [] + FakeNode.routes[("POST", "/v1/volumes/data/grow")] = lambda body, path: (calls.append(body) or 503, {}) + args = dict(name="data", resource_id="vol_123", size_gib=40, expected_generation=0, operation_id=GROW_OPERATION) + args[field] = value + with pytest.raises(ValueError): + Client(node).grow_volume(**args) + assert calls == [] + + +@pytest.mark.parametrize("status", [400, 401, 404, 409, 412, 428, 503]) +def test_grow_volume_preserves_errors_without_retry(node, status): + calls = [] + + def grow(body, path): + calls.append(body) + return status, {"error": "growth unavailable", "code": "volume_grow_recovery_required"} + + FakeNode.routes[("POST", "/v1/volumes/data/grow")] = grow + with pytest.raises(APIError) as raised: + Client(node).grow_volume("data", "vol_123", 40, expected_generation=0, operation_id=GROW_OPERATION) + assert raised.value.status == status + assert raised.value.code == "volume_grow_recovery_required" + assert len(calls) == 1 + + +def test_grow_volume_timeout_and_invalid_reply(node, black_hole): + with pytest.raises(APIError) as raised: + Client(black_hole, timeout=0.05).grow_volume( + "data", "vol_123", 40, expected_generation=0, operation_id=GROW_OPERATION + ) + assert raised.value.status == 0 + volume = growth_volume("completed") + volume["growth"]["operation_id"] = "different" + FakeNode.routes[("POST", "/v1/volumes/data/grow")] = lambda body, path: (200, {"volume": volume}) + with pytest.raises(APIError, match="growth result"): + Client(node).grow_volume("data", "vol_123", 40, expected_generation=0, operation_id=GROW_OPERATION) From 7dae8201f375f627493337f40a925d82105cf433 Mon Sep 17 00:00:00 2001 From: minatoaquaMK2 Date: Tue, 20 Oct 2026 06:54:25 +0000 Subject: [PATCH 09/75] Fix growth recovery ownership and stopped acknowledgement handling --- docs/managed-volumes.md | 14 +- docs/sandboxd-api.md | 4 +- sandboxd/engine/engine.go | 2 +- sandboxd/engine/volume_grow.go | 35 +- sandboxd/engine/volume_grow_test.go | 15 +- sandboxd/pool/hibernate.go | 6 +- sandboxd/pool/managed_attachment.go | 41 +- sandboxd/pool/managed_grow.go | 237 ++++++++--- sandboxd/pool/managed_grow_recovery_test.go | 440 ++++++++++++++++++++ sandboxd/pool/managed_grow_test.go | 76 +++- sandboxd/pool/managed_volume.go | 5 +- sandboxd/pool/reconcile.go | 37 +- sandboxd/pool/remove.go | 12 +- sandboxd/server/managed_volumes.go | 4 +- sandboxd/types/api.go | 28 +- sandboxd/types/types_test.go | 23 + sdk/go/managed_volume.go | 5 +- sdk/go/options.go | 25 +- sdk/go/volume_grow_test.go | 20 + 19 files changed, 860 insertions(+), 169 deletions(-) create mode 100644 sandboxd/pool/managed_grow_recovery_test.go diff --git a/docs/managed-volumes.md b/docs/managed-volumes.md index c3cf5525..f59dd4a3 100644 --- a/docs/managed-volumes.md +++ b/docs/managed-volumes.md @@ -124,7 +124,8 @@ additional disks, egress, and unsupported runtimes are refused before admission. Configure every runtime lifecycle caller to use the same qualified Cocoon binary. Online block growth is qualified for Cloud Hypervisor, not Firecracker. -Read the volume's immutable `id` and `capacity_generation`. Generate one UUID +Read the volume's immutable `id` and `capacity_generation` from its individual +GET response. The initial generation is zero; catalog summaries omit it. Generate one UUID for the operation and retain it with the absolute target and original generation. ```python @@ -181,8 +182,15 @@ silently restarts the VM to resolve it. A `409` with `volume_grow_recovery_required` requires operator recovery. Inspect the volume, preserve its runtime record, and stop the exact recorded VM generation using the qualified runtime before repeating the same POST. A confirmed stopped -generation permits ownership cleanup but does not count as online growth -success. Offline data repair is outside this API. +generation with a matching runtime receipt permits ownership cleanup but does +not count as online growth success. An unknown receipt becomes `old_vmm_gone`; +an acknowledged receipt stays `acknowledged` even when the guest stopped before +capacity observation. Both settle as `recovery_required` without changing +capacity or generation. + +If no matching runtime receipt exists, stopping the VM and retrying does not +settle the intent. Retain the claim and registry records; operator quarantine +outside this API is required. Offline data repair is outside this API. The first growth durably upgrades the managed registry to version 3. Older binaries refuse that registry. Keep the upgraded binary when restarting it. diff --git a/docs/sandboxd-api.md b/docs/sandboxd-api.md index 4bd71e4a..8aa34a5f 100644 --- a/docs/sandboxd-api.md +++ b/docs/sandboxd-api.md @@ -257,7 +257,9 @@ entry's response is byte-identical to v1. When `managed_volume_dir` is enabled, this list also includes caller-visible managed summaries and requires an explicit root or tenant token. Individual -managed GET responses carry holder details. The node-local create, inspect +managed GET responses carry holder details, `capacity_generation` (including +initial zero), and the latest growth operation. Catalog summaries retain their +existing shape; use the individual GET for the authoritative capacity generation. The node-local create, inspect conditional-delete, and online-growth routes are documented in [Tenant managed persistent volumes](managed-volumes.md#manage-a-resource). diff --git a/sandboxd/engine/engine.go b/sandboxd/engine/engine.go index 2c636d6f..8610b02e 100644 --- a/sandboxd/engine/engine.go +++ b/sandboxd/engine/engine.go @@ -414,7 +414,7 @@ func (e *Engine) run(ctx context.Context, args ...string) ([]byte, error) { var stdout, stderr bytes.Buffer cmd.Stdout, cmd.Stderr = &stdout, &stderr if err := cmd.Run(); err != nil { - return nil, fmt.Errorf("cocoon %s: %w: %s", strings.Join(args[:min(len(args), 2)], " "), err, tail(stderr.String())) + return stdout.Bytes(), fmt.Errorf("cocoon %s: %w: %s", strings.Join(args[:min(len(args), 2)], " "), err, tail(stderr.String())) } return stdout.Bytes(), nil } diff --git a/sandboxd/engine/volume_grow.go b/sandboxd/engine/volume_grow.go index 2797efb6..96a8ed1b 100644 --- a/sandboxd/engine/volume_grow.go +++ b/sandboxd/engine/volume_grow.go @@ -1,13 +1,11 @@ package engine import ( - "bytes" "context" "encoding/json/v2" "errors" "fmt" "os" - "os/exec" "path/filepath" "regexp" "slices" @@ -15,6 +13,11 @@ import ( "strings" ) +const ( + diskObserved = "observed" + diskUnobserved = "unobserved" +) + type DiskRef struct { VMID string `json:"vm_id"` DiskID string `json:"disk_id"` @@ -94,39 +97,32 @@ func (b DiskBinding) Validate() error { } return nil } + func (b DiskBinding) SameIdentity(other DiskBinding) bool { return b.Ref == other.Ref && b.VMM == other.VMM && b.Serial == other.Serial && b.Path == other.Path && b.Mode == other.Mode && b.Format == other.Format && b.Backing.Device == other.Backing.Device && b.Backing.Inode == other.Backing.Inode && slices.Equal(b.Backing.FDs, other.Backing.FDs) } + func (b DiskBinding) Equal(other DiskBinding) bool { return b.SameIdentity(other) && b.ObservedBytes == other.ObservedBytes && b.Revision == other.Revision } + func (r DiskGrowRequest) Equal(other DiskGrowRequest) bool { return r.ID == other.ID && r.TargetBytes == other.TargetBytes && r.Expected.Equal(other.Expected) } -func (e *Engine) diskCommand(ctx context.Context, args ...string) ([]byte, error) { - ctx, cancel := context.WithTimeout(ctx, cmdTimeout) - defer cancel() - cmd := exec.CommandContext(ctx, e.bin, args...) - var stdout, stderr bytes.Buffer - cmd.Stdout, cmd.Stderr = &stdout, &stderr - runErr := cmd.Run() - return stdout.Bytes(), runErr -} - func (e *Engine) InspectDisk(ctx context.Context, ref DiskRef) (DiskInspection, error) { var result DiskInspection - raw, err := e.diskCommand(ctx, "vm", "disk", "inspect", ref.VMID, "--disk-id", ref.DiskID, "--output", "json") + raw, err := e.run(ctx, "vm", "disk", "inspect", ref.VMID, "--disk-id", ref.DiskID, "--output", "json") if parseErr := json.Unmarshal(raw, &result, json.RejectUnknownMembers(true)); parseErr != nil { return DiskInspection{}, errors.Join(err, parseErr) } - if result.Requested != ref || result.GuestBlockStatus != "unobserved" { + if result.Requested != ref || result.GuestBlockStatus != diskUnobserved { return DiskInspection{}, errors.New("unsupported disk inspection") } - if result.Live.State != "observed" && result.Live.State != "unavailable" { + if result.Live.State != diskObserved && result.Live.State != "unavailable" { return DiskInspection{}, errors.New("invalid disk observation") } - if result.Live.State == "observed" && (result.Live.Binding == nil || result.Live.Binding.Validate() != nil || result.Live.Binding.Ref != ref) { + if result.Live.State == diskObserved && (result.Live.Binding == nil || result.Live.Binding.Validate() != nil || result.Live.Binding.Ref != ref) { return DiskInspection{}, errors.New("invalid observed binding") } return result, err @@ -136,7 +132,7 @@ func (e *Engine) GrowDisk(ctx context.Context, request DiskGrowRequest) (DiskGro if request.Expected.Validate() != nil || !ValidGrowID(request.ID) || request.TargetBytes <= request.Expected.ObservedBytes || request.TargetBytes%4096 != 0 { return DiskGrowResult{}, errors.New("invalid disk grow request") } - expected := DiskInspection{Requested: request.Expected.Ref, Live: DiskObservation{State: "observed", Binding: &request.Expected}, GuestBlockStatus: "unobserved"} + expected := DiskInspection{Requested: request.Expected.Ref, Live: DiskObservation{State: diskObserved, Binding: &request.Expected}, GuestBlockStatus: diskUnobserved} raw, err := json.Marshal(expected) if err != nil { return DiskGrowResult{}, err @@ -151,11 +147,11 @@ func (e *Engine) GrowDisk(ctx context.Context, request DiskGrowRequest) (DiskGro return DiskGrowResult{}, err } var result DiskGrowResult - raw, err = e.diskCommand(ctx, "vm", "disk", "grow", "--expect", f.Name(), "--request-id", request.ID, "--target-bytes", strconv.FormatInt(request.TargetBytes, 10), "--output", "json") + raw, err = e.run(ctx, "vm", "disk", "grow", "--expect", f.Name(), "--request-id", request.ID, "--target-bytes", strconv.FormatInt(request.TargetBytes, 10), "--output", "json") if parseErr := json.Unmarshal(raw, &result, json.RejectUnknownMembers(true)); parseErr != nil { return DiskGrowResult{}, errors.Join(err, parseErr) } - if result.GuestBlockStatus != "unobserved" { + if result.GuestBlockStatus != diskUnobserved { return DiskGrowResult{}, errors.New("unsupported disk grow response") } return result, err @@ -177,6 +173,7 @@ func (g GuestVolumeObservation) SameIdentity(other GuestVolumeObservation) bool g.SizeBytes = other.SizeBytes return g == other } + func (g GuestVolumeObservation) Valid() bool { return g.BootID != "" && g.Serial != "" && strings.HasPrefix(g.Device, "/dev/") && g.DevNum != "" && g.MountID != "" && filepath.IsAbs(g.Mount) && g.Root == "/" && g.Mode == "rw" && g.SizeBytes > 0 } diff --git a/sandboxd/engine/volume_grow_test.go b/sandboxd/engine/volume_grow_test.go index 094efc17..c48ce123 100644 --- a/sandboxd/engine/volume_grow_test.go +++ b/sandboxd/engine/volume_grow_test.go @@ -21,11 +21,11 @@ func TestGrowthQualifiedWireAndPrivateExpectation(t *testing.T) { } var expected DiskInspection var result DiskGrowResult - if err := json.Unmarshal(inspect, &expected, json.RejectUnknownMembers(true)); err != nil { - t.Fatal(err) + if parseErr := json.Unmarshal(inspect, &expected, json.RejectUnknownMembers(true)); parseErr != nil { + t.Fatal(parseErr) } - if err := json.Unmarshal(grow, &result, json.RejectUnknownMembers(true)); err != nil { - t.Fatal(err) + if parseErr := json.Unmarshal(grow, &result, json.RejectUnknownMembers(true)); parseErr != nil { + t.Fatal(parseErr) } if expected.Live.Binding.ObservedBytes != 20<<30 || result.Receipt.Request.TargetBytes != 40<<30 || !result.Receipt.Request.Expected.Equal(*expected.Live.Binding) || len(expected.Live.Binding.Backing.FDs) != 2 { t.Fatal("qualified schema changed") @@ -42,6 +42,7 @@ stat -c '%a' "$5" > "$GROW_CAPTURE/mode" printf '%s' "$5" > "$GROW_CAPTURE/temp-path" printf '%s\n' "$@" > "$GROW_CAPTURE/args" printf '%s' "$GROW_RESULT" +echo "qualified runtime failure" >&2 exit 1 `) e := New("cocoon", nil, nil, false, false, "") @@ -50,7 +51,7 @@ exit 1 t.Fatalf("inspection %+v %v", got, err) } receipt, err := e.GrowDisk(t.Context(), result.Receipt.Request) - if err == nil || receipt.Receipt == nil || receipt.Receipt.Outcome != "acknowledged" { + if err == nil || !strings.Contains(err.Error(), "cocoon vm disk:") || !strings.Contains(err.Error(), "qualified runtime failure") || receipt.Receipt == nil || receipt.Receipt.Outcome != "acknowledged" { t.Fatalf("nonzero stdout lost: %+v %v", receipt, err) } raw, _ := os.ReadFile(filepath.Join(dir, "expected")) @@ -100,8 +101,8 @@ func TestGrowthRejectsMalformedWireEvenWithAValidReceiptPrefix(t *testing.T) { t.Fatal(err) } var qualified DiskGrowResult - if err := json.Unmarshal(raw, &qualified); err != nil { - t.Fatal(err) + if parseErr := json.Unmarshal(raw, &qualified); parseErr != nil { + t.Fatal(parseErr) } t.Setenv("GROW_BAD_JSON", strings.TrimSpace(string(raw))+" invalid trailing bytes") fakeCocoon(t, "#!/bin/sh\nprintf '%s' \"$GROW_BAD_JSON\"\nexit 1\n") diff --git a/sandboxd/pool/hibernate.go b/sandboxd/pool/hibernate.go index 9d37a040..ef266260 100644 --- a/sandboxd/pool/hibernate.go +++ b/sandboxd/pool/hibernate.go @@ -250,7 +250,11 @@ func (m *Manager) stopVM(ctx context.Context, name string) { if m.growthOwnsVM(name) { return } - if err := m.eng.Stop(ctx, name); err != nil { + target, allowed := m.growthTeardownTarget(ctx, name) + if !allowed { + return + } + if err := m.eng.Stop(ctx, target); err != nil { log.WithFunc("pool.stopVM").Warnf(ctx, "stop vm %s: %v", name, err) } } diff --git a/sandboxd/pool/managed_attachment.go b/sandboxd/pool/managed_attachment.go index 4e48946f..fda3296d 100644 --- a/sandboxd/pool/managed_attachment.go +++ b/sandboxd/pool/managed_attachment.go @@ -266,7 +266,10 @@ func (m *Manager) recoverManaged(ctx context.Context, claims map[string]*types.S sb, exists := claims[h.SandboxID] if m.growthOwnsHold(h.SandboxID) { owned[h.VMName] = true - if !exists || !sleepMatchesClaim(h, sb) { + r.mu.Lock() + g := r.data.Volumes[h.Refs[0].Volume.Name].Growth + r.mu.Unlock() + if !exists || !sleepMatchesClaim(h, sb) || g == nil || !m.growthRecordMatches(*g, sb, live[h.VMName]) { return owned, managedStorageError() } continue @@ -279,21 +282,11 @@ func (m *Manager) recoverManaged(ctx context.Context, claims map[string]*types.S owned[h.VMName] = true continue } - rec, present := live[h.VMName] - valid := exists && sb.VMName == h.VMName && sb.Tenant == h.Tenant && sb.Failed != egressQuarantineFailure && h.State != holdRetiring && present - if valid { - identity, err := r.process(rec.PID) - if err != nil && !errors.Is(err, os.ErrNotExist) { - return owned, managedStorageError() - } - valid = err == nil && identity == h.Process + valid, err := r.recoverableManagedClaim(h, sb, live) + if err != nil { + return owned, err } if valid { - for _, volume := range heldVolumes(sb) { - if !slices.ContainsFunc(h.Refs, func(ref volumeRef) bool { return ref.Volume == volume }) { - return owned, managedStorageError() - } - } owned[h.VMName] = true if err := m.activateManaged(h.SandboxID); err != nil { return owned, err @@ -327,6 +320,26 @@ func (m *Manager) recoverManaged(ctx context.Context, claims map[string]*types.S return owned, nil } +func (r *volumeRegistry) recoverableManagedClaim(h volumeHold, sb *types.Sandbox, live map[string]types.VMRecord) (bool, error) { + rec, present := live[h.VMName] + if sb == nil || sb.VMName != h.VMName || sb.Tenant != h.Tenant || sb.Failed == egressQuarantineFailure || h.State == holdRetiring || !present { + return false, nil + } + identity, err := r.process(rec.PID) + if err != nil && !errors.Is(err, os.ErrNotExist) { + return false, managedStorageError() + } + if err != nil || identity != h.Process { + return false, nil + } + for _, volume := range heldVolumes(sb) { + if !slices.ContainsFunc(h.Refs, func(ref volumeRef) bool { return ref.Volume == volume }) { + return false, managedStorageError() + } + } + return true, nil +} + func (m *Manager) managedLiveAdd(id string, volumes []resolvedVolume) error { if len(volumes) == 0 { return nil diff --git a/sandboxd/pool/managed_grow.go b/sandboxd/pool/managed_grow.go index 6b4161de..527f2ae1 100644 --- a/sandboxd/pool/managed_grow.go +++ b/sandboxd/pool/managed_grow.go @@ -9,6 +9,17 @@ import ( "github.com/cocoonstack/sandbox/sandboxd/types" ) +const ( + growthPending = "pending" + growthCompleted = "completed" + growthRecovery = "recovery_required" + growthUnobserved = "unobserved" + growthUnknown = "outcome_unknown" + growthAcknowledged = "acknowledged" + growthVMMGone = "old_vmm_gone" + growthObserved = "observed" +) + type volumeGrowEngine interface { InspectDisk(context.Context, engine.DiskRef) (engine.DiskInspection, error) GrowDisk(context.Context, engine.DiskGrowRequest) (engine.DiskGrowResult, error) @@ -23,18 +34,26 @@ type managedGrowth struct { Guest engine.GuestVolumeObservation `json:"guest"` State string `json:"state"` Outcome string `json:"outcome"` + VMMGone bool `json:"vmm_gone,omitempty"` GuestSizeBytes int64 `json:"guest_size_bytes"` } func (g *managedGrowth) fenced() bool { - return g != nil && g.State != "completed" && g.Outcome != "old_vmm_gone" + return g != nil && g.State != growthCompleted && !g.settled() } + +func (g *managedGrowth) settled() bool { + return g != nil && (g.Outcome == growthVMMGone || g.VMMGone) +} + func growBusy() error { return volumeError(409, "volume_grow_busy", "volume growth owns the sandbox lifecycle") } + func growRecovery() error { return volumeError(409, "volume_grow_recovery_required", "volume growth requires recovery; retain the operation ID and retry after recovery") } + func (r *volumeRegistry) growthOwnsHold(id string) bool { for _, v := range r.data.Volumes { if v.Growth != nil && (v.Growth.fenced() || r.fault != nil) && v.Growth.SandboxID == id { @@ -43,6 +62,7 @@ func (r *volumeRegistry) growthOwnsHold(id string) bool { } return false } + func (m *Manager) growthOwnsHold(id string) bool { if m.managed == nil { return false @@ -51,6 +71,7 @@ func (m *Manager) growthOwnsHold(id string) bool { defer m.managed.mu.Unlock() return m.managed.growthOwnsHold(id) } + func (m *Manager) growthOwnsVM(name string) bool { if m.managed == nil { return false @@ -65,6 +86,33 @@ func (m *Manager) growthOwnsVM(name string) bool { return false } +func (m *Manager) growthTeardownTarget(ctx context.Context, name string) (string, bool) { + r := m.managed + if r == nil { + return name, true + } + r.mu.Lock() + var settled []managedGrowth + for _, v := range r.data.Volumes { + g := v.Growth + if g == nil || g.VMName != name || !g.settled() { + continue + } + if _, held := r.data.Holds[g.SandboxID]; held { + settled = append(settled, *g) + } + } + r.mu.Unlock() + target := name + for _, g := range settled { + if !m.growthProcessStopped(ctx, g) || target != name && target != g.Runtime.Expected.Ref.VMID { + return "", false + } + target = g.Runtime.Expected.Ref.VMID + } + return target, true +} + func (r *volumeRegistry) growVolume(name, expectedID, tenant string, req types.VolumeGrow) (managedVolume, error) { if r.fault != nil { return managedVolume{}, managedStorageError() @@ -120,7 +168,7 @@ func (m *Manager) GrowVolume(ctx context.Context, name, expectedID string, req t if v.Growth != nil && v.Growth.Request == req && !v.Growth.fenced() { info := r.info(v) r.mu.Unlock() - if v.Growth.State != "completed" { + if v.Growth.State != growthCompleted { return info, growRecovery() } return info, nil @@ -165,7 +213,7 @@ func (m *Manager) GrowVolume(ctx context.Context, name, expectedID string, req t defer r.mu.Unlock() v = r.data.Volumes[name] info := r.info(v) - if v.Growth.State == "recovery_required" { + if v.Growth.State == growthRecovery { return info, growRecovery() } return info, nil @@ -185,6 +233,12 @@ func (m *Manager) growthHolderLocked(sb *types.Sandbox, v managedVolume) (volume return h, nil } +func (r *volumeRegistry) growthBindingMatches(v managedVolume, h volumeHold, rec types.VMRecord, inspection engine.DiskInspection) bool { + b := inspection.Live.Binding + p, err := r.process(rec.PID) + return err == nil && p == h.Process && b != nil && b.Validate() == nil && b.Ref == inspection.Requested && b.Ref.VMID == rec.ID && b.Ref.DiskID == rec.AttachedDevices.Disks[0].ID && b.VMM == (engine.DiskProcess{BootID: p.BootID, PID: p.PID, Start: p.Start}) && b.Serial == v.Name && b.Path == r.path(v.ID) && b.Backing.Device == v.Backing.Device && b.Backing.Inode == v.Backing.Inode && b.ObservedBytes == v.Size && b.Revision != ^uint64(0) +} + func (m *Manager) admitGrowth(ctx context.Context, sb *types.Sandbox, v managedVolume, req types.VolumeGrow, ge volumeGrowEngine) error { r := m.managed m.mu.Lock() @@ -204,30 +258,38 @@ func (m *Manager) admitGrowth(ctx context.Context, sb *types.Sandbox, v managedV return growRecovery() } inspection, err := ge.InspectDisk(ctx, engine.DiskRef{VMID: rec.ID, DiskID: disk.ID}) - if err != nil || inspection.Error != nil || inspection.GuestBlockStatus != "unobserved" || inspection.Live.State != "observed" || inspection.Live.Binding == nil { + if err != nil || inspection.Error != nil || inspection.GuestBlockStatus != growthUnobserved || inspection.Live.State != growthObserved || inspection.Live.Binding == nil { return volumeError(503, "volume_grow_unsupported", "runtime cannot provide a qualified disk binding") } binding := *inspection.Live.Binding - p, err := r.process(rec.PID) - if err != nil || p != h.Process || binding.Validate() != nil || binding.Ref != inspection.Requested || binding.Ref.VMID != rec.ID || binding.Ref.DiskID != disk.ID || binding.VMM != (engine.DiskProcess{BootID: p.BootID, PID: p.PID, Start: p.Start}) || binding.Serial != v.Name || binding.Path != r.path(v.ID) || binding.Backing.Device != v.Backing.Device || binding.Backing.Inode != v.Backing.Inode || binding.ObservedBytes != v.Size || binding.Revision == ^uint64(0) { + if !r.growthBindingMatches(v, h, rec, inspection) { return growRecovery() } - if inspection.LatestReceipt != nil && inspection.LatestReceipt.Outcome == "outcome_unknown" { + if inspection.LatestReceipt != nil && inspection.LatestReceipt.Outcome == growthUnknown { return growRecovery() } guest, err := ge.ObserveVolume(ctx, rec.VsockSocket, v.Name, h.Refs[0].Volume.Mount) if err != nil || !guest.Valid() || guest.Serial != v.Name || guest.SizeBytes != v.Size { return growRecovery() } + next := managedGrowth{Request: req, Runtime: engine.DiskGrowRequest{Expected: binding, ID: req.OperationID, TargetBytes: req.SizeGiB << 30}, SandboxID: sb.ID, VMName: h.VMName, Guest: guest, State: growthPending, Outcome: growthUnobserved} + return m.commitGrowthIntent(sb, v, next) +} + +func (m *Manager) commitGrowthIntent(sb *types.Sandbox, v managedVolume, next managedGrowth) error { + if err := m.store.commitDurable(m.store.mark()); err != nil { + return err + } + r := m.managed m.mu.Lock() defer m.mu.Unlock() r.mu.Lock() defer r.mu.Unlock() - current, err := r.growVolume(v.Name, string(v.ID), v.Tenant, req) + current, err := r.growVolume(v.Name, string(v.ID), v.Tenant, next.Request) if err != nil { return err } - if current.Growth != nil && current.Growth.Request == req { + if current.Growth != nil && current.Growth.Request == next.Request { return nil } if _, err := m.growthHolderLocked(sb, current); err != nil { @@ -243,11 +305,48 @@ func (m *Manager) admitGrowth(ctx context.Context, sb *types.Sandbox, v managedV } } current.State = volumeGrowing - current.Growth = &managedGrowth{Request: req, Runtime: engine.DiskGrowRequest{Expected: binding, ID: req.OperationID, TargetBytes: req.SizeGiB << 30}, SandboxID: sb.ID, VMName: h.VMName, Guest: guest, State: "pending", Outcome: "unobserved"} + current.Growth = &next r.data.Volumes[v.Name] = current return r.commit() } +func (g managedGrowth) canDispatch(i engine.DiskInspection, err error) bool { + return g.Outcome == growthUnobserved && g.State == growthPending && err == nil && i.Error == nil && i.Live.State == growthObserved && i.Live.Binding != nil && i.Live.Binding.Equal(g.Runtime.Expected) && (i.LatestReceipt == nil || i.LatestReceipt.Revision <= g.Runtime.Expected.Revision) +} + +func (g managedGrowth) acknowledgedBinding(live engine.DiskObservation, revision uint64) bool { + b := live.Binding + return live.State == growthObserved && b != nil && b.Validate() == nil && b.SameIdentity(g.Runtime.Expected) && b.ObservedBytes == g.Runtime.TargetBytes && b.Revision == revision +} + +func (m *Manager) growthProcessStopped(ctx context.Context, g managedGrowth) bool { + p := g.Runtime.Expected.VMM + if !m.managed.processGone(processIdentity{BootID: p.BootID, PID: p.PID, Start: p.Start}) { + return false + } + rec, present, err := m.eng.Inspect(ctx, g.VMName) + if err != nil { + return false + } + return !present || rec.ID == g.Runtime.Expected.Ref.VMID && rec.Config.Name == g.VMName && rec.State == vmStateStopped && (rec.PID == 0 || rec.PID == p.PID) +} + +func (m *Manager) growthOwnerMatches(ctx context.Context, g managedGrowth, sb *types.Sandbox) bool { + if sb == nil || sb.VMName != g.VMName { + return false + } + rec, present, err := m.eng.Inspect(ctx, g.VMName) + return err == nil && present && m.growthRecordMatches(g, sb, rec) +} + +func (m *Manager) growthRecordMatches(g managedGrowth, sb *types.Sandbox, rec types.VMRecord) bool { + if sb == nil || rec.ID != g.Runtime.Expected.Ref.VMID || rec.Config.Name != g.VMName || rec.State != vmStateRunning || rec.VsockSocket != sb.VsockSocket || rec.PID != g.Runtime.Expected.VMM.PID { + return false + } + p, err := m.managed.process(rec.PID) + return err == nil && p == (processIdentity{BootID: g.Runtime.Expected.VMM.BootID, PID: g.Runtime.Expected.VMM.PID, Start: g.Runtime.Expected.VMM.Start}) +} + func (m *Manager) driveGrowth(ctx context.Context, name string, sb *types.Sandbox, ge volumeGrowEngine) error { r := m.managed r.mu.Lock() @@ -258,17 +357,16 @@ func (m *Manager) driveGrowth(ctx context.Context, name string, sb *types.Sandbo return nil } inspection, inspectErr := ge.InspectDisk(ctx, g.Runtime.Expected.Ref) - if inspection.Requested != g.Runtime.Expected.Ref || inspection.GuestBlockStatus != "unobserved" { + if inspection.Requested != g.Runtime.Expected.Ref || inspection.GuestBlockStatus != growthUnobserved { return nil } receipt, live := inspection.LatestReceipt, inspection.Live if receipt == nil || !receipt.Request.Equal(g.Runtime) { - // Only the original source binding permits a first dispatch. A known receipt is query-only. - if sb == nil || g.Outcome != "unobserved" || g.State != "pending" || inspectErr != nil || inspection.Error != nil || live.State != "observed" || live.Binding == nil || !live.Binding.Equal(g.Runtime.Expected) { - return m.saveGrowth(v, g, "recovery_required", g.Outcome, 0) + if sb == nil { + return nil } - if receipt != nil && receipt.Revision > g.Runtime.Expected.Revision { - return m.saveGrowth(v, g, "recovery_required", g.Outcome, 0) + if !g.canDispatch(inspection, inspectErr) { + return m.saveGrowth(v, g, growthRecovery, g.Outcome, 0) } result, _ := ge.GrowDisk(ctx, g.Runtime) receipt, live = result.Receipt, result.Live @@ -277,25 +375,33 @@ func (m *Manager) driveGrowth(ctx context.Context, name string, sb *types.Sandbo } } if !receipt.Request.Equal(g.Runtime) || receipt.Revision != g.Runtime.Expected.Revision+1 { - return m.saveGrowth(v, g, "recovery_required", g.Outcome, 0) + return m.saveGrowth(v, g, growthRecovery, g.Outcome, 0) } switch receipt.Outcome { - case "outcome_unknown": - return m.saveGrowth(v, g, "recovery_required", "outcome_unknown", 0) - case "old_vmm_gone": - p := g.Runtime.Expected.VMM - if !r.processGone(processIdentity{BootID: p.BootID, PID: p.PID, Start: p.Start}) { - return m.saveGrowth(v, g, "recovery_required", g.Outcome, 0) + case growthUnknown: + return m.saveGrowth(v, g, growthRecovery, growthUnknown, 0) + case growthVMMGone: + if !m.growthProcessStopped(ctx, g) { + return m.saveGrowth(v, g, growthRecovery, g.Outcome, 0) + } + return m.saveGrowth(v, g, growthRecovery, growthVMMGone, 0) + case growthAcknowledged: + if m.growthProcessStopped(ctx, g) { + g.VMMGone = true + return m.saveGrowth(v, g, growthRecovery, growthAcknowledged, 0) } - return m.saveGrowth(v, g, "recovery_required", "old_vmm_gone", 0) - case "acknowledged": - if live.State != "observed" || live.Binding == nil || live.Binding.Validate() != nil || !live.Binding.SameIdentity(g.Runtime.Expected) || live.Binding.ObservedBytes != g.Runtime.TargetBytes || live.Binding.Revision != receipt.Revision { - return m.saveGrowth(v, g, "recovery_required", "acknowledged", 0) + if !g.acknowledgedBinding(live, receipt.Revision) { + return m.saveGrowth(v, g, growthRecovery, growthAcknowledged, 0) } default: - return m.saveGrowth(v, g, "recovery_required", g.Outcome, 0) + return m.saveGrowth(v, g, growthRecovery, g.Outcome, 0) } - if err := m.saveGrowth(v, g, "pending", "acknowledged", 0); err != nil { + return m.observeGrowth(ctx, v, g, sb, ge, live, receipt.Revision) +} + +func (m *Manager) observeGrowth(ctx context.Context, v managedVolume, g managedGrowth, sb *types.Sandbox, ge volumeGrowEngine, live engine.DiskObservation, revision uint64) error { + r := m.managed + if err := m.saveGrowth(v, g, growthPending, growthAcknowledged, 0); err != nil { return err } if sb == nil { @@ -306,23 +412,23 @@ func (m *Manager) driveGrowth(ctx context.Context, name string, sb *types.Sandbo return nil } if !guest.Valid() || !guest.SameIdentity(g.Guest) { - return m.saveGrowth(v, g, "recovery_required", "acknowledged", 0) + return m.saveGrowth(v, g, growthRecovery, growthAcknowledged, 0) } if guest.SizeBytes == g.Guest.SizeBytes { return nil } if guest.SizeBytes != g.Runtime.TargetBytes { - return m.saveGrowth(v, g, "recovery_required", "acknowledged", guest.SizeBytes) + return m.saveGrowth(v, g, growthRecovery, growthAcknowledged, guest.SizeBytes) } closing, closeErr := ge.InspectDisk(ctx, g.Runtime.Expected.Ref) - if closeErr != nil || closing.Error != nil || closing.Requested != g.Runtime.Expected.Ref || closing.LatestReceipt == nil || !closing.LatestReceipt.Request.Equal(g.Runtime) || closing.LatestReceipt.Revision != receipt.Revision || closing.LatestReceipt.Outcome != "acknowledged" || closing.Live.State != "observed" || closing.Live.Binding == nil || !closing.Live.Binding.Equal(*live.Binding) { - return m.saveGrowth(v, g, "recovery_required", "acknowledged", guest.SizeBytes) + if closeErr != nil || closing.Error != nil || closing.Requested != g.Runtime.Expected.Ref || closing.LatestReceipt == nil || !closing.LatestReceipt.Request.Equal(g.Runtime) || closing.LatestReceipt.Revision != revision || closing.LatestReceipt.Outcome != growthAcknowledged || closing.Live.State != growthObserved || closing.Live.Binding == nil || !closing.Live.Binding.Equal(*live.Binding) { + return m.saveGrowth(v, g, growthRecovery, growthAcknowledged, guest.SizeBytes) } p, processErr := r.process(g.Runtime.Expected.VMM.PID) if processErr != nil || p != (processIdentity{BootID: g.Runtime.Expected.VMM.BootID, PID: g.Runtime.Expected.VMM.PID, Start: g.Runtime.Expected.VMM.Start}) { - return m.saveGrowth(v, g, "recovery_required", "acknowledged", guest.SizeBytes) + return m.saveGrowth(v, g, growthRecovery, growthAcknowledged, guest.SizeBytes) } - return m.saveGrowth(v, g, "completed", "acknowledged", guest.SizeBytes) + return m.saveGrowth(v, g, growthCompleted, growthAcknowledged, guest.SizeBytes) } func (m *Manager) saveGrowth(v managedVolume, g managedGrowth, state, outcome string, guestSize int64) error { @@ -335,7 +441,7 @@ func (m *Manager) saveGrowth(v managedVolume, g managedGrowth, state, outcome st } g.State, g.Outcome, g.GuestSizeBytes = state, outcome, guestSize current.Growth = &g - if state == "completed" { + if state == growthCompleted { if !current.Growth.fenced() && current.CapacityGeneration != g.Request.ExpectedGeneration { return managedStorageError() } @@ -354,7 +460,7 @@ func (m *Manager) saveGrowth(v managedVolume, g managedGrowth, state, outcome st } r.data.Holds[id] = h } - } else if outcome == "old_vmm_gone" { + } else if g.settled() { current.State = volumeRecovery } r.data.Volumes[v.Name] = current @@ -369,40 +475,68 @@ func (r *volumeRegistry) validateGrowth(v managedVolume) error { } return nil } + if r.data.Version != 3 { + return errors.New("invalid volume growth registry version") + } + if err := r.validateGrowthIntent(v); err != nil { + return err + } + return r.validateGrowthProgress(v) +} + +func (r *volumeRegistry) validateGrowthIntent(v managedVolume) error { + g := v.Growth b := g.Runtime.Expected - if r.data.Version != 3 || !engine.ValidGrowID(g.Request.OperationID) || g.Runtime.ID != g.Request.OperationID || g.Request.SizeGiB < 1 || g.Request.SizeGiB > 1024 || g.Runtime.TargetBytes != g.Request.SizeGiB<<30 || b.Validate() != nil || b.Revision == ^uint64(0) || b.Path != r.path(v.ID) || b.Serial != v.Name || b.Backing.Inode != v.Backing.Inode || b.Backing.Device != v.Backing.Device || b.ObservedBytes < 1<<30 || b.ObservedBytes%(1<<30) != 0 || b.ObservedBytes >= g.Runtime.TargetBytes || g.Request.ExpectedGeneration == ^uint64(0) || !g.Guest.Valid() || g.Guest.Serial != v.Name || g.Guest.SizeBytes != b.ObservedBytes || g.SandboxID == "" || g.VMName == "" { + if !engine.ValidGrowID(g.Request.OperationID) || g.Runtime.ID != g.Request.OperationID || g.Request.SizeGiB < 1 || g.Request.SizeGiB > 1024 || g.Runtime.TargetBytes != g.Request.SizeGiB<<30 || b.Validate() != nil || b.Revision == ^uint64(0) || b.Path != r.path(v.ID) || b.Serial != v.Name || b.Backing.Inode != v.Backing.Inode || b.Backing.Device != v.Backing.Device || b.ObservedBytes < 1<<30 || b.ObservedBytes%(1<<30) != 0 || b.ObservedBytes >= g.Runtime.TargetBytes || g.Request.ExpectedGeneration == ^uint64(0) || !g.Guest.Valid() || g.Guest.Serial != v.Name || g.Guest.SizeBytes != b.ObservedBytes || g.SandboxID == "" || g.VMName == "" { return errors.New("invalid volume growth intent") } + return nil +} + +func (r *volumeRegistry) validateGrowthProgress(v managedVolume) error { + g := v.Growth + b := g.Runtime.Expected + if g.VMMGone && (g.Outcome != growthAcknowledged || g.State != growthRecovery) { + return errors.New("invalid stopped growth generation") + } switch g.Outcome { - case "unobserved", "outcome_unknown", "acknowledged", "old_vmm_gone": + case growthUnobserved, growthUnknown, growthAcknowledged, growthVMMGone: default: return errors.New("invalid growth outcome") } switch g.State { - case "completed": - if (v.State != volumeReady && v.State != volumeDeleting && v.State != volumeRecovery) || g.Outcome != "acknowledged" || g.GuestSizeBytes != g.Runtime.TargetBytes || v.Size != g.Runtime.TargetBytes || g.Request.ExpectedGeneration == ^uint64(0) || v.CapacityGeneration != g.Request.ExpectedGeneration+1 { + case growthCompleted: + if (v.State != volumeReady && v.State != volumeDeleting && v.State != volumeRecovery) || g.Outcome != growthAcknowledged || g.GuestSizeBytes != g.Runtime.TargetBytes || v.Size != g.Runtime.TargetBytes || g.Request.ExpectedGeneration == ^uint64(0) || v.CapacityGeneration != g.Request.ExpectedGeneration+1 { return errors.New("inconsistent completed growth") } - case "pending", "recovery_required": - if g.State == "pending" && g.Outcome != "unobserved" && g.Outcome != "acknowledged" { + case growthPending, growthRecovery: + if g.State == growthPending && g.Outcome != growthUnobserved && g.Outcome != growthAcknowledged { return errors.New("pending growth has terminal runtime outcome") } - if v.Size != b.ObservedBytes || v.CapacityGeneration != g.Request.ExpectedGeneration || g.Outcome == "old_vmm_gone" && ((v.State != volumeRecovery && v.State != volumeDeleting) || g.State != "recovery_required") || g.Outcome != "old_vmm_gone" && v.State != volumeGrowing { + if v.Size != b.ObservedBytes || v.CapacityGeneration != g.Request.ExpectedGeneration || g.settled() && ((v.State != volumeRecovery && v.State != volumeDeleting) || g.State != growthRecovery) || !g.settled() && v.State != volumeGrowing { return errors.New("inconsistent unresolved growth") } if g.fenced() { - h, ok := r.data.Holds[g.SandboxID] - if !ok || h.VMName != g.VMName || h.State != holdClaimed || h.Sleep != nil || len(h.Refs) != 1 || h.Refs[0].ID != v.ID || !h.Refs[0].Volume.RW() || h.Refs[0].Volume.Mount == "" || h.Process != (processIdentity{BootID: b.VMM.BootID, PID: b.VMM.PID, Start: b.VMM.Start}) { - return errors.New("growth lost owner") - } + return r.validateGrowthOwner(v) } + default: return errors.New("invalid growth state") } return nil } -func (m *Manager) recoverGrowth(ctx context.Context, claims map[string]*types.Sandbox) error { +func (r *volumeRegistry) validateGrowthOwner(v managedVolume) error { + g := v.Growth + b := g.Runtime.Expected + h, ok := r.data.Holds[g.SandboxID] + if !ok || h.VMName != g.VMName || h.State != holdClaimed || h.Sleep != nil || len(h.Refs) != 1 || h.Refs[0].ID != v.ID || !h.Refs[0].Volume.RW() || h.Refs[0].Volume.Mount == "" || h.Process != (processIdentity{BootID: b.VMM.BootID, PID: b.VMM.PID, Start: b.VMM.Start}) { + return errors.New("growth lost owner") + } + return nil +} + +func (m *Manager) recoverGrowth(ctx context.Context, claims map[string]*types.Sandbox, live map[string]types.VMRecord) error { r := m.managed if r == nil { return nil @@ -426,12 +560,15 @@ func (m *Manager) recoverGrowth(ctx context.Context, claims map[string]*types.Sa for _, v := range pending { sb := claims[v.Growth.SandboxID] h, held := m.managedHold(v.Growth.SandboxID) - if sb != nil && (!held || !sleepMatchesClaim(h, sb) || sb.VMName != v.Growth.VMName || sb.Tenant != v.Tenant) { + if sb != nil && (!held || !sleepMatchesClaim(h, sb) || sb.VMName != v.Growth.VMName || sb.Tenant != v.Tenant || !m.growthRecordMatches(*v.Growth, sb, live[v.Growth.VMName]) || !m.growthOwnerMatches(ctx, *v.Growth, sb)) { sb = nil } if err := m.driveGrowth(ctx, v.Name, sb, ge); err != nil { return err } + if m.growthOwnsHold(v.Growth.SandboxID) && sb == nil { + return managedStorageError() + } } return nil } diff --git a/sandboxd/pool/managed_grow_recovery_test.go b/sandboxd/pool/managed_grow_recovery_test.go new file mode 100644 index 00000000..5a6769e4 --- /dev/null +++ b/sandboxd/pool/managed_grow_recovery_test.go @@ -0,0 +1,440 @@ +package pool + +import ( + "context" + "os" + "reflect" + "testing" + + "github.com/cocoonstack/sandbox/sandboxd/types" + + "github.com/cocoonstack/sandbox/sandboxd/engine" +) + +func TestGrowthMissingVMRecordPreservesDurableOwner(t *testing.T) { + m, e, sb, cfg, v := newGrowthTest(t) + e.unknown = true + _, _ = m.GrowVolume(t.Context(), "data", v.ID, growRequest(), "t1") + if err := m.FlushClaims(); err != nil { + t.Fatal(err) + } + m.Close() + e.mu.Lock() + delete(e.vms, sb.VMName) + e.mu.Unlock() + e.processLives.Store(true) + m = openGrowthTest(t, cfg, e) + if err := m.Reconcile(t.Context()); err == nil { + t.Error("reconcile must fail closed before publishing or saving claims") + } + if _, found := m.Sandbox(sb.ID); found { + t.Fatal("missing VM inventory published owner") + } + claims, err := m.store.load() + if err != nil { + t.Fatal(err) + } + if claims[sb.ID] == nil { + t.Fatal("reconcile deleted durable growth owner claim") + } + if !m.growthOwnsHold(sb.ID) { + t.Fatal("expected retained unresolved growth fence") + } + + m.Close() + m = openGrowthTest(t, cfg, e) + if err := m.Reconcile(t.Context()); err == nil { + t.Fatal("missing inventory must keep restart closed") + } else { + t.Log("missing inventory remains fail closed without deleting the claim", err) + } +} + +func TestGrowthChangedVMRecordCannotPublishReplacement(t *testing.T) { + m, e, sb, cfg, v := newGrowthTest(t) + e.unknown = true + _, _ = m.GrowVolume(t.Context(), "data", v.ID, growRequest(), "t1") + if err := m.FlushClaims(); err != nil { + t.Fatal(err) + } + m.Close() + e.mu.Lock() + e.pids[sb.VMName] = 90001 + e.vms[sb.VMName] = "/tmp/replacement-generation.sock" + e.mu.Unlock() + e.growMu.Lock() + e.binding.VMM.PID = 90001 + e.binding.VMM.Start = 90001 + e.growMu.Unlock() + m = openGrowthTest(t, cfg, e) + hOriginal, _ := m.managedHold(sb.ID) + m.managed.process = func(pid int) (processIdentity, error) { + if pid == hOriginal.Process.PID { + return hOriginal.Process, nil + } + return e.identity(pid) + } + if err := m.Reconcile(t.Context()); err == nil { + t.Error("reconcile must fail closed before publishing or saving claims") + } + if _, found := m.Sandbox(sb.ID); found { + t.Error("replacement generation published as original claim") + } + claims, err := m.store.load() + if err != nil { + t.Fatal(err) + } + if claims[sb.ID] == nil || claims[sb.ID].VsockSocket != sb.VsockSocket { + t.Fatal("original durable claim was replaced") + } + + if !m.growthOwnsHold(sb.ID) { + t.Fatal("growth not retained") + } + h, _ := m.managedHold(sb.ID) + if h.Process.PID == 90001 { + t.Fatal("expected original binding preserved") + } +} + +type reviewUnavailableGrowEngine struct{ *growthTestEngine } + +func (e *reviewUnavailableGrowEngine) InspectDisk(ctx context.Context, ref engine.DiskRef) (engine.DiskInspection, error) { + out, err := e.growthTestEngine.InspectDisk(ctx, ref) + out.Live = engine.DiskObservation{State: "unavailable", Reason: "recorded VMM stopped"} + return out, err +} + +func TestGrowthStoppedACKCanQuarantineAndRelease(t *testing.T) { + m, e, sb, _, v := newGrowthTest(t) + e.lag = true + got, err := m.GrowVolume(t.Context(), "data", v.ID, growRequest(), "t1") + if err != nil || got.Growth.State != "pending" || got.Growth.RuntimeOutcome != "acknowledged" { + t.Fatalf("admission %+v %v", got, err) + } + if err = e.Stop(t.Context(), sb.VMName); err != nil { + t.Fatal(err) + } + m.managed.process = func(int) (processIdentity, error) { return processIdentity{}, os.ErrNotExist } + m.eng = &reviewUnavailableGrowEngine{e} + for range 3 { + _, err = m.GrowVolume(t.Context(), "data", v.ID, growRequest(), "t1") + requireVolumeStatus(t, err, 409) + } + if e.receipt.Outcome != "acknowledged" { + t.Fatal("ACK should stay ACK under qualified runtime RefreshLocked") + } + if m.growthOwnsHold(sb.ID) { + t.Fatal("proven stopped acknowledged generation retains fence") + } + if err = m.Release(t.Context(), sb.ID, Cred{Operator: true}); err != nil { + t.Fatal(err) + } + got, err = m.GetVolume("data", "t1") + if err != nil || got.CapacityGeneration != 0 || got.Growth.RuntimeOutcome != "acknowledged" || got.Growth.State != "recovery_required" { + t.Fatalf("result %+v %v", got, err) + } +} + +func TestGrowthClaimDurabilityFailurePreventsIntent(t *testing.T) { + m, e, sb, _, v := newGrowthTest(t) + if m.store.sync { + t.Fatal("test requires ordinary asynchronous claims") + } + oldPath := m.store.path + m.store.path = t.TempDir() + defer func() { m.store.path = oldPath }() + _, err := m.GrowVolume(t.Context(), "data", v.ID, growRequest(), "t1") + if err == nil { + t.Error("growth ignored claim durability failure") + } + if e.calls != 0 || m.managed.data.Volumes["data"].Growth != nil { + t.Fatal("growth admitted before durable claim") + } + if _, found := m.Sandbox(sb.ID); !found { + t.Fatal("failed admission lost owner") + } +} + +type growthInventoryEngine struct { + *growthTestEngine + change func(*types.VMRecord) + omit bool +} + +func (e *growthInventoryEngine) List(ctx context.Context) ([]types.VMRecord, error) { + records, err := e.growthTestEngine.List(ctx) + if e.omit { + return nil, err + } + for i := range records { + e.change(&records[i]) + } + return records, err +} + +func (e *growthInventoryEngine) Inspect(ctx context.Context, name string) (types.VMRecord, bool, error) { + rec, found, err := e.growthTestEngine.Inspect(ctx, name) + e.change(&rec) + return rec, found, err +} + +func TestGrowthRecoveryRequiresOriginalInventoryAndProcess(t *testing.T) { + for _, mutation := range []string{"inventory missing", "immutable VM ID", "process start", "socket", "unreadable process"} { + t.Run(mutation, func(t *testing.T) { + m, e, sb, cfg, v := newGrowthTest(t) + e.unknown = true + _, growErr := m.GrowVolume(t.Context(), "data", v.ID, growRequest(), "t1") + requireVolumeStatus(t, growErr, 409) + if err := m.FlushClaims(); err != nil { + t.Fatal(err) + } + original, err := os.ReadFile(m.store.path) + if err != nil { + t.Fatal(err) + } + hold, _ := m.managedHold(sb.ID) + m.Close() + m = openGrowthTest(t, cfg, e) + inv := &growthInventoryEngine{growthTestEngine: e, change: func(*types.VMRecord) {}} + m.eng = inv + switch mutation { + case "inventory missing": + inv.omit = true + case "immutable VM ID": + inv.change = func(rec *types.VMRecord) { rec.ID = "replacement-vm" } + case "socket": + inv.change = func(rec *types.VMRecord) { rec.VsockSocket = "/replacement.sock" } + case "process start": + m.managed.process = func(int) (processIdentity, error) { p := hold.Process; p.Start++; return p, nil } + case "unreadable process": + e.processUnknown.Store(true) + } + if err = m.Reconcile(t.Context()); err == nil { + t.Fatal("mismatched owner was adopted") + } + if _, found := m.Sandbox(sb.ID); found { + t.Fatal("mismatched guest published") + } + after, readErr := os.ReadFile(m.store.path) + if readErr != nil || string(after) != string(original) { + t.Fatalf("owner journal changed: %v", readErr) + } + current, _ := m.managedHold(sb.ID) + if !reflect.DeepEqual(hold, current) || !m.growthOwnsHold(sb.ID) { + t.Fatal("original hold or growth fence changed") + } + if e.calls != 1 || len(e.removes) != 0 { + t.Fatal("startup dispatched or destroyed a fenced generation") + } + }) + } +} + +func TestGrowthStoppedACKRejectsReplacementAndUnprovenDeath(t *testing.T) { + for _, mutation := range []string{"process alive", "process unreadable", "replacement VM", "replacement PID", "running record", "creating record"} { + t.Run(mutation, func(t *testing.T) { + m, e, sb, _, v := newGrowthTest(t) + e.lag = true + if _, err := m.GrowVolume(t.Context(), "data", v.ID, growRequest(), "t1"); err != nil { + t.Fatal(err) + } + if err := e.Stop(t.Context(), sb.VMName); err != nil { + t.Fatal(err) + } + m.managed.process = func(int) (processIdentity, error) { return processIdentity{}, os.ErrNotExist } + inv := &growthInventoryEngine{growthTestEngine: e, change: func(*types.VMRecord) {}} + m.eng = inv + switch mutation { + case "process alive": + m.managed.process = e.identity + case "process unreadable": + m.managed.process = func(int) (processIdentity, error) { return processIdentity{}, os.ErrPermission } + case "replacement VM": + inv.change = func(rec *types.VMRecord) { rec.ID = "replacement-vm" } + case "replacement PID": + inv.change = func(rec *types.VMRecord) { rec.PID++ } + case "running record": + inv.change = func(rec *types.VMRecord) { rec.State = "running" } + case "creating record": + inv.change = func(rec *types.VMRecord) { rec.State = "creating" } + } + _, _ = m.GrowVolume(t.Context(), "data", v.ID, growRequest(), "t1") + if !m.growthOwnsHold(sb.ID) || m.removeVM(t.Context(), sb.VMName) { + t.Fatal("unverified generation permitted cleanup") + } + if len(e.removes) != 0 || e.calls != 1 { + t.Fatal("unverified generation removed or redispatched") + } + }) + } +} + +func TestGrowthSettledACKCannotRemoveLaterReplacement(t *testing.T) { + m, e, sb, _, v := newGrowthTest(t) + e.lag = true + if _, err := m.GrowVolume(t.Context(), "data", v.ID, growRequest(), "t1"); err != nil { + t.Fatal(err) + } + if err := e.Stop(t.Context(), sb.VMName); err != nil { + t.Fatal(err) + } + m.managed.process = func(int) (processIdentity, error) { return processIdentity{}, os.ErrNotExist } + _, err := m.GrowVolume(t.Context(), "data", v.ID, growRequest(), "t1") + requireVolumeStatus(t, err, 409) + if m.growthOwnsHold(sb.ID) { + t.Fatal("exact stopped ACK did not settle") + } + m.eng = &growthInventoryEngine{growthTestEngine: e, change: func(rec *types.VMRecord) { rec.ID = "replacement-vm" }} + if m.removeVM(t.Context(), sb.VMName) || len(e.removes) != 0 { + t.Fatal("settled operation removed replacement VM") + } +} + +type growthRemovalRaceEngine struct { + *growthTestEngine + removed string +} + +func (e *growthRemovalRaceEngine) Remove(_ context.Context, selector string) error { + e.removed = selector + return nil +} + +func TestGrowthSettledRemovalUsesImmutableVMID(t *testing.T) { + m, e, sb, _, v := newGrowthTest(t) + e.lag = true + if _, err := m.GrowVolume(t.Context(), "data", v.ID, growRequest(), "t1"); err != nil { + t.Fatal(err) + } + if err := e.Stop(t.Context(), sb.VMName); err != nil { + t.Fatal(err) + } + m.managed.process = func(int) (processIdentity, error) { return processIdentity{}, os.ErrNotExist } + _, err := m.GrowVolume(t.Context(), "data", v.ID, growRequest(), "t1") + requireVolumeStatus(t, err, 409) + race := &growthRemovalRaceEngine{growthTestEngine: e} + m.eng = race + m.removeVM(t.Context(), sb.VMName) + if race.removed != "immutable-grow-vm" { + t.Fatalf("removal used reusable VM selector %q", race.removed) + } +} + +func TestGrowthStoppedACKSettlementSurvivesRestart(t *testing.T) { + m, e, sb, cfg, v := newGrowthTest(t) + e.lag = true + if _, err := m.GrowVolume(t.Context(), "data", v.ID, growRequest(), "t1"); err != nil { + t.Fatal(err) + } + if err := e.Stop(t.Context(), sb.VMName); err != nil { + t.Fatal(err) + } + dead := func(int) (processIdentity, error) { return processIdentity{}, os.ErrNotExist } + m.managed.process = dead + _, err := m.GrowVolume(t.Context(), "data", v.ID, growRequest(), "t1") + requireVolumeStatus(t, err, 409) + if err = m.managed.validate(); err != nil { + t.Fatal(err) + } + m.Close() + m = openGrowthTest(t, cfg, e) + m.managed.process = dead + if err = m.Reconcile(t.Context()); err != nil { + t.Fatal(err) + } + got, err := m.GetVolume("data", "t1") + if err != nil || got.Growth.RuntimeOutcome != "acknowledged" || got.Growth.State != "recovery_required" || got.CapacityGeneration != 0 || got.SizeBytes != 1<<30 || len(got.Holders) != 0 { + t.Fatalf("settled restart: %+v %v", got, err) + } + if !m.managed.data.Volumes["data"].Growth.VMMGone || e.calls != 1 { + t.Fatal("settlement lost or redispatched") + } + if err = m.DeleteVolume(t.Context(), "data", v.ID, "t1"); err != nil { + t.Fatal(err) + } +} + +func TestGrowthRegistryRejectsFalseStoppedSettlement(t *testing.T) { + for _, outcome := range []string{"unobserved", "outcome_unknown", "acknowledged", "old_vmm_gone"} { + t.Run(outcome, func(t *testing.T) { + m, e, sb, _, v := newGrowthTest(t) + e.lag = true + if _, err := m.GrowVolume(t.Context(), "data", v.ID, growRequest(), "t1"); err != nil { + t.Fatal(err) + } + current := m.managed.data.Volumes["data"] + current.Growth.Outcome, current.Growth.VMMGone = outcome, true + if m.managed.validate() == nil { + t.Fatal("pending intent accepted settlement flag") + } + current.Growth.State = "recovery_required" + current.State = volumeRecovery + m.managed.data.Volumes["data"] = current + if err := m.managed.validate(); (err == nil) != (outcome == "acknowledged") { + t.Fatalf("stopped settlement %q accepted incorrectly: %v", outcome, err) + } + if _, held := m.managedHold(sb.ID); !held { + t.Fatal("validation changed owner") + } + }) + } +} + +func TestGrowthMissingInventoryCannotCompleteBeforeOwnerValidation(t *testing.T) { + m, e, sb, cfg, v := newGrowthTest(t) + e.lag = true + if _, err := m.GrowVolume(t.Context(), "data", v.ID, growRequest(), "t1"); err != nil { + t.Fatal(err) + } + original, err := os.ReadFile(m.store.path) + if err != nil { + t.Fatal(err) + } + observations := e.observations + m.Close() + e.lag = false + m = openGrowthTest(t, cfg, e) + m.eng = &growthInventoryEngine{growthTestEngine: e, omit: true, change: func(*types.VMRecord) {}} + if err = m.Reconcile(t.Context()); err == nil { + t.Error("missing inventory allowed completion and adoption") + } + got, getErr := m.GetVolume("data", "t1") + if getErr != nil || got.CapacityGeneration != 0 || got.SizeBytes != 1<<30 || !m.growthOwnsHold(sb.ID) { + t.Fatalf("missing owner escaped fence: %+v %v", got, getErr) + } + after, readErr := os.ReadFile(m.store.path) + if readErr != nil || string(after) != string(original) { + t.Fatalf("original claim changed: %v", readErr) + } + if e.observations != observations || len(e.removes) != 0 { + t.Fatal("unverified inventory observed guest or removed VM") + } +} + +func TestGrowthMissingInventoryPreservesUndispatchedIntent(t *testing.T) { + m, e, sb, cfg, v := newGrowthTest(t) + source := m.managed.data.Volumes["data"] + if err := m.admitGrowth(t.Context(), sb, source, growRequest(), e); err != nil { + t.Fatal(err) + } + m.Close() + m = openGrowthTest(t, cfg, e) + m.eng = &growthInventoryEngine{growthTestEngine: e, omit: true, change: func(*types.VMRecord) {}} + if err := m.Reconcile(t.Context()); err == nil { + t.Fatal("missing inventory must fail closed") + } + pending, err := m.GetVolume("data", "t1") + if err != nil || pending.Growth.State != "pending" || pending.Growth.RuntimeOutcome != "unobserved" || e.calls != 0 { + t.Fatalf("undispatched intent changed: %+v %v", pending, err) + } + m.Close() + m = openGrowthTest(t, cfg, e) + if err = m.Reconcile(t.Context()); err != nil { + t.Fatal(err) + } + got, err := m.GrowVolume(t.Context(), "data", v.ID, growRequest(), "t1") + if err != nil || got.Growth.State != "completed" || e.calls != 1 { + t.Fatalf("recovered inventory cannot resume original intent: %+v %v", got, err) + } +} diff --git a/sandboxd/pool/managed_grow_test.go b/sandboxd/pool/managed_grow_test.go index d2b477ad..afdb1192 100644 --- a/sandboxd/pool/managed_grow_test.go +++ b/sandboxd/pool/managed_grow_test.go @@ -26,6 +26,7 @@ type growthTestEngine struct { lag, unknown, lostResponse, mutateAfterGuest bool guestChange bool entered, resume chan struct{} + vmName string intentSeen bool } @@ -38,18 +39,45 @@ func (e *growthTestEngine) DiskAttach(ctx context.Context, name string, spec eng if e.disks == nil { e.disks = map[string][]types.VMDisk{} } + e.vmName = name e.disks[name] = []types.VMDisk{{ID: "actual-ch-id", Name: spec.Name, Path: spec.Path, ReadOnly: !spec.RW}} return nil } +func (e *growthTestEngine) resolveVM(selector string) string { + e.mu.Lock() + defer e.mu.Unlock() + if selector == "immutable-grow-vm" { + return e.vmName + } + return selector +} + +func (e *growthTestEngine) Remove(ctx context.Context, selector string) error { + return e.managedTestEngine.Remove(ctx, e.resolveVM(selector)) +} + +func (e *growthTestEngine) Stop(ctx context.Context, selector string) error { + return e.managedTestEngine.Stop(ctx, e.resolveVM(selector)) +} + +func (e *growthTestEngine) List(ctx context.Context) ([]types.VMRecord, error) { + records, err := e.managedTestEngine.List(ctx) + for i := range records { + records[i].ID = "immutable-grow-vm" + } + return records, err +} + func (e *growthTestEngine) Inspect(ctx context.Context, name string) (types.VMRecord, bool, error) { - rec, found, err := e.managedTestEngine.Inspect(ctx, name) + rec, found, err := e.managedTestEngine.Inspect(ctx, e.resolveVM(name)) rec.ID = "immutable-grow-vm" for i := range rec.AttachedDevices.Disks { rec.AttachedDevices.Disks[i].ID = "actual-ch-id" } return rec, found, err } + func (e *growthTestEngine) InspectDisk(_ context.Context, ref engine.DiskRef) (engine.DiskInspection, error) { e.growMu.Lock() defer e.growMu.Unlock() @@ -59,6 +87,7 @@ func (e *growthTestEngine) InspectDisk(_ context.Context, ref engine.DiskRef) (e } return engine.DiskInspection{Requested: ref, Live: engine.DiskObservation{State: "observed", Binding: &b}, LatestReceipt: e.receipt, GuestBlockStatus: "unobserved"}, nil } + func (e *growthTestEngine) GrowDisk(_ context.Context, req engine.DiskGrowRequest) (engine.DiskGrowResult, error) { e.growMu.Lock() e.calls++ @@ -96,7 +125,8 @@ func (e *growthTestEngine) GrowDisk(_ context.Context, req engine.DiskGrowReques } return result, nil } -func (e *growthTestEngine) ObserveVolume(_ context.Context, _ string, serial, mount string) (engine.GuestVolumeObservation, error) { + +func (e *growthTestEngine) ObserveVolume(_ context.Context, _, serial, mount string) (engine.GuestVolumeObservation, error) { e.growMu.Lock() defer e.growMu.Unlock() e.observations++ @@ -121,6 +151,7 @@ func openGrowthTest(t *testing.T, cfg *config.Config, e *growthTestEngine) *Mana t.Cleanup(func() { _ = m.Close() }) return m } + func newGrowthTest(t *testing.T) (*Manager, *growthTestEngine, *types.Sandbox, *config.Config, types.ManagedVolumeInfo) { t.Helper() cfg := managedConfig(t) @@ -141,6 +172,7 @@ func newGrowthTest(t *testing.T) (*Manager, *growthTestEngine, *types.Sandbox, * e.binding = engine.DiskBinding{Ref: engine.DiskRef{VMID: "immutable-grow-vm", DiskID: "actual-ch-id"}, VMM: engine.DiskProcess{BootID: h.Process.BootID, PID: h.Process.PID, Start: h.Process.Start}, Serial: "data", Path: ref.Path, Mode: "read_write", Format: "raw", Backing: engine.DiskBacking{Device: ref.Backing.Device, Inode: ref.Backing.Inode, FDs: []engine.DiskFD{{Number: 250, Access: "read_write"}, {Number: 261, Access: "read_write"}}}, ObservedBytes: ref.Backing.Size} return m, e, sb, cfg, v } + func growRequest() types.VolumeGrow { return types.VolumeGrow{SizeGiB: 2, OperationID: "b51b3d2b-4ccc-41d5-b430-3046340c30b0"} } @@ -172,16 +204,16 @@ func TestManagedGrowthCommitReplayCASAndRestart(t *testing.T) { } _, err = m.GrowVolume(t.Context(), "data", v.ID, req, "t1") requireVolumeStatus(t, err, 412) - if err := m.managed.validate(); err != nil { - t.Fatal(err) + if validateErr := m.managed.validate(); validateErr != nil { + t.Fatal(validateErr) } - if err := m.FlushClaims(); err != nil { - t.Fatal(err) + if flushErr := m.FlushClaims(); flushErr != nil { + t.Fatal(flushErr) } m.Close() m = openGrowthTest(t, cfg, e) - if err := m.Reconcile(t.Context()); err != nil { - t.Fatal(err) + if reconcileErr := m.Reconcile(t.Context()); reconcileErr != nil { + t.Fatal(reconcileErr) } got, err = m.GetVolume("data", "t1") if err != nil || got.CapacityGeneration != 2 || got.SizeBytes != 3<<30 { @@ -205,15 +237,15 @@ func TestManagedGrowthACKGuestLagAndLostResponseRecoverWithoutDispatch(t *testin if m.managedPaused(sb) { t.Fatal("growth paused data plane") } - if err := m.FlushClaims(); err != nil { - t.Fatal(err) + if flushErr := m.FlushClaims(); flushErr != nil { + t.Fatal(flushErr) } m.Close() e.lag = false e.lostResponse = false m = openGrowthTest(t, cfg, e) - if err := m.Reconcile(t.Context()); err != nil { - t.Fatal(err) + if reconcileErr := m.Reconcile(t.Context()); reconcileErr != nil { + t.Fatal(reconcileErr) } got, err = m.GrowVolume(t.Context(), "data", v.ID, growRequest(), "t1") if err != nil || got.Growth.State != "completed" || got.CapacityGeneration != 1 || e.calls != 1 { @@ -246,13 +278,13 @@ func TestManagedGrowthUnknownFencesLifecycleUntilExplicitExactStop(t *testing.T) if m.removeVM(t.Context(), sb.VMName) || m.finishManaged(sb.ID, nil) { t.Fatal("destructive fallback escaped guard") } - if _, _, err := m.beginVolumeRun(t.Context(), sb); err == nil { + if _, _, runErr := m.beginVolumeRun(t.Context(), sb); runErr == nil { t.Fatal("live mutation admitted") } - if err := m.Hibernate(t.Context(), sb.ID, Cred{Operator: true}); err == nil { + if sleepErr := m.Hibernate(t.Context(), sb.ID, Cred{Operator: true}); sleepErr == nil { t.Fatal("sleep admitted") } - if err := m.DeleteVolume(t.Context(), "data", v.ID, "t1"); err == nil { + if deleteErr := m.DeleteVolume(t.Context(), "data", v.ID, "t1"); deleteErr == nil { t.Fatal("delete admitted") } e.mu.Lock() @@ -280,6 +312,9 @@ func TestManagedGrowthUnknownFencesLifecycleUntilExplicitExactStop(t *testing.T) t.Fatal("process read error proved death") } e.processUnknown.Store(false) + if err = e.Stop(t.Context(), sb.VMName); err != nil { + t.Fatal(err) + } m.managed.process = func(int) (processIdentity, error) { return processIdentity{}, os.ErrNotExist } got, err := m.GrowVolume(t.Context(), "data", v.ID, growRequest(), "t1") requireVolumeStatus(t, err, 409) @@ -375,8 +410,8 @@ func TestManagedGrowthMissingClaimAndPriorOrphanPreserveOwner(t *testing.T) { m.managed.mu.Unlock() rec, _, _ := e.Inspect(t.Context(), sb.VMName) live := map[string]types.VMRecord{sb.VMName: rec} - if err := m.recoverGrowth(t.Context(), map[string]*types.Sandbox{}); err != nil { - t.Fatal(err) + if err := m.recoverGrowth(t.Context(), map[string]*types.Sandbox{}, live); err == nil { + t.Fatal("missing growth owner must fail closed") } if _, err := m.recoverManaged(t.Context(), map[string]*types.Sandbox{}, live); err == nil { t.Fatal("missing claim must fail closed") @@ -427,7 +462,7 @@ func TestManagedGrowthConcurrentReplayAndReadOnlyGET(t *testing.T) { if err := os.Rename(path, path+".replaced"); err != nil { t.Fatal(err) } - if err := os.WriteFile(path, nil, 0600); err != nil { + if err := os.WriteFile(path, nil, 0o600); err != nil { t.Fatal(err) } if _, err := m.GetVolume("data", "t1"); err == nil { @@ -522,7 +557,7 @@ func TestManagedGrowthRefusesUnmountedOrUnappliedOwner(t *testing.T) { } func TestManagedGrowthMigratesVersionTwoBeforeIntentAndKeepsSleepValid(t *testing.T) { - m, _, _, _, v := newGrowthTest(t) + m, e, sb, _, v := newGrowthTest(t) r := m.managed r.data.Version = 2 if err := r.commit(); err != nil { @@ -541,6 +576,9 @@ func TestManagedGrowthMigratesVersionTwoBeforeIntentAndKeepsSleepValid(t *testin if _, err := m.GrowVolume(t.Context(), "data", v.ID, growRequest(), "t1"); err != nil { t.Fatal(err) } + if !e.intentSeen || snapshots[len(snapshots)-1].Volumes["data"].Growth.SandboxID != sb.ID { + t.Fatal("migration lost growth owner") + } if len(snapshots) < 2 || snapshots[0].Version != 3 || snapshots[0].Volumes["data"].Growth != nil || snapshots[1].Volumes["data"].Growth == nil { t.Fatal("migration was not durable before intent") } diff --git a/sandboxd/pool/managed_volume.go b/sandboxd/pool/managed_volume.go index fd428489..2011bdb7 100644 --- a/sandboxd/pool/managed_volume.go +++ b/sandboxd/pool/managed_volume.go @@ -264,8 +264,7 @@ func (r *volumeRegistry) holders(id volumeID) []types.VolumeHolder { } func (r *volumeRegistry) info(v managedVolume) types.ManagedVolumeInfo { - info := types.ManagedVolumeInfo{Name: v.Name, DefaultMount: types.DefaultVolumeMount(v.Name), SizeBytes: v.Size, Available: v.State == volumeReady, Nodes: 1, Writable: true, Kind: "managed", ID: string(v.ID), Tenant: v.Tenant, Node: v.Node, State: string(v.State), CreatedAt: v.CreatedAt, Holders: r.holders(v.ID)} - info.CapacityGeneration = v.CapacityGeneration + info := types.ManagedVolumeInfo{CapacityGeneration: v.CapacityGeneration, Name: v.Name, DefaultMount: types.DefaultVolumeMount(v.Name), SizeBytes: v.Size, Available: v.State == volumeReady, Nodes: 1, Writable: true, Kind: "managed", ID: string(v.ID), Tenant: v.Tenant, Node: v.Node, State: string(v.State), CreatedAt: v.CreatedAt, Holders: r.holders(v.ID)} if g := v.Growth; g != nil { info.Growth = &types.VolumeGrowth{OperationID: g.Request.OperationID, ExpectedGeneration: g.Request.ExpectedGeneration, TargetBytes: g.Runtime.TargetBytes, State: g.State, RuntimeOutcome: g.Outcome, GuestSizeBytes: g.GuestSizeBytes, FilesystemStatus: "not_managed"} } @@ -436,5 +435,5 @@ func growthBackingMatches(v managedVolume, actual backingIdentity) bool { if actual == v.Backing { return true } - return v.Growth != nil && v.Growth.Outcome == "old_vmm_gone" && actual.Device == v.Backing.Device && actual.Inode == v.Backing.Inode && actual.Size == v.Growth.Runtime.TargetBytes + return v.Growth.settled() && actual.Device == v.Backing.Device && actual.Inode == v.Backing.Inode && actual.Size == v.Growth.Runtime.TargetBytes } diff --git a/sandboxd/pool/reconcile.go b/sandboxd/pool/reconcile.go index f6398efe..bf5b0e23 100644 --- a/sandboxd/pool/reconcile.go +++ b/sandboxd/pool/reconcile.go @@ -24,10 +24,6 @@ func (m *Manager) Reconcile(ctx context.Context) error { if err != nil { return err } - if err := m.recoverGrowth(ctx, claims); err != nil { - return err - } - fenced := m.fenceArchived(ctx, claims) vms, err := m.eng.List(ctx) if err != nil { return fmt.Errorf("list vms: %w", err) @@ -36,6 +32,10 @@ func (m *Manager) Reconcile(ctx context.Context) error { for _, vm := range vms { live[vm.Config.Name] = vm } + if err = m.recoverGrowth(ctx, claims, live); err != nil { + return err + } + fenced := m.fenceArchived(ctx, claims) snaps, snapsErr := m.eng.SnapshotList(ctx) imageIDs := m.imageIDs(ctx) @@ -97,6 +97,23 @@ func (m *Manager) Reconcile(ctx context.Context) error { for _, sb := range adopted { m.recordHibernate(ctx, sb) } + logger := log.WithFunc("pool.Reconcile") + m.sweepLocalStaging(ctx, pooled) + + if fenced { + m.reclaimOrphanArchiveCks(ctx, claims) + } + m.retryArchiveDeletes(ctx) + removed := m.sweepStaleVMs(ctx, live, owned) + + m.sweepOrphanSnapshots(ctx, snaps, snapsErr, referenced) + m.resyncEgress(ctx, live, removed) + m.warnDeniedClaims(ctx) + logger.Infof(ctx, "adopted %d claims, %d VMs live", len(m.claimed), len(live)) + return saveErr +} + +func (m *Manager) sweepLocalStaging(ctx context.Context, pooled map[string]bool) { logger := log.WithFunc("pool.Reconcile") if err := m.ckpts.SweepStaging(); err != nil { logger.Error(ctx, err, "sweep checkpoint staging") @@ -118,18 +135,6 @@ func (m *Manager) Reconcile(ctx context.Context) error { // a crash mid-export leaves a fork-* staging dir nothing in this life reuses _ = utils.RemoveDirEntries(m.dataDir, func(name string) bool { return strings.HasPrefix(name, "fork-") }) _ = utils.RemoveDirEntries(filepath.Join(m.dataDir, readerDir), nil) - - if fenced { - m.reclaimOrphanArchiveCks(ctx, claims) - } - m.retryArchiveDeletes(ctx) - removed := m.sweepStaleVMs(ctx, live, owned) - - m.sweepOrphanSnapshots(ctx, snaps, snapsErr, referenced) - m.resyncEgress(ctx, live, removed) - m.warnDeniedClaims(ctx) - logger.Infof(ctx, "adopted %d claims, %d VMs live", len(m.claimed), len(live)) - return saveErr } // sweepStaleVMs removes sbx-prefixed VMs no claim owns and returns the ones confirmed gone. diff --git a/sandboxd/pool/remove.go b/sandboxd/pool/remove.go index a9d6bfea..f62c6dbd 100644 --- a/sandboxd/pool/remove.go +++ b/sandboxd/pool/remove.go @@ -17,6 +17,10 @@ func (m *Manager) removeVM(ctx context.Context, name string) bool { if m.growthOwnsVM(name) { return false } + target, allowed := m.growthTeardownTarget(ctx, name) + if !allowed { + return false + } m.out.ClosePrebound(name) // cancellation-immune: on a canceled ctx `cocoon vm rm` no-ops and orphans. ctx = context.WithoutCancel(ctx) @@ -26,22 +30,22 @@ func (m *Manager) removeVM(ctx context.Context, name string) bool { if hold.Sleep != nil || m.growthOwnsHold(hold.SandboxID) { return false } - _ = m.eng.Remove(ctx, name) + _ = m.eng.Remove(ctx, target) return false } } if managed && hold.Sleep != nil { return m.removeManagedSleep(ctx, hold) } - err := m.eng.Remove(ctx, name) + err := m.eng.Remove(ctx, target) if managed { - return m.confirmGone(ctx, name) && m.managed.processGone(hold.Process) + return m.confirmGone(ctx, target) && m.managed.processGone(hold.Process) } if err == nil { return true } log.WithFunc("pool.removeVM").Warnf(ctx, "remove vm %s: %v; verifying", name, err) - return m.confirmGone(ctx, name) + return m.confirmGone(ctx, target) } func (m *Manager) confirmGone(ctx context.Context, name string) bool { diff --git a/sandboxd/server/managed_volumes.go b/sandboxd/server/managed_volumes.go index 95e1839a..a9c39f3f 100644 --- a/sandboxd/server/managed_volumes.go +++ b/sandboxd/server/managed_volumes.go @@ -165,8 +165,8 @@ func (s *Server) handleGrowVolume(w http.ResponseWriter, r *http.Request) { } grower, ok := mgr.(managedVolumeGrower) if !ok { - if _, err := mgr.GetVolume(r.PathValue("name"), tenantFrom(r.Context())); err != nil { - if !writePoolErr(w, err) { + if _, getErr := mgr.GetVolume(r.PathValue("name"), tenantFrom(r.Context())); getErr != nil { + if !writePoolErr(w, getErr) { volumeHTTPError(w, 503, "volume_storage_unavailable", "volume storage unavailable; retry") } return diff --git a/sandboxd/types/api.go b/sandboxd/types/api.go index 91478542..01dd1551 100644 --- a/sandboxd/types/api.go +++ b/sandboxd/types/api.go @@ -13,26 +13,26 @@ const ( // VolumeInfo is the caller-visible, host-path-free catalog projection. type VolumeInfo struct { - CapacityGeneration uint64 `json:"capacity_generation"` - Kind string `json:"kind,omitempty"` - ID string `json:"id,omitempty"` - Tenant string `json:"tenant,omitempty"` - Node string `json:"node,omitempty"` - State string `json:"state,omitempty"` - CreatedAt time.Time `json:"created_at,omitzero"` - Name string `json:"name"` - DefaultMount string `json:"default_mount"` - SizeBytes int64 `json:"size_bytes"` - Available bool `json:"available"` - Nodes int `json:"nodes"` + Kind string `json:"kind,omitempty"` + ID string `json:"id,omitempty"` + Tenant string `json:"tenant,omitempty"` + Node string `json:"node,omitempty"` + State string `json:"state,omitempty"` + CreatedAt time.Time `json:"created_at,omitzero"` + Name string `json:"name"` + DefaultMount string `json:"default_mount"` + SizeBytes int64 `json:"size_bytes"` + Available bool `json:"available"` + Nodes int `json:"nodes"` // Writable reports whether the operator allows rw claims of this name. Writable bool `json:"writable,omitzero"` } type ManagedVolumeInfo struct { VolumeInfo - Holders []VolumeHolder `json:"holders,omitempty"` - Growth *VolumeGrowth `json:"growth,omitempty"` + CapacityGeneration uint64 `json:"capacity_generation"` + Holders []VolumeHolder `json:"holders,omitempty"` + Growth *VolumeGrowth `json:"growth,omitempty"` } type VolumeHolder struct { diff --git a/sandboxd/types/types_test.go b/sandboxd/types/types_test.go index 982763b2..67ffc0ba 100644 --- a/sandboxd/types/types_test.go +++ b/sandboxd/types/types_test.go @@ -168,3 +168,26 @@ func pairsOf(n int, value string) Metadata { } return md } + +func TestVolumeCatalogWireAndManagedGeneration(t *testing.T) { + catalog := VolumeInfo{Name: "data", DefaultMount: "/volumes/data", SizeBytes: 42, Available: true, Nodes: 1} + raw, err := json.Marshal(catalog) + if err != nil { + t.Fatal(err) + } + const baseline = `{"name":"data","default_mount":"/volumes/data","size_bytes":42,"available":true,"nodes":1}` + if string(raw) != baseline { + t.Fatalf("catalog changed from baseline: %s", raw) + } + raw, err = json.Marshal(ManagedVolumeInfo{VolumeInfo: catalog}) + if err != nil { + t.Fatal(err) + } + var detail map[string]any + if err = json.Unmarshal(raw, &detail); err != nil { + t.Fatal(err) + } + if generation, present := detail["capacity_generation"]; !present || generation != float64(0) { + t.Fatalf("managed initial generation missing: %s", raw) + } +} diff --git a/sdk/go/managed_volume.go b/sdk/go/managed_volume.go index f897ebd4..53dc2900 100644 --- a/sdk/go/managed_volume.go +++ b/sdk/go/managed_volume.go @@ -14,8 +14,9 @@ import ( type ManagedVolumeInfo struct { VolumeInfo - Holders []VolumeHolder `json:"holders,omitempty"` - Growth *VolumeGrowth `json:"growth,omitempty"` + CapacityGeneration uint64 `json:"capacity_generation"` + Holders []VolumeHolder `json:"holders,omitempty"` + Growth *VolumeGrowth `json:"growth,omitempty"` } type ManagedVolumeError struct { diff --git a/sdk/go/options.go b/sdk/go/options.go index b40a2986..0d04873a 100644 --- a/sdk/go/options.go +++ b/sdk/go/options.go @@ -39,19 +39,18 @@ type Volume struct { // VolumeInfo describes one visible fleet catalog entry. type VolumeInfo struct { - Kind string `json:"kind,omitempty"` - ID string `json:"id,omitempty"` - Tenant string `json:"tenant,omitempty"` - Node string `json:"node,omitempty"` - State string `json:"state,omitempty"` - CreatedAt time.Time `json:"created_at,omitzero"` - Name string `json:"name"` - DefaultMount string `json:"default_mount"` - SizeBytes int64 `json:"size_bytes"` - CapacityGeneration uint64 `json:"capacity_generation"` - Available bool `json:"available"` - Nodes int `json:"nodes"` - Writable bool `json:"writable,omitzero"` + Kind string `json:"kind,omitempty"` + ID string `json:"id,omitempty"` + Tenant string `json:"tenant,omitempty"` + Node string `json:"node,omitempty"` + State string `json:"state,omitempty"` + CreatedAt time.Time `json:"created_at,omitzero"` + Name string `json:"name"` + DefaultMount string `json:"default_mount"` + SizeBytes int64 `json:"size_bytes"` + Available bool `json:"available"` + Nodes int `json:"nodes"` + Writable bool `json:"writable,omitzero"` } type VolumeHolder struct { diff --git a/sdk/go/volume_grow_test.go b/sdk/go/volume_grow_test.go index 20e893d6..dfd10c15 100644 --- a/sdk/go/volume_grow_test.go +++ b/sdk/go/volume_grow_test.go @@ -163,3 +163,23 @@ func TestGrowVolumeLostResponseReplay(t *testing.T) { t.Fatalf("replay = %+v, %v, requests %d", got, err, requests.Load()) } } + +func TestVolumeGenerationBelongsToManagedDetail(t *testing.T) { + catalog := VolumeInfo{Name: "data", DefaultMount: "/volumes/data", SizeBytes: 42, Available: true, Nodes: 1} + raw, err := json.Marshal(catalog) + if err != nil { + t.Fatal(err) + } + const baseline = `{"name":"data","default_mount":"/volumes/data","size_bytes":42,"available":true,"nodes":1}` + if string(raw) != baseline { + t.Fatalf("catalog changed: %s", raw) + } + var detail ManagedVolumeInfo + if err = json.Unmarshal([]byte(`{"name":"data","capacity_generation":0}`), &detail); err != nil { + t.Fatal(err) + } + raw, err = json.Marshal(detail) + if err != nil || !strings.Contains(string(raw), `"capacity_generation":0`) { + t.Fatalf("managed initial generation missing: %s %v", raw, err) + } +} From ef87ce3b740f69e9bf4985c85583bf5af643039d Mon Sep 17 00:00:00 2001 From: minatoaquaMK2 Date: Tue, 20 Oct 2026 10:15:43 +0000 Subject: [PATCH 10/75] sdk: cover bounded errors and complete volume holders --- sdk/go/client_error_test.go | 110 ++++++++++++++++++++++++++++++++++++ 1 file changed, 110 insertions(+) create mode 100644 sdk/go/client_error_test.go diff --git a/sdk/go/client_error_test.go b/sdk/go/client_error_test.go new file mode 100644 index 00000000..d523a5c4 --- /dev/null +++ b/sdk/go/client_error_test.go @@ -0,0 +1,110 @@ +package sandbox + +import ( + "encoding/json/v2" + "errors" + "fmt" + "io" + "net/http" + "reflect" + "strings" + "testing" +) + +func TestAPIErrorBoundsResponseBody(t *testing.T) { + const limit = 1 << 20 + const prefix = `{"error":"` + const suffix = `"}` + oversizeMessage := fmt.Sprintf("error response body exceeds %d bytes", limit) + for _, tt := range []struct { + name string + body string + terminalErr error + wantMessage string + wantBytes int + }{ + {name: "normal JSON", body: `{"error":"slow down"}`, wantMessage: "slow down", wantBytes: 21}, + {name: "invalid JSON", body: "not JSON", wantBytes: 8}, + { + name: "exact limit", body: prefix + strings.Repeat("x", limit-len(prefix)-len(suffix)) + suffix, + wantMessage: strings.Repeat("x", limit-len(prefix)-len(suffix)), wantBytes: limit, + }, + { + name: "oversized valid JSON", body: prefix + strings.Repeat("x", 2*limit) + suffix, + wantMessage: oversizeMessage, wantBytes: limit + 1, + }, + { + name: "oversized invalid JSON", body: strings.Repeat("x", 2*limit), + wantMessage: oversizeMessage, wantBytes: limit + 1, + }, + { + name: "oversized trailing bytes", body: `{"error":"held","code":"volume_busy","holders":[{"sandbox_id":"sb_1"}]}` + strings.Repeat(" ", 2*limit), + wantMessage: oversizeMessage, wantBytes: limit + 1, + }, + { + name: "error after cap", body: strings.Repeat("x", limit+1), terminalErr: errors.New("reader failed after cap"), + wantMessage: oversizeMessage, wantBytes: limit + 1, + }, + } { + t.Run(tt.name, func(t *testing.T) { + body := &countedErrorBody{reader: strings.NewReader(tt.body), terminalErr: tt.terminalErr} + err := apiError("claim", &http.Response{StatusCode: http.StatusTooManyRequests, Body: io.NopCloser(body)}) + got, ok := errors.AsType[*APIError](err) + if !ok || got.Verb != "claim" || got.Status != http.StatusTooManyRequests { + t.Fatalf("apiError = %v, want claim 429 APIError", err) + } + if body.bytes != tt.wantBytes { + t.Errorf("read %d response bytes, want %d", body.bytes, tt.wantBytes) + } + if body.terminalErrors != 0 { + t.Errorf("read beyond body cap and observed %d terminal errors", body.terminalErrors) + } + if got.Message != tt.wantMessage { + t.Errorf("message length %d, want %d; matched=%v", len(got.Message), len(tt.wantMessage), got.Message == tt.wantMessage) + } + if _, managed := errors.AsType[*ManagedVolumeError](err); managed { + t.Errorf("returned partially decoded managed error %v", err) + } + }) + } +} + +func TestAPIErrorReadsCompleteManagedHolders(t *testing.T) { + holders := make([]VolumeHolder, 100) + for i := range holders { + holders[i] = VolumeHolder{SandboxID: fmt.Sprintf("sb_%032d", i), Mode: "ro", State: "claimed"} + } + raw, err := json.Marshal(errorResponse{Error: "volume held", Code: "volume_busy", Volume: "data", Holders: holders}) + if err != nil { + t.Fatal(err) + } + if len(raw) <= 4096 { + t.Fatalf("holder response has %d bytes, want more than former 4096-byte limit", len(raw)) + } + body := &countedErrorBody{reader: strings.NewReader(string(raw))} + err = apiError("claim", &http.Response{StatusCode: http.StatusConflict, Body: io.NopCloser(body)}) + got, ok := errors.AsType[*ManagedVolumeError](err) + if !ok || got.Status != http.StatusConflict || got.Message != "volume held" || got.Code != "volume_busy" || got.Volume != "data" { + t.Fatalf("apiError = %v, want intact managed volume conflict", err) + } + if body.bytes != len(raw) || !reflect.DeepEqual(got.Holders, holders) { + t.Fatalf("read %d/%d bytes and received %d/%d intact holders", body.bytes, len(raw), len(got.Holders), len(holders)) + } +} + +type countedErrorBody struct { + reader *strings.Reader + terminalErr error + bytes int + terminalErrors int +} + +func (r *countedErrorBody) Read(p []byte) (int, error) { + n, err := r.reader.Read(p) + r.bytes += n + if err == io.EOF && r.terminalErr != nil { + r.terminalErrors++ + return n, r.terminalErr + } + return n, err +} From 4fe647e4b92a5e7455f602f935524716275dbc4e Mon Sep 17 00:00:00 2001 From: minatoaquaMK2 Date: Tue, 20 Oct 2026 10:15:43 +0000 Subject: [PATCH 11/75] sdk: bound control-plane error responses --- sdk/go/client.go | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/sdk/go/client.go b/sdk/go/client.go index 082c7727..80f9b5f4 100644 --- a/sdk/go/client.go +++ b/sdk/go/client.go @@ -19,7 +19,8 @@ import ( ) const ( - idleConnsPerHost = 100 + idleConnsPerHost = 100 + maxErrorBodyBytes = 1 << 20 templateQueryParam = "template" netQueryParam = "net" @@ -445,9 +446,13 @@ func encodeBody(verb string, v any) ([]byte, error) { } func apiError(verb string, resp *http.Response) error { + body, _ := io.ReadAll(io.LimitReader(resp.Body, maxErrorBodyBytes+1)) var er errorResponse - _ = json.UnmarshalRead(resp.Body, &er) - _, _ = io.Copy(io.Discard, resp.Body) + if len(body) > maxErrorBodyBytes { + er.Error = fmt.Sprintf("error response body exceeds %d bytes", maxErrorBodyBytes) + } else { + _ = json.Unmarshal(body, &er) + } base := &APIError{Verb: verb, Status: resp.StatusCode, Message: er.Error} if er.Code != "" || len(er.Holders) != 0 { return &ManagedVolumeError{APIError: base, Code: er.Code, Volume: er.Volume, Holders: er.Holders} From 72422795d6adec461b095330f0d012a192695ad6 Mon Sep 17 00:00:00 2001 From: minatoaquaMK2 Date: Tue, 20 Oct 2026 10:17:50 +0000 Subject: [PATCH 12/75] docs: document managed volume routes and security boundary --- docs/index.md | 4 +- docs/managed-volumes.md | 13 ++++--- docs/sandboxd-api.md | 84 ++++++++++++++++++++++++++++++++++++++--- docs/security.md | 25 ++++++++++++ 4 files changed, 113 insertions(+), 13 deletions(-) diff --git a/docs/index.md b/docs/index.md index 7cccc2a2..fa0d3a9b 100644 --- a/docs/index.md +++ b/docs/index.md @@ -29,8 +29,8 @@ vsock-only I/O (hardened default); `net=egress` attaches a bridge/CNI NIC. online pool reconfigure, metrics, the usage journal - [Replace a dataset in a running sandbox](live-volumes.md) — swapping a read-only catalog volume on a running claim with the volume API -- [Tenant managed persistent volumes](managed-volumes.md) — runtime resource - ownership, warm attachment, recovery, feature map and verification evidence +- [Tenant managed persistent volumes](managed-volumes.md) — node-local tenant + resources, warm attachment, hibernate, online growth, and recovery limits - [Go SDK](sdk.md) — connecting (single node and clusters), every option, the full sandbox surface, error handling - [Python SDK](sdk-python.md) — the guest and data-plane surface for the diff --git a/docs/managed-volumes.md b/docs/managed-volumes.md index f59dd4a3..87003343 100644 --- a/docs/managed-volumes.md +++ b/docs/managed-volumes.md @@ -1,9 +1,9 @@ # Tenant managed persistent volumes Managed volumes keep a tenant's data disk when a sandbox claim is released or -replaced with a different template. This phase adds node-local create, inspect, -list and conditional delete, claim-time attachment to warm VMs, and durable -single-writer exclusion. It builds on the existing +replaced with a different template. The node-local API supports create, inspect, +list, and conditional delete. Claims can attach managed disks to warm VMs with +durable single-writer exclusion. This builds on the existing [read-only live volume API](live-volumes.md). This is a subset of [issue #300](https://github.com/cocoonstack/sandbox/issues/300). @@ -285,8 +285,8 @@ attachments and delayed removal retain ownership. These rules also apply when A failed egress recovery fences the claim and attempts VM removal even if managed retirement persistence fails. A stubborn VM or unavailable registry -can keep the hold for operator recovery. This phase does not promise that an -uncooperative VMM can always be killed. +can keep the hold for operator recovery. An uncooperative VMM may still retain +the hold. After restart, the persisted egress-quarantine failure marker resumes retirement and removal. Ordinary failed claims keep their existing ownership until release. @@ -301,4 +301,5 @@ remain in force. 2. Add archive/export and restore of external data disks. 3. Add Kubernetes lifecycle/status ownership once the daemon contracts are stable. -These are proposed next steps. None is implemented or qualified by this phase. +These are proposed next steps. Owner-node placement and archive export are not +implemented or qualified. Kubernetes lifecycle ownership also remains open. diff --git a/docs/sandboxd-api.md b/docs/sandboxd-api.md index 8aa34a5f..439ae7f8 100644 --- a/docs/sandboxd-api.md +++ b/docs/sandboxd-api.md @@ -19,8 +19,10 @@ All bodies are JSON. Three token kinds: Endpoints below that say "node API token" accept root or tenant unless marked root-only. Errors are `{"error": "message"}` with the status codes -listed per endpoint. Request bodies are capped at 1 MiB; a larger one is -rejected with `400 {"error": "invalid request body"}`. +listed per endpoint. Managed volume routes also return a `code` field and +may return a `volume` name and `holders` array. Request bodies are capped +at 1 MiB; a larger one is rejected with 400. Managed routes use their +endpoint-specific error bodies. ## POST /v1/claim @@ -259,9 +261,81 @@ When `managed_volume_dir` is enabled, this list also includes caller-visible managed summaries and requires an explicit root or tenant token. Individual managed GET responses carry holder details, `capacity_generation` (including initial zero), and the latest growth operation. Catalog summaries retain their -existing shape; use the individual GET for the authoritative capacity generation. The node-local create, inspect -conditional-delete, and online-growth routes are documented in -[Tenant managed persistent volumes](managed-volumes.md#manage-a-resource). +existing shape; use the individual GET for the authoritative capacity generation. + +## PUT /v1/volumes/{name} + +Creates a node-local managed volume. Auth requires an explicit root or tenant +bearer token. A tenant token makes that tenant the owner. A root token must +name an existing tenant in `tenant`. A tenant cannot name another owner. + +The strict JSON body requires `size_gib`, an integer from 1 through 1024. +`tenant` is optional for a tenant token. The name must start with a lowercase +letter and contain at most 19 more lowercase letters, digits, underscores, or +hyphens. It must not start with `cocoon-`. + +```json +{"size_gib":20} +``` + +`201` returns the `volume` envelope and an `ETag` containing the quoted, +immutable `volume.id`. For example: + +```json +{"volume":{"kind":"managed","id":"vol_1234567890abcdef1234567890abcdef", + "tenant":"acme","node":"node-1","state":"ready", + "created_at":"2026-10-20T00:00:00Z","name":"work", + "default_mount":"/volumes/work","size_bytes":21474836480, + "available":true,"nodes":1,"writable":true,"capacity_generation":0}} +``` + +An identical request for the same owner and size returns `200` with the same +ID and ETag. The volume is a sparse raw disk. Creation does not format a +filesystem. `holders` is present when sandboxes hold the volume. `growth` is +present when a growth operation exists. + +Errors use `{"error":"message","code":"volume_code"}`. They include `400` +for an invalid body, name, size, or owner; `401` for a missing or invalid token; +`403` when a tenant selects another owner; `404` for another tenant's existing +volume; `409` for a catalog name, different size or owner, or pending deletion; +and `503` for unavailable storage or tenant lookup. A create interrupted after +allocation may return `503` with code `volume_create_incomplete` and require +operator recovery. `503` responses include `Retry-After`; the recovery codes +do not promise that an immediate retry will succeed. + +## GET /v1/volumes/{name} + +Auth requires an explicit root or tenant bearer token. Root can inspect any +managed volume. A tenant can inspect only its own. An unknown or foreign name +returns the same `404` body with code `volume_not_found`. + +`200` returns `{"volume": {...}}` and the quoted immutable ID in `ETag`. +The volume includes `capacity_generation` and, when present, `growth` with its +latest operation. `holders` contains public `sandbox_id`, `mode`, and `state` +values when the volume is held. Host backing paths and process IDs are absent. +GET observes growth progress without advancing it. + +Other errors include `401` for a missing or invalid token and `503` when the +managed store or backing inspection is unavailable. A `503` includes +`Retry-After`. + +## DELETE /v1/volumes/{name} + +Auth requires an explicit root or tenant bearer token. Send the quoted ID +from the volume's `ETag` as `If-Match`. Root can delete any managed volume; +a tenant can delete only its own. + +`204` has no body. Repeating a completed deletion with the same name and ID +also returns `204`. A new resource with the same name has a new ID, so replaying +the old deletion does not remove it. + +Errors use the managed volume error body. They include `401` for a missing or +invalid token; `404` for an unknown or foreign name or tombstone; `409` for a +held volume or pending growth; `412` for a malformed or mismatched `If-Match`; +`428` for a missing `If-Match`; and `503` for unavailable storage or backing +identity that requires operator recovery. A busy response has code +`volume_busy` and includes the volume name and public holder list. `503` +responses include `Retry-After`. ## POST /v1/volumes/{name}/grow diff --git a/docs/security.md b/docs/security.md index 6c519f49..c737e52b 100644 --- a/docs/security.md +++ b/docs/security.md @@ -130,6 +130,31 @@ to every other tenant's next read. A dataset that genuinely needs multiple writers needs an out-of-band process for who writes when, which the catalog ACL does not provide. +Managed volumes are node-local tenant resources, separate from the operator's +catalog. Management requires an explicit root or tenant bearer token; a +sandbox token cannot create, inspect, grow, or delete a resource. Tenant +listings omit other tenants' resources. Individual reads and mutations return +the same 404 for a foreign or unknown volume. Root can manage resources for +any tenant, but must name an existing tenant when creating one. API responses +expose resource IDs and public holder IDs, modes, and states. They do not +expose backing paths or process IDs. + +The managed directory and its ancestors must stay under trusted host control. +The daemon records the backing file's device, inode, and size; an identity +mismatch blocks use and requires operator recovery. Deletion requires the +quoted immutable resource ID in `If-Match`. A replay of a completed deletion +cannot delete a new resource created under the same name. A held volume +cannot be deleted, and only its tenant or root can request deletion. + +Online growth has a separate capacity generation and an operation ID. An +uncertain runtime result retains its growth record and blocks conflicting +mutations until the outcome is resolved. The API can report pending progress +or operator recovery; a 503 and `Retry-After` alone do not establish that an +immediate retry is safe. Use the same operation ID and original generation +when reconciling a lost growth response. See [Tenant managed persistent +volumes](managed-volumes.md#grow-a-running-volume) for the supported workload +and recovery rules. + ## Known limitations Facts to plan around, stated so the boundary is honest: From 5fbbfb6734dfb02191709e7b4cec2606f39c25fa Mon Sep 17 00:00:00 2001 From: minatoaquaMK2 Date: Tue, 20 Oct 2026 10:32:39 +0000 Subject: [PATCH 13/75] server: cover permanent volume recovery retry hints --- sandboxd/server/volume_grow_test.go | 31 +++++++++++++++++++++++++++++ 1 file changed, 31 insertions(+) diff --git a/sandboxd/server/volume_grow_test.go b/sandboxd/server/volume_grow_test.go index 05df878e..ebbd5145 100644 --- a/sandboxd/server/volume_grow_test.go +++ b/sandboxd/server/volume_grow_test.go @@ -19,6 +19,37 @@ const ( growBody = `{"size_gib":40,"expected_generation":0,"operation_id":"` + growOperation + `"}` ) +func TestManagedVolumeRecoveryRetryHints(t *testing.T) { + for _, tc := range []struct { + code string + retry bool + }{ + {"volume_create_incomplete", false}, + {"volume_backing_mismatch", false}, + {"volume_storage_unavailable", true}, + {"volume_grow_unsupported", false}, + } { + t.Run(tc.code, func(t *testing.T) { + mgr := &volumeManagerFake{fakeManager: &fakeManager{}, get: func(string, string) (types.ManagedVolumeInfo, error) { + return types.ManagedVolumeInfo{}, &pool.VolumeError{Status: http.StatusServiceUnavailable, Code: tc.code, Message: "volume unavailable"} + }} + ts := newTestServer(t, "root", mgr, nil) + resp := doReq(t, http.MethodGet, ts.URL+"/v1/volumes/data", "root", "") + defer resp.Body.Close() + var body pool.VolumeError + if err := json.UnmarshalRead(resp.Body, &body); err != nil { + t.Fatal(err) + } + if resp.StatusCode != http.StatusServiceUnavailable || body.Code != tc.code || body.Message != "volume unavailable" { + t.Fatalf("status=%d error=%+v", resp.StatusCode, body) + } + if retry := resp.Header.Get("Retry-After") != ""; retry != tc.retry { + t.Fatalf("Retry-After=%q, want retry hint=%v", resp.Header.Get("Retry-After"), tc.retry) + } + }) + } +} + type volumeManagerFake struct { *fakeManager get func(string, string) (types.ManagedVolumeInfo, error) From db17368f248345ec28afe0281a438430d52d8693 Mon Sep 17 00:00:00 2001 From: minatoaquaMK2 Date: Tue, 20 Oct 2026 10:33:22 +0000 Subject: [PATCH 14/75] server: omit retry hints for terminal volume refusals --- sandboxd/server/server_http.go | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/sandboxd/server/server_http.go b/sandboxd/server/server_http.go index 59a90d98..e209eda8 100644 --- a/sandboxd/server/server_http.go +++ b/sandboxd/server/server_http.go @@ -172,9 +172,7 @@ func metadataFilter(pairs []string) (types.Metadata, error) { func writePoolErr(w http.ResponseWriter, err error) bool { if volumeErr, ok := errors.AsType[*pool.VolumeError](err); ok { - if volumeErr.Status == http.StatusServiceUnavailable { - w.Header().Set("Retry-After", retryAfterSeconds) - } + mapPoolErr(w, err) writeJSON(w, volumeErr.Status, volumeErr) return true } @@ -188,6 +186,10 @@ func writePoolErr(w http.ResponseWriter, err error) bool { func mapPoolErr(w http.ResponseWriter, err error) (code int, msg string, ok bool) { code, msg, ok = lookupPoolErr(err) if ok && code == http.StatusServiceUnavailable { + if volumeErr, managed := errors.AsType[*pool.VolumeError](err); managed && + (volumeErr.Code == "volume_create_incomplete" || volumeErr.Code == "volume_backing_mismatch" || volumeErr.Code == "volume_grow_unsupported") { + return code, msg, ok + } w.Header().Set("Retry-After", retryAfterSeconds) } return code, msg, ok From 08d7094b00b44d32362a8e4bfb9f19e83c7b2d04 Mon Sep 17 00:00:00 2001 From: minatoaquaMK2 Date: Tue, 20 Oct 2026 10:34:00 +0000 Subject: [PATCH 15/75] docs: distinguish retry hints from operator recovery --- docs/sandboxd-api.md | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/docs/sandboxd-api.md b/docs/sandboxd-api.md index 439ae7f8..03ac0e60 100644 --- a/docs/sandboxd-api.md +++ b/docs/sandboxd-api.md @@ -300,8 +300,8 @@ for an invalid body, name, size, or owner; `401` for a missing or invalid token; volume; `409` for a catalog name, different size or owner, or pending deletion; and `503` for unavailable storage or tenant lookup. A create interrupted after allocation may return `503` with code `volume_create_incomplete` and require -operator recovery. `503` responses include `Retry-After`; the recovery codes -do not promise that an immediate retry will succeed. +operator recovery. `volume_create_incomplete` and `volume_backing_mismatch` +omit `Retry-After`. Transient storage failures include a retry hint. ## GET /v1/volumes/{name} @@ -334,8 +334,8 @@ invalid token; `404` for an unknown or foreign name or tombstone; `409` for a held volume or pending growth; `412` for a malformed or mismatched `If-Match`; `428` for a missing `If-Match`; and `503` for unavailable storage or backing identity that requires operator recovery. A busy response has code -`volume_busy` and includes the volume name and public holder list. `503` -responses include `Retry-After`. +`volume_busy` and includes the volume name and public holder list. A backing +mismatch omits `Retry-After`; transient storage failures include a retry hint. ## POST /v1/volumes/{name}/grow @@ -371,6 +371,7 @@ Errors are `400` invalid arguments, `401` missing or invalid management token, `412` wrong identity or stale generation, `428` missing `If-Match`, and `503` unsupported runtime or unavailable storage. A terminal recovery result returns `409` with code `volume_grow_recovery_required`; GET retains its progress. +An unsupported growth capability omits `Retry-After`. See [Grow a running volume](managed-volumes.md#grow-a-running-volume) for the supported workload and recovery limits. From 2930b4173fc0a4e1ce75353d14c02dda1bc73875 Mon Sep 17 00:00:00 2001 From: minatoaquaMK2 Date: Tue, 20 Oct 2026 10:43:41 +0000 Subject: [PATCH 16/75] docs: describe managed storage fault and release recovery --- docs/managed-volumes.md | 34 +++++++++++++++++++++++++--------- 1 file changed, 25 insertions(+), 9 deletions(-) diff --git a/docs/managed-volumes.md b/docs/managed-volumes.md index 87003343..0d270cf6 100644 --- a/docs/managed-volumes.md +++ b/docs/managed-volumes.md @@ -276,13 +276,21 @@ before any disk effect. A live operation records the union of applied, desired and possibly attached entries before changing a device. A confirmed retry can trim that union only after device cleanup and claim-journal commit succeed. -Release durably marks a hold retiring before cancelling the existing -`volumeRun` and dropping the claim. A successful runtime removal command alone -does not free the hold. Fresh runtime inspection must report absence and the -recorded boot ID/PID/start generation must be gone. Failed writes, uncertain -attachments and delayed removal retain ownership. These rules also apply when +Release durably removes the claim before retiring its hold and removing the +VM. A claim write that fails before publication restores the active claim and +keeps its VM. An error after publication has an uncertain outcome and fences +managed operations. A successful runtime removal command alone does not free +the hold. Fresh runtime inspection must report absence and the recorded boot +ID, PID, and start generation must be gone. Failed registry writes retain the +hold until persistence and cleanup succeed. These barriers also apply when `sync_claims` is disabled. +Temporary registry write failures retry in the background. The daemon logs +failure and recovery transitions and exports managed write failure and recovery +counters. An abandoned admission without a bound VM also retries its cleanup. +GET, listing, and placement checks do not persist recovery state. A temporary +backing inspection error therefore cannot permanently disable a healthy volume. + A failed egress recovery fences the claim and attempts VM removal even if managed retirement persistence fails. A stubborn VM or unavailable registry can keep the hold for operator recovery. An uncooperative VMM may still retain @@ -290,10 +298,18 @@ the hold. After restart, the persisted egress-quarantine failure marker resumes retirement and removal. Ordinary failed claims keep their existing ownership until release. -Unclaimed stopped VM records with no live process are removed before startup -continues. Unreadable process metadata and unconfirmed removal still block -startup. The stale-create collector and recorded process-generation checks -remain in force. +Unclaimed stopped VM records are removed only after the recorded process is +proved gone. Managed metadata or inventory uncertainty fences managed work +while the node continues serving ordinary volume-free claims. It preserves +uncertain claims, VM records, snapshots, and backing files. A matching healthy +durable claim takes precedence over a legacy retiring hold. Unsafe configured +directories and failure to acquire the daemon's directory lock still prevent +startup. + +An interrupted allocation with no recorded backing inode requires operator +quarantine. The API does not delete or adopt that unproven file. Preserve the +registry and backing files before recovery. Supported additive registry fields +survive writes; unsupported versions and lifecycle states remain fenced. ## Follow-up order From b2cd1590bb4aaf095bd6b70bd3ce70cd527414a7 Mon Sep 17 00:00:00 2001 From: minatoaquaMK2 Date: Tue, 20 Oct 2026 10:44:52 +0000 Subject: [PATCH 17/75] pool: Keep managed storage faults scoped and retry ownership snapshots. --- sandboxd/main.go | 1 + sandboxd/pool/claim.go | 28 +- sandboxd/pool/claims.go | 33 +- sandboxd/pool/failover.go | 17 +- sandboxd/pool/managed_attachment.go | 205 +++-- sandboxd/pool/managed_foundation_test.go | 703 ++++++++++++++++++ sandboxd/pool/managed_grow.go | 67 +- sandboxd/pool/managed_grow_recovery_test.go | 63 +- sandboxd/pool/managed_grow_test.go | 8 +- sandboxd/pool/managed_recovery.go | 8 +- .../pool/managed_recovery_regression_test.go | 5 +- .../pool/managed_sleep_boundaries_test.go | 15 +- sandboxd/pool/managed_sleep_recovery.go | 15 +- sandboxd/pool/managed_sleep_registry.go | 16 +- sandboxd/pool/managed_sleep_test.go | 6 +- .../pool/managed_startup_boundary_test.go | 5 +- sandboxd/pool/managed_volume.go | 209 ++++-- sandboxd/pool/managed_volume_test.go | 7 +- sandboxd/pool/reconcile.go | 110 ++- sandboxd/pool/remove.go | 2 +- sandboxd/pool/telemetry.go | 59 +- sandboxd/pool/volume.go | 8 +- sandboxd/pool/volume_mutation.go | 38 +- sandboxd/pool/volume_registry.go | 149 +++- sandboxd/pool/volume_registry_fields.go | 103 +++ sandboxd/pool/volume_registry_fields_test.go | 122 +++ sandboxd/server/metrics.go | 2 + 27 files changed, 1720 insertions(+), 284 deletions(-) create mode 100644 sandboxd/pool/managed_foundation_test.go create mode 100644 sandboxd/pool/volume_registry_fields.go create mode 100644 sandboxd/pool/volume_registry_fields_test.go diff --git a/sandboxd/main.go b/sandboxd/main.go index 1c6298a9..a40e468b 100644 --- a/sandboxd/main.go +++ b/sandboxd/main.go @@ -90,6 +90,7 @@ func main() { if err != nil { logger.Fatalf(ctx, err, "init pool manager") } + defer func() { _ = mgr.Close() }() ctx, stop := signal.NotifyContext(ctx, os.Interrupt, syscall.SIGTERM) defer stop() diff --git a/sandboxd/pool/claim.go b/sandboxd/pool/claim.go index ac38b599..2416dd19 100644 --- a/sandboxd/pool/claim.go +++ b/sandboxd/pool/claim.go @@ -303,6 +303,9 @@ func (m *Manager) dialPassive(ctx context.Context, id string, port uint16) (net. // releaseResolved re-checks under m.mu that sb is still the live claim: no double teardown. func (m *Manager) releaseResolved(ctx context.Context, id string, sb *types.Sandbox) error { + if m.managedClaimUncertain(sb) { + return managedStorageError() + } if m.managedLifecycle(sb) { sb.Transition.Lock() defer sb.Transition.Unlock() @@ -326,10 +329,11 @@ func (m *Manager) releaseResolved(ctx context.Context, id string, sb *types.Sand marked = ck m.mu.Lock() } - if err := m.retireManaged(id); err != nil { + if m.growthOwnsHold(id) { m.mu.Unlock() - return err + return growBusy() } + previousHold, _ := m.managedHold(id) volumeHolds := m.cancelVolumeRun(sb) snap, ck, vmName := sb.HibernateSnap, sb.ArchiveCk, sb.VMName if ck != "" { @@ -338,9 +342,15 @@ func (m *Manager) releaseResolved(ctx context.Context, id string, sb *types.Sand m.dropClaimLocked(sb) js := m.store.del(id) m.mu.Unlock() - if saveErr := m.store.commit(js); saveErr != nil { + commit := m.store.commit + if previousHold.SandboxID != "" { + commit = m.store.commitDurable + } + saveErr := commit(js) + if saveErr != nil { + m.restoreManagedClaim(previousHold, saveErr) m.mu.Lock() - m.addClaimLocked(sb) // roll back so memory matches the still-durable claim + m.addClaimLocked(sb) m.restoreVolumeHolds(sb, volumeHolds) delete(m.pendingCks, ck) rb := m.store.set(sb) @@ -355,6 +365,7 @@ func (m *Manager) releaseResolved(ctx context.Context, id string, sb *types.Sand logger.Errorf(ctx, saveErr, "persist release of %s", id) return fmt.Errorf("release %s: %w", id, saveErr) } + _ = m.retireManaged(id) // cleanup must survive the caller hanging up; the claim is already dropped. ctx = context.WithoutCancel(ctx) if ck != "" { @@ -589,6 +600,7 @@ func (m *Manager) abortVolumeClaim(ctx context.Context, vmName string, reserved } func (m *Manager) reapOnce(ctx context.Context) { + m.retryManagedSnapshot() now := time.Now() type victim struct { action reapAction @@ -602,11 +614,14 @@ func (m *Manager) reapOnce(ctx context.Context) { var dropped, purges []string var unrows []*types.Sandbox for id, sb := range m.claimed { + if m.managedClaimUncertain(sb) { + continue + } // a zero deadline means no expiry (an archived claim kept forever). if sb.Deadline.IsZero() || !now.After(sb.Deadline) { continue } - if m.managedLifecycle(sb) { + if _, held := m.managedHold(sb.ID); held { managedExpired = append(managedExpired, sb) continue } @@ -631,9 +646,6 @@ func (m *Manager) reapOnce(ctx context.Context) { } expired = append(expired, victim{action: action, id: id, sb: sb}) default: - if err := m.retireManaged(id); err != nil { - continue - } expired = append(expired, victim{action: reapDestroy, id: id, vmName: sb.VMName, snap: sb.HibernateSnap, tenant: sb.Tenant, sb: sb, holds: m.cancelVolumeRun(sb)}) m.dropClaimLocked(sb) dropped = append(dropped, id) diff --git a/sandboxd/pool/claims.go b/sandboxd/pool/claims.go index e1e85672..9e0efe61 100644 --- a/sandboxd/pool/claims.go +++ b/sandboxd/pool/claims.go @@ -17,7 +17,6 @@ import ( "github.com/projecteru2/core/log" "github.com/cocoonstack/sandbox/sandboxd/types" - "github.com/cocoonstack/sandbox/sandboxd/utils" ) const ( @@ -74,8 +73,9 @@ type claimRow struct { // claimStore persists claimed sandboxes across daemon restarts; warm VMs are not persisted. type claimStore struct { - path string - sync bool + path string + sync bool + syncDirectory func(string) error // writeMu orders commits and guards every row's raw and buf. writeMu sync.Mutex @@ -90,7 +90,7 @@ type claimStore struct { } func newClaimStore(dataDir string, sync bool) *claimStore { - return &claimStore{path: filepath.Join(dataDir, "claims.json"), sync: sync, rows: map[string]*claimRow{}} + return &claimStore{path: filepath.Join(dataDir, "claims.json"), sync: sync, syncDirectory: syncDir, rows: map[string]*claimRow{}} } func (s *claimStore) load() (map[string]*types.Sandbox, error) { @@ -241,8 +241,18 @@ func (s *claimStore) replaceMode(raw []byte, durable bool) error { if err != nil { return fmt.Errorf("write claims: %w", err) } - if err := utils.ReplaceFileSync(f, s.path, raw); err != nil { - return fmt.Errorf("commit claims: %w", err) + defer func() { _ = os.Remove(tmp) }() + if _, err := f.Write(raw); err != nil { + return errors.Join(err, f.Close()) + } + if err := errors.Join(f.Sync(), f.Close()); err != nil { + return err + } + if err := os.Rename(tmp, s.path); err != nil { + return err + } + if err := s.syncDirectory(filepath.Dir(s.path)); err != nil { + return &claimPublishError{err} } return nil } @@ -286,3 +296,14 @@ func volumesWantDTO(sb *types.Sandbox) []types.Volume { } return append([]types.Volume{}, sb.VolumesWant...) } + +type claimPublishError struct{ error } + +func (e *claimPublishError) Unwrap() error { return e.error } + +func (s *claimStore) contains(id string) bool { + s.mu.Lock() + defer s.mu.Unlock() + _, ok := s.rows[id] + return ok +} diff --git a/sandboxd/pool/failover.go b/sandboxd/pool/failover.go index 926a8404..602d66d6 100644 --- a/sandboxd/pool/failover.go +++ b/sandboxd/pool/failover.go @@ -411,18 +411,23 @@ func (m *Manager) forgetClaim(ctx context.Context, sb *types.Sandbox) bool { m.mu.Unlock() return false } - if _, managed := m.managedHold(sb.ID); managed { - if err := m.retireManaged(sb.ID); err != nil { - m.mu.Unlock() - return false - } + _, managed := m.managedHold(sb.ID) + if managed { m.cancelVolumeRun(sb) } m.dropClaimLocked(sb) js := m.store.del(sb.ID) m.mu.Unlock() - if err := m.store.commit(js); err != nil { + commit := m.store.commit + if managed { + commit = m.store.commitDurable + } + if err := commit(js); err != nil { m.store.recommit(ctx, js) + return !managed + } + if managed { + _ = m.retireManaged(sb.ID) } return true } diff --git a/sandboxd/pool/managed_attachment.go b/sandboxd/pool/managed_attachment.go index fda3296d..4638d557 100644 --- a/sandboxd/pool/managed_attachment.go +++ b/sandboxd/pool/managed_attachment.go @@ -14,6 +14,8 @@ import ( "github.com/cocoonstack/sandbox/sandboxd/types" ) +var errProcessMetadata = errors.New("process metadata unavailable") + func readProcessIdentity(pid int) (processIdentity, error) { return readProcessIdentityAt("/proc", pid) } @@ -26,7 +28,7 @@ func readProcessIdentityAt(procDir string, pid int) (processIdentity, error) { boot, err := fs.ReadFile(proc, "sys/kernel/random/boot_id") if err != nil { // Missing boot metadata cannot prove that the PID is gone. - return processIdentity{}, fmt.Errorf("read VMM boot identity: %v", err) + return processIdentity{}, fmt.Errorf("%w: read VMM boot identity: %w", errProcessMetadata, err) } bootID := strings.TrimSpace(string(boot)) if bootID == "" { @@ -56,7 +58,7 @@ func (r *volumeRegistry) processGone(p processIdentity) bool { return false } current, err := r.process(p.PID) - if errors.Is(err, os.ErrNotExist) { + if processMissing(err) { return true } return err == nil && current != p @@ -64,6 +66,20 @@ func (r *volumeRegistry) processGone(p processIdentity) bool { func (m *Manager) admitManaged(tenant string, volumes []resolvedVolume) ([]types.Volume, error) { applied := appliedVolumes(volumes) + for i := range volumes { + var err error + if volumes[i].managedID == "" { + volumes[i].backing, err = catalogBacking(volumes[i].disk.Path) + } else { + volumes[i].backing, err = inspectBacking(volumes[i].disk.Path) + if err == nil { + err = m.managed.verifyDirectories() + } + } + if err != nil { + return nil, managedStorageError() + } + } m.mu.Lock() defer m.mu.Unlock() if err := m.quotaErr(1, tenant); err != nil { @@ -86,39 +102,51 @@ func (m *Manager) admitManaged(tenant string, volumes []resolvedVolume) ([]types if err := m.reserveVolumes(applied); err != nil { return nil, err } - if err := m.confirmVolumesClean(volumes); err != nil { - m.releaseVolumes(applied) - return nil, err - } r.data.Holds[h.SandboxID] = h for i := range volumes { volumes[i].holdID = h.SandboxID } - if err := r.commit(); err != nil { + r.mu.Unlock() + m.mu.Unlock() + cleanErr := m.confirmVolumesClean(volumes) + r.mu.Lock() + if cleanErr != nil { + delete(r.data.Holds, h.SandboxID) + } + err := r.commitLocked() + r.mu.Unlock() + m.mu.Lock() + r.mu.Lock() + if cleanErr != nil && err == nil { + m.releaseVolumes(applied) + return nil, cleanErr + } + if err != nil { + h.State = holdRetiring + r.data.Holds[h.SandboxID] = h return nil, err } + if current, ok := r.data.Holds[h.SandboxID]; !ok || current.State != holdAttaching || current.VMName != "" { + return nil, managedStorageError() + } return applied, nil } func (r *volumeRegistry) referenceForAdmission(tenant string, volume resolvedVolume) (volumeRef, error) { if volume.managedID == "" { - identity, err := catalogBacking(volume.disk.Path) - if err != nil { - return volumeRef{}, ErrVolumeUnavailable - } - return volumeRef{Volume: volume.applied, Path: volume.disk.Path, Backing: identity}, nil + return volumeRef{Volume: volume.applied, Path: volume.disk.Path, Backing: volume.backing}, nil } v, ok := r.data.Volumes[volume.applied.Name] if !ok || v.ID != volume.managedID || (tenant != "" && v.Tenant != tenant) { return volumeRef{}, managedNotFound() } - if v.State != volumeReady || r.verify(v) != nil { + if v.State != volumeReady || v.Backing != volume.backing { return volumeRef{}, managedStorageError() } for _, other := range r.data.Holds { for _, ref := range other.Refs { if ref.ID == v.ID && (ref.Volume.RW() || volume.applied.RW()) { - return volumeRef{}, &VolumeError{Status: 409, Code: volumeBusyCode, Message: "volume is held by a sandbox", Volume: v.Name, Holders: r.holders(v.ID)} + return volumeRef{}, &VolumeError{Status: 409, Code: volumeBusyCode, Message: volumeBusyMessage, Volume: v.Name, Holders: r.holders(v.ID)} } } } @@ -144,8 +172,16 @@ func (m *Manager) releaseUnbound(volumes []resolvedVolume, reserved []types.Volu return } delete(r.data.Holds, id) - if r.commit() == nil { + m.mu.Unlock() + err := r.commitLocked() + r.mu.Unlock() + m.mu.Lock() + r.mu.Lock() + if err == nil { m.releaseVolumes(reserved) + } else { + h.State = holdRetiring + r.data.Holds[id] = h } } @@ -176,7 +212,7 @@ func (m *Manager) bindManaged(ctx context.Context, sb *types.Sandbox, volumes [] h.VMName = sb.VMName h.Process = p r.data.Holds[id] = h - return r.commit() + return r.commitLocked() } func (m *Manager) activateManaged(id string) error { @@ -193,30 +229,36 @@ func (m *Manager) activateManaged(id string) error { if r.fault != nil || h.State == holdRetiring { return managedStorageError() } + if h.State == holdClaimed { + return nil + } h.State = holdClaimed r.data.Holds[id] = h - return r.commit() + return r.commitLocked() } -func (m *Manager) finishManaged(id string, clears []string) bool { +func (m *Manager) finishManaged(id string, clears []string, charges ...[]types.Volume) bool { r := m.managed - m.mu.Lock() - defer m.mu.Unlock() + r.fileMu.Lock() + defer r.fileMu.Unlock() r.mu.Lock() - defer r.mu.Unlock() h, ok := r.data.Holds[id] + blocked := r.closed || r.fault != nil || r.writeFault != nil || r.growthOwnsHold(id) || r.discharging[id] + r.mu.Unlock() if !ok { - return true + return !blocked } - if r.fault != nil || r.growthOwnsHold(id) { + if blocked { return false } - commit := m.store.commit - if h.Sleep != nil { - commit = m.store.commitDurable - } - if err := commit(m.store.del(id)); err != nil { - return false + if m.store.contains(id) { + commit := m.store.commit + if h.Sleep != nil { + commit = m.store.commitDurable + } + if err := commit(m.store.del(id)); err != nil { + return false + } } for _, path := range clears { for _, ref := range h.Refs { @@ -228,19 +270,60 @@ func (m *Manager) finishManaged(id string, clears []string) bool { } } } + r.mu.Lock() + defer r.mu.Unlock() + if _, ok := r.data.Holds[id]; !ok { + return true + } + if r.discharging == nil { + r.discharging = map[string]bool{} + } + r.discharging[id] = true delete(r.data.Holds, id) - if r.commit() != nil { + if r.commitLocked() != nil { r.data.Holds[id] = h + delete(r.discharging, id) return false } - refs := make([]types.Volume, len(h.Refs)) - for i, ref := range h.Refs { - refs[i] = ref.Volume + r.mu.Unlock() + m.mu.Lock() + volumes := make([]types.Volume, 0, len(h.Refs)) + for _, ref := range h.Refs { + volumes = append(volumes, ref.Volume) + } + for _, charge := range charges { + for _, volume := range charge { + if volumeIndex(volumes, volume.Name) < 0 { + volumes = append(volumes, volume) + } + } } - m.releaseVolumes(refs) + m.releaseVolumes(volumes) + m.mu.Unlock() + r.mu.Lock() + delete(r.discharging, id) return true } +func (m *Manager) restoreManagedClaim(previous volumeHold, err error) { + if previous.SandboxID == "" { + return + } + r := m.managed + r.mu.Lock() + defer r.mu.Unlock() + if _, published := errors.AsType[*claimPublishError](err); published { + r.fault = err + return + } + h, ok := r.data.Holds[previous.SandboxID] + if ok && h.State == holdRetiring { + h.State = previous.State + r.data.Holds[h.SandboxID] = h + _ = r.commitLocked() + } +} + func (m *Manager) recoverManaged(ctx context.Context, claims map[string]*types.Sandbox, live map[string]types.VMRecord) (map[string]bool, error) { owned := map[string]bool{} r := m.managed @@ -274,10 +357,20 @@ func (m *Manager) recoverManaged(ctx context.Context, claims map[string]*types.S } continue } - if h.Sleep != nil && h.State != holdRetiring { + if h.Sleep != nil && (h.State != holdRetiring || exists && sleepMatchesClaim(h, sb)) { if !exists || !sleepMatchesClaim(h, sb) { return owned, managedStorageError() } + if h.State == holdRetiring { + r.mu.Lock() + h.State = holdClaimed + r.data.Holds[h.SandboxID] = h + err := r.commitLocked() + r.mu.Unlock() + if err != nil { + return owned, err + } + } projectSleepState(sb, h.Sleep) owned[h.VMName] = true continue @@ -288,6 +381,12 @@ func (m *Manager) recoverManaged(ctx context.Context, claims map[string]*types.S } if valid { owned[h.VMName] = true + if h.State == holdRetiring { + r.mu.Lock() + h.State = holdAttaching + r.data.Holds[h.SandboxID] = h + r.mu.Unlock() + } if err := m.activateManaged(h.SandboxID); err != nil { return owned, err } @@ -322,11 +421,17 @@ func (m *Manager) recoverManaged(ctx context.Context, claims map[string]*types.S func (r *volumeRegistry) recoverableManagedClaim(h volumeHold, sb *types.Sandbox, live map[string]types.VMRecord) (bool, error) { rec, present := live[h.VMName] - if sb == nil || sb.VMName != h.VMName || sb.Tenant != h.Tenant || sb.Failed == egressQuarantineFailure || h.State == holdRetiring || !present { + if sb == nil || sb.VMName != h.VMName || sb.Tenant != h.Tenant || sb.Failed == egressQuarantineFailure || !present { return false, nil } + if rec.PID == 0 && rec.State == vmStateStopped { + if r.processGone(h.Process) { + return false, nil + } + return false, managedStorageError() + } identity, err := r.process(rec.PID) - if err != nil && !errors.Is(err, os.ErrNotExist) { + if err != nil && !processMissing(err) { return false, managedStorageError() } if err != nil || identity != h.Process { @@ -340,7 +445,7 @@ func (r *volumeRegistry) recoverableManagedClaim(h volumeHold, sb *types.Sandbox return true, nil } -func (m *Manager) managedLiveAdd(id string, volumes []resolvedVolume) error { +func (m *Manager) stageManagedLiveAdd(id string, volumes []resolvedVolume) error { if len(volumes) == 0 { return nil } @@ -363,14 +468,10 @@ func (m *Manager) managedLiveAdd(id string, volumes []resolvedVolume) error { return volumeError(409, "volume_attachment_uncertain", "previous attachment requires sandbox release") } } - identity, err := catalogBacking(v.disk.Path) - if err != nil { - return ErrVolumeUnavailable - } - h.Refs = append(h.Refs, volumeRef{Volume: v.applied, Path: v.disk.Path, Backing: identity}) + h.Refs = append(h.Refs, volumeRef{Volume: v.applied, Path: v.disk.Path, Backing: v.backing}) } r.data.Holds[id] = h - return r.commit() + return nil } func (m *Manager) managedLiveRemove(id string, volumes []types.Volume) error { @@ -402,7 +503,7 @@ func (m *Manager) managedLiveRemove(id string, volumes []types.Volume) error { } h.Refs = refs r.data.Holds[id] = h - return r.commit() + return r.commitLocked() } func catalogBacking(path string) (backingIdentity, error) { @@ -416,3 +517,17 @@ func catalogBacking(path string) (backingIdentity, error) { raw := st.Sys().(*syscall.Stat_t) return backingIdentity{Device: backingDeviceID(raw.Dev), Inode: raw.Ino, Size: st.Size()}, nil } + +func (m *Manager) commitManaged() error { + if m.managed == nil { + return nil + } + r := m.managed + r.mu.Lock() + defer r.mu.Unlock() + return r.commitLocked() +} + +func processMissing(err error) bool { + return errors.Is(err, os.ErrNotExist) && !errors.Is(err, errProcessMetadata) +} diff --git a/sandboxd/pool/managed_foundation_test.go b/sandboxd/pool/managed_foundation_test.go new file mode 100644 index 00000000..520d3248 --- /dev/null +++ b/sandboxd/pool/managed_foundation_test.go @@ -0,0 +1,703 @@ +package pool + +import ( + "bytes" + "errors" + "os" + "os/exec" + "path/filepath" + "strconv" + "sync" + "sync/atomic" + "syscall" + "testing" + "time" + + "github.com/cocoonstack/sandbox/sandboxd/config" + "github.com/cocoonstack/sandbox/sandboxd/types" +) + +func foundationConfig(t *testing.T) *config.Config { + t.Helper() + dir := t.TempDir() + return &config.Config{DataDir: filepath.Join(dir, "data"), ManagedVolumeDir: filepath.Join(dir, "managed"), APIToken: "root", Pools: []config.PoolSpec{{PoolKey: testKey, Warm: 1}}} +} + +func foundationClaim(t *testing.T, m *Manager, names ...string) *types.Sandbox { + t.Helper() + sb, err := m.ClaimProvision(t.Context(), testKey, managedOptions(names...)) + if err != nil { + t.Fatal(err) + } + return sb +} + +func TestFoundationReleaseCommitAllowsUnrelatedRenew(t *testing.T) { + m := openManagedTest(t, foundationConfig(t), newManagedEngine()) + createManagedTest(t, m, "data") + sb := foundationClaim(t, m, "data") + unrelated := foundationClaim(t, m) + entered, resume := make(chan struct{}), make(chan struct{}) + m.managed.commitFile = func(path string, raw []byte) error { + close(entered) + <-resume + return writeVolumeSnapshot(path, raw) + } + released := make(chan error, 1) + go func() { released <- m.Release(t.Context(), sb.ID, Cred{Operator: true}) }() + select { + case <-entered: + case <-time.After(3 * time.Second): + t.Fatal("release never reached registry commit") + } + managerLocked := !m.mu.TryLock() + if !managerLocked { + m.mu.Unlock() + } + registryLocked := !m.managed.mu.TryLock() + if !registryLocked { + m.managed.mu.Unlock() + } + renewed := make(chan error, 1) + go func() { + _, err := m.Renew(t.Context(), unrelated.ID, Cred{Operator: true}, time.Minute, "") + renewed <- err + }() + blocked := false + select { + case err := <-renewed: + if err != nil { + t.Fatal(err) + } + case <-time.After(150 * time.Millisecond): + blocked = true + } + t.Logf("at stalled registry commit manager_locked=%t registry_locked=%t unrelated_renew_blocked=%t", managerLocked, registryLocked, blocked) + m.managed.commitFile = writeVolumeSnapshot + close(resume) + if err := <-released; err != nil { + t.Fatal(err) + } + if blocked { + if err := <-renewed; err != nil { + t.Fatal(err) + } + } + if managerLocked || registryLocked || blocked { + t.Error("registry persistence blocks unrelated lease renewal while holding global locks") + } +} + +func TestFoundationCreateCommitAllowsConcurrentVolumeRun(t *testing.T) { + m := openManagedTest(t, foundationConfig(t), newManagedEngine()) + createManagedTest(t, m, "data") + sb := foundationClaim(t, m, "data") + unrelated := foundationClaim(t, m) + entered, resume := make(chan struct{}), make(chan struct{}) + m.managed.commitFile = func(path string, raw []byte) error { close(entered); <-resume; return writeVolumeSnapshot(path, raw) } + created := make(chan error, 1) + go func() { + _, _, err := m.CreateVolume(t.Context(), "other", types.VolumeCreate{SizeGiB: 1}, "t1") + created <- err + }() + select { + case <-entered: + case <-time.After(3 * time.Second): + t.Fatal("create never reached registry commit") + } + if !m.mu.TryLock() { + close(resume) + t.Fatal("create unexpectedly holds manager mutex before indirect waiter") + } + m.mu.Unlock() + runDone := make(chan error, 1) + go func() { + _, run, err := m.beginVolumeRun(t.Context(), sb) + if err == nil { + m.finishVolumeRun(sb, run) + } + runDone <- err + }() + managerLocked := false + deadline := time.Now().Add(time.Second) + for time.Now().Before(deadline) { + if !m.mu.TryLock() { + managerLocked = true + break + } + m.mu.Unlock() + time.Sleep(time.Millisecond) + } + renewed := make(chan error, 1) + go func() { + _, err := m.Renew(t.Context(), unrelated.ID, Cred{Operator: true}, time.Minute, "") + renewed <- err + }() + blocked := false + select { + case err := <-renewed: + if err != nil { + t.Fatal(err) + } + case <-time.After(150 * time.Millisecond): + blocked = true + } + t.Logf("create commit initially manager unlocked; volume-run waiter manager_locked=%t unrelated_renew_blocked=%t", managerLocked, blocked) + m.managed.commitFile = writeVolumeSnapshot + close(resume) + if err := <-created; err != nil { + t.Fatal(err) + } + if err := <-runDone; err != nil { + t.Fatal(err) + } + if blocked { + if err := <-renewed; err != nil { + t.Fatal(err) + } + } + if managerLocked || blocked { + t.Error("registry I/O blocks global manager through volume-run lock ordering") + } +} + +func TestFoundationRegistryFaultRecoversForReleaseAndReap(t *testing.T) { + m := openManagedTest(t, foundationConfig(t), newManagedEngine()) + for _, name := range []string{"first", "second", "third"} { + createManagedTest(t, m, name) + } + first := foundationClaim(t, m, "first") + second := foundationClaim(t, m, "second") + third := foundationClaim(t, m, "third") + var calls atomic.Int64 + m.managed.commitFile = func(string, []byte) error { calls.Add(1); return syscall.ENOSPC } + err := m.Release(t.Context(), first.ID, Cred{Operator: true}) + if _, live := m.Sandbox(first.ID); live { + t.Fatal("registry failure prevented durable claim release") + } + if _, held := m.managedHold(first.ID); !held { + t.Fatal("registry failure discharged the reservation") + } + m.managed.commitFile = func(path string, raw []byte) error { calls.Add(1); return writeVolumeSnapshot(path, raw) } + secondErr := m.Release(t.Context(), second.ID, Cred{Operator: true}) + m.mu.Lock() + third.Deadline = time.Now().Add(-time.Minute) + m.mu.Unlock() + m.reapOnce(t.Context()) + waitFor(t, func() bool { _, live := m.Sandbox(third.ID); return !live }) + _, retained := m.Sandbox(third.ID) + t.Logf("first_release=%v second_release=%v expired_claim_retained=%t registry_commit_calls=%d", err, secondErr, retained, calls.Load()) + if secondErr != nil || retained { + t.Error("one registry write fault blocks unrelated release and expiry after storage hook is healthy") + } +} + +func TestFoundationReleaseRollbackKeepsLiveClaim(t *testing.T) { + cfg := foundationConfig(t) + e := newManagedEngine() + m := openManagedTest(t, cfg, e) + createManagedTest(t, m, "data") + sb := foundationClaim(t, m, "data") + if err := os.Mkdir(m.store.path+".tmp", 0o700); err != nil { + t.Fatal(err) + } + releaseErr := m.Release(t.Context(), sb.ID, Cred{Operator: true}) + if releaseErr == nil { + t.Fatal("injected claim persistence failure did not fail release") + } + h, held := m.managedHold(sb.ID) + _, live := m.Sandbox(sb.ID) + _, patchErr := m.PatchVolumes(t.Context(), sb.ID, "t1", types.VolumePatch{}) + t.Logf("release_error=%v claim_restored=%t hold_present=%t hold_phase=%s patch_error=%v", releaseErr, live, held, h.State, patchErr) + if err := os.Remove(m.store.path + ".tmp"); err != nil { + t.Fatal(err) + } + waitFor(t, func() bool { return !m.store.recommitting.Load() && m.store.synced() }) + if err := m.Close(); err != nil { + t.Fatal(err) + } + next := openManagedTest(t, cfg, e) + if err := next.Reconcile(t.Context()); err != nil { + t.Fatal(err) + } + next.retryRemovals(t.Context()).Wait() + _, adopted := next.Sandbox(sb.ID) + _, present, inspectErr := e.Inspect(t.Context(), sb.VMName) + t.Logf("after_restart claim_present=%t VM_present=%t inspect_error=%v", adopted, present, inspectErr) + if h.State == holdRetiring || errors.Is(patchErr, ErrFailed) || !adopted || !present { + t.Error("failed release restored claim but retained retirement and restart destroyed its VM") + } +} + +func TestManagedRegistryCloseWaitsForWriter(t *testing.T) { + cfg := managedConfig(t) + m := openManagedTest(t, cfg, newManagedEngine()) + entered, resume := make(chan struct{}), make(chan struct{}) + m.managed.commitFile = func(path string, raw []byte) error { + close(entered) + <-resume + return writeVolumeSnapshot(path, raw) + } + created := make(chan error, 1) + go func() { + _, _, err := m.CreateVolume(t.Context(), "data", types.VolumeCreate{SizeGiB: 1}, "t1") + created <- err + }() + <-entered + closed := make(chan error, 1) + go func() { closed <- m.Close() }() + select { + case <-closed: + t.Fatal("closed the daemon lock during an active registry write") + case <-time.After(20 * time.Millisecond): + } + if other, err := openVolumeRegistry(cfg.ManagedVolumeDir, "node"); err == nil { + _ = other.close() + t.Fatal("another daemon acquired the lock while a writer was active") + } + close(resume) + <-created + if err := <-closed; err != nil { + t.Fatal(err) + } +} + +func TestManagedCorruptionRetainsManagedClaimAndServesOrdinaryClaims(t *testing.T) { + cfg := managedConfig(t) + e := newManagedEngine() + m := openManagedTest(t, cfg, e) + createManagedTest(t, m, "data") + managed := foundationClaim(t, m, "data") + ordinary := foundationClaim(t, m) + m.Close() + if err := os.WriteFile(filepath.Join(cfg.ManagedVolumeDir, "registry.json"), []byte("{"), 0o600); err != nil { + t.Fatal(err) + } + m = openManagedTest(t, cfg, e) + if err := m.Reconcile(t.Context()); err != nil { + t.Fatal(err) + } + if _, live := m.Sandbox(managed.ID); !live { + t.Fatal("metadata uncertainty discarded the durable managed claim") + } + if _, err := m.Renew(t.Context(), ordinary.ID, Cred{Operator: true}, time.Minute, ""); err != nil { + t.Fatal(err) + } + if err := m.Release(t.Context(), ordinary.ID, Cred{Operator: true}); err != nil { + t.Fatal(err) + } + if _, present, err := e.Inspect(t.Context(), ordinary.VMName); err != nil || present { + t.Fatalf("ordinary VM survived release: %t %v", present, err) + } + if _, present, err := e.Inspect(t.Context(), managed.VMName); err != nil || !present { + t.Fatalf("uncertain managed VM was removed: %t %v", present, err) + } + requireVolumeStatus(t, m.Release(t.Context(), managed.ID, Cred{Operator: true}), 503) + if _, err := m.ClaimProvision(t.Context(), testKey, ClaimOptions{}); err != nil { + t.Fatal(err) + } +} + +func TestManagedIdleRetryRecoversWriteFailure(t *testing.T) { + m := openManagedTest(t, managedConfig(t), newManagedEngine()) + createManagedTest(t, m, "data") + sb := foundationClaim(t, m, "data") + m.managed.commitFile = func(string, []byte) error { return syscall.ENOSPC } + _ = m.Release(t.Context(), sb.ID, Cred{Operator: true}) + if m.Counters().ManagedWriteFailures == 0 { + t.Fatal("write failure was not counted") + } + m.managed.commitFile = writeVolumeSnapshot + m.reapOnce(t.Context()) + waitFor(t, func() bool { return m.ManagedVolumeStatus() == nil }) + if m.Counters().ManagedWriteRecoveries != 1 { + t.Fatal("maintenance did not retry the retained registry state") + } + m.retryRemovals(t.Context()).Wait() + if _, held := m.managedHold(sb.ID); held { + t.Fatal("recovered teardown retained a discharged hold") + } +} + +func TestManagedRetiredCatalogNameRemainsReserved(t *testing.T) { + cfg := managedConfig(t) + path := filepath.Join(filepath.Dir(cfg.DataDir), "catalog.img") + if err := os.WriteFile(path, []byte("catalog"), 0o600); err != nil { + t.Fatal(err) + } + cfg.Volumes = []config.VolumeSpec{{Name: "catalog", Path: path, Writable: true}} + e := newManagedEngine() + m := openManagedTest(t, cfg, e) + sb := foundationClaim(t, m, "catalog") + m.Close() + cfg.Volumes = nil + m = openManagedTest(t, cfg, e) + if err := m.Reconcile(t.Context()); err != nil { + t.Fatal(err) + } + _, _, err := m.CreateVolume(t.Context(), "catalog", types.VolumeCreate{SizeGiB: 1}, "t1") + requireVolumeStatus(t, err, 409) + createManagedTest(t, m, "other") + if err := m.Release(t.Context(), sb.ID, Cred{Operator: true}); err != nil { + t.Fatal(err) + } + createManagedTest(t, m, "catalog") +} + +func TestManagedReadFailureDoesNotChangeDurableLifecycle(t *testing.T) { + if os.Getenv("MANAGED_EMFILE_CHILD") == "" { + cmd := exec.Command(os.Args[0], "-test.run=^TestManagedReadFailureDoesNotChangeDurableLifecycle$") + cmd.Env = append(os.Environ(), "MANAGED_EMFILE_CHILD=1") + if out, err := cmd.CombinedOutput(); err != nil { + t.Fatalf("isolated EMFILE test: %v\n%s", err, out) + } + return + } + m := openManagedTest(t, managedConfig(t), newManagedEngine()) + createManagedTest(t, m, "data") + path := filepath.Join(m.managed.dir, "registry.json") + before, err := os.ReadFile(path) + if err != nil { + t.Fatal(err) + } + var previous syscall.Rlimit + if err = syscall.Getrlimit(syscall.RLIMIT_NOFILE, &previous); err != nil { + t.Fatal(err) + } + limit := previous + limit.Cur = 64 + if err = syscall.Setrlimit(syscall.RLIMIT_NOFILE, &limit); err != nil { + t.Fatal(err) + } + var files []*os.File + for { + f, openErr := os.Open("/dev/null") + if errors.Is(openErr, syscall.EMFILE) { + break + } + if openErr != nil { + t.Fatal(openErr) + } + files = append(files, f) + } + _, getErr := m.GetVolume("data", "t1") + _, placementErr := m.VolumePlacement(testKey, "t1", []string{"data"}) + for _, f := range files { + _ = f.Close() + } + if err = syscall.Setrlimit(syscall.RLIMIT_NOFILE, &previous); err != nil { + t.Fatal(err) + } + requireVolumeStatus(t, getErr, 503) + requireVolumeStatus(t, placementErr, 503) + after, err := os.ReadFile(path) + if err != nil { + t.Fatal(err) + } + if !bytes.Equal(before, after) { + t.Fatal("read-only lookup changed the durable registry") + } + info, err := m.GetVolume("data", "t1") + if err != nil || info.State != "ready" { + t.Fatalf("transient open error persisted: %+v %v", info, err) + } +} + +func TestManagedCatalogAdmissionCanceledDuringCommitReleasesAllCharges(t *testing.T) { + cfg := managedConfig(t) + cfg.Volumes = []config.VolumeSpec{{Name: "catalog", Path: writeVolumeImage(t, "catalog.img", "catalog"), Writable: true}} + m := openManagedTest(t, cfg, newManagedEngine()) + createManagedTest(t, m, "data") + sb := foundationClaim(t, m, "data") + entered, resume := make(chan struct{}), make(chan struct{}) + var once sync.Once + m.managed.commitFile = func(path string, raw []byte) error { + once.Do(func() { close(entered); <-resume }) + return writeVolumeSnapshot(path, raw) + } + patched := make(chan error, 1) + go func() { + _, err := m.PatchVolumes(t.Context(), sb.ID, "t1", types.VolumePatch{Add: []types.Volume{{Name: "catalog"}}}) + patched <- err + }() + <-entered + released := make(chan error, 1) + go func() { released <- m.Release(t.Context(), sb.ID, Cred{Operator: true}) }() + waitFor(t, func() bool { _, live := m.Sandbox(sb.ID); return !live }) + close(resume) + if err := <-released; err != nil { + t.Fatal(err) + } + if err := <-patched; err == nil { + t.Fatal("canceled admission succeeded") + } + if got := volumeHoldersOf(m, "catalog"); got != (volumeHolders{}) { + t.Fatalf("canceled catalog admission leaked charge: %+v", got) + } + if _, err := m.ClaimProvision(t.Context(), testKey, managedOptions("catalog")); err != nil { + t.Fatal(err) + } +} + +func TestFoundationCatalogAddCommitsOutsideTransitionAndManager(t *testing.T) { + cfg := foundationConfig(t) + path := filepath.Join(filepath.Dir(cfg.DataDir), "catalog.raw") + if err := os.WriteFile(path, []byte("catalog"), 0o600); err != nil { + t.Fatal(err) + } + cfg.Volumes = []config.VolumeSpec{{Name: "catalog", Path: path}} + m := openManagedTest(t, cfg, newManagedEngine()) + createManagedTest(t, m, "data") + sb := foundationClaim(t, m, "data") + entered, resume := make(chan struct{}), make(chan struct{}) + m.managed.commitFile = func(path string, raw []byte) error { close(entered); <-resume; return writeVolumeSnapshot(path, raw) } + patched := make(chan error, 1) + go func() { + _, err := m.PatchVolumes(t.Context(), sb.ID, "t1", types.VolumePatch{Add: []types.Volume{{Name: "catalog"}}}) + patched <- err + }() + select { + case <-entered: + case <-time.After(3 * time.Second): + close(resume) + t.Fatal("catalog add never reached registry commit") + } + transitionLocked := !sb.Transition.TryLock() + if !transitionLocked { + sb.Transition.Unlock() + } + managerLocked := !m.mu.TryLock() + if !managerLocked { + m.mu.Unlock() + } + registryLocked := !m.managed.mu.TryLock() + if !registryLocked { + m.managed.mu.Unlock() + } + t.Logf("catalog add registry commit transition_locked=%t manager_locked=%t registry_locked=%t", transitionLocked, managerLocked, registryLocked) + m.managed.commitFile = writeVolumeSnapshot + close(resume) + if err := <-patched; err != nil { + t.Fatal(err) + } + if transitionLocked || managerLocked || registryLocked { + t.Error("catalog live-add persistence runs under Transition and both global mutexes") + } +} + +func TestFoundationFinishSkipsAbsentClaimWrite(t *testing.T) { + e := newManagedEngine() + m := openManagedTest(t, foundationConfig(t), e) + createManagedTest(t, m, "data") + sb := foundationClaim(t, m, "data") + m.releaseDelay = time.Hour + if err := m.Release(t.Context(), sb.ID, Cred{Operator: true}); err != nil { + t.Fatal(err) + } + if err := e.Remove(t.Context(), sb.VMName); err != nil { + t.Fatal(err) + } + before := m.store.mark() + m.store.writeMu.Lock() + done := make(chan bool, 1) + go func() { done <- m.finishManaged(sb.ID, nil) }() + select { + case ok := <-done: + if !ok { + t.Error("finish failed") + } + case <-time.After(time.Second): + m.store.writeMu.Unlock() + <-done + t.Fatal("finish waited for the claim writer with no claim row to remove") + } + m.store.writeMu.Unlock() + if after := m.store.mark(); after != before { + t.Fatalf("finish sequenced a redundant claim deletion: before=%v after=%v", before, after) + } +} + +func TestFoundationMountedReleaseRollbackKeepsLiveClaim(t *testing.T) { + cfg := foundationConfig(t) + cfg.ManagedVolumeHibernate = &config.ManagedVolumeHibernate{HookArgv: []string{"/usr/local/bin/volume-hook"}, HookTimeoutSeconds: 1} + m := openManagedTest(t, cfg, newManagedEngine()) + createManagedTest(t, m, "data") + opts := managedOptions("data") + opts.Volumes[0].AttachOnly = false + sb, err := m.ClaimProvision(t.Context(), testKey, opts) + if err != nil { + t.Fatal(err) + } + if !m.managedLifecycle(sb) { + t.Fatal("mounted claim did not select managed lifecycle release") + } + if err := os.Mkdir(m.store.path+".tmp", 0o700); err != nil { + t.Fatal(err) + } + releaseErr := m.Release(t.Context(), sb.ID, Cred{Operator: true}) + if releaseErr == nil { + t.Fatal("injected mounted release persistence failure did not fail") + } + h, held := m.managedHold(sb.ID) + _, live := m.Sandbox(sb.ID) + if err := os.Remove(m.store.path + ".tmp"); err != nil { + t.Fatal(err) + } + unsynced := !m.store.synced() + t.Logf("managed lifecycle release_error=%v claim_restored=%t hold_present=%t hold_phase=%s claims_store_unsynced=%t recommitting=%t", releaseErr, live, held, h.State, unsynced, m.store.recommitting.Load()) + if err := m.store.commitDurable(m.store.mark()); err != nil { + t.Fatal(err) + } + if h.State == holdRetiring { + t.Error("managed lifecycle failed release also restores claim without restoring its hold phase") + } +} + +func TestManagedClaimPostRenameFailureFencesWithoutTeardown(t *testing.T) { + m := openManagedTest(t, managedConfig(t), newManagedEngine()) + createManagedTest(t, m, "data") + sb := foundationClaim(t, m, "data") + m.store.syncDirectory = func(string) error { return syscall.ENOSPC } + if err := m.Release(t.Context(), sb.ID, Cred{Operator: true}); err == nil { + t.Fatal("ambiguous claim deletion reported success") + } + waitFor(t, func() bool { return !m.store.recommitting.Load() && m.store.synced() }) + requireVolumeStatus(t, m.ManagedVolumeStatus(), 503) + m.reapOnce(t.Context()) + requireVolumeStatus(t, m.ManagedVolumeStatus(), 503) + if _, live := m.Sandbox(sb.ID); !live { + t.Fatal("ambiguous deletion discarded the claim") + } + if _, present, err := m.eng.Inspect(t.Context(), sb.VMName); err != nil || !present { + t.Fatalf("ambiguous deletion tore down the VM: %t %v", present, err) + } + if _, held := m.managedHold(sb.ID); !held { + t.Fatal("ambiguous deletion discharged the hold") + } +} + +func TestManagedStoppedZeroPIDRequiresOriginalProcessGone(t *testing.T) { + for _, gone := range []bool{false, true} { + t.Run(strconv.FormatBool(gone), func(t *testing.T) { + cfg := managedConfig(t) + e := newManagedEngine() + m := openManagedTest(t, cfg, e) + createManagedTest(t, m, "data") + sb := foundationClaim(t, m, "data") + h, _ := m.managedHold(sb.ID) + m.Close() + e.mu.Lock() + e.stopped[sb.VMName] = true + e.pids[sb.VMName] = 0 + e.mu.Unlock() + m = openManagedTest(t, cfg, e) + m.managed.process = func(int) (processIdentity, error) { + if gone { + return processIdentity{}, os.ErrNotExist + } + return h.Process, nil + } + if err := m.Reconcile(t.Context()); err != nil { + t.Fatal(err) + } + _, held := m.managedHold(sb.ID) + if held == gone { + t.Fatalf("original process gone=%t but hold retained=%t", gone, held) + } + if !gone { + requireVolumeStatus(t, m.ManagedVolumeStatus(), 503) + } + }) + } +} + +func TestManagedHealthyDurableClaimDefeatsRetirement(t *testing.T) { + cfg := managedConfig(t) + e := newManagedEngine() + m := openManagedTest(t, cfg, e) + createManagedTest(t, m, "data") + sb := foundationClaim(t, m, "data") + if err := m.retireManaged(sb.ID); err != nil { + t.Fatal(err) + } + m.Close() + m = openManagedTest(t, cfg, e) + if err := m.Reconcile(t.Context()); err != nil { + t.Fatal(err) + } + h, held := m.managedHold(sb.ID) + if _, live := m.Sandbox(sb.ID); !live || !held || h.State != holdClaimed { + t.Fatalf("durable healthy owner was not restored: live=%t hold=%+v", live, h) + } + if _, present, err := e.Inspect(t.Context(), sb.VMName); err != nil || !present { + t.Fatalf("durable owner's VM was removed: %t %v", present, err) + } +} + +func TestManagedFailedUnboundCancellationHealsWithoutRestart(t *testing.T) { + m := openManagedTest(t, managedConfig(t), newManagedEngine()) + createManagedTest(t, m, "data") + specs, reserved, err := m.admitOptions(t.Context(), testKey, managedOptions("data")) + if err != nil { + t.Fatal(err) + } + m.managed.commitFile = func(string, []byte) error { return syscall.ENOSPC } + m.releaseUnbound(specs, reserved) + if volumeHoldersOf(m, "data").writers != 1 { + t.Fatal("failed discharge released the charge before durable deletion") + } + m.managed.commitFile = writeVolumeSnapshot + m.reapOnce(t.Context()) + waitFor(t, func() bool { return volumeHoldersOf(m, "data") == (volumeHolders{}) }) + if _, err := m.ClaimProvision(t.Context(), testKey, managedOptions("data")); err != nil { + t.Fatal(err) + } +} + +func TestManagedBlockedRetryAllowsOrdinaryExpiry(t *testing.T) { + m := openManagedTest(t, managedConfig(t), newManagedEngine()) + createManagedTest(t, m, "data") + specs, reserved, err := m.admitOptions(t.Context(), testKey, managedOptions("data")) + if err != nil { + t.Fatal(err) + } + ordinary := foundationClaim(t, m) + m.managed.commitFile = func(string, []byte) error { return syscall.ENOSPC } + m.releaseUnbound(specs, reserved) + entered, resume := make(chan struct{}), make(chan struct{}) + var once sync.Once + m.managed.commitFile = func(path string, raw []byte) error { + once.Do(func() { close(entered); <-resume }) + return writeVolumeSnapshot(path, raw) + } + m.mu.Lock() + ordinary.Deadline = time.Now().Add(-time.Minute) + m.mu.Unlock() + done := make(chan struct{}) + go func() { m.reapOnce(t.Context()); close(done) }() + <-entered + select { + case <-done: + case <-time.After(time.Second): + close(resume) + <-done + t.Fatal("managed retry blocked ordinary expiry") + } + if _, live := m.Sandbox(ordinary.ID); live { + t.Error("ordinary expired claim remained live during managed retry") + } + close(resume) + waitFor(t, func() bool { return !m.managed.retrying.Load() }) +} + +func TestManagedMissingBootMetadataCannotProveProcessGone(t *testing.T) { + proc := t.TempDir() + r := &volumeRegistry{process: func(pid int) (processIdentity, error) { return readProcessIdentityAt(proc, pid) }} + _, err := r.process(123) + if !errors.Is(err, os.ErrNotExist) { + t.Fatalf("boot metadata error lost its underlying cause: %v", err) + } + if r.processGone(processIdentity{BootID: "boot", PID: 123, Start: 1}) { + t.Fatal("missing boot metadata was treated as an absent VMM") + } +} diff --git a/sandboxd/pool/managed_grow.go b/sandboxd/pool/managed_grow.go index 527f2ae1..6f3daf60 100644 --- a/sandboxd/pool/managed_grow.go +++ b/sandboxd/pool/managed_grow.go @@ -56,7 +56,7 @@ func growRecovery() error { func (r *volumeRegistry) growthOwnsHold(id string) bool { for _, v := range r.data.Volumes { - if v.Growth != nil && (v.Growth.fenced() || r.fault != nil) && v.Growth.SandboxID == id { + if v.Growth != nil && (v.Growth.fenced() || r.fault != nil || r.writeFault != nil) && v.Growth.SandboxID == id { return true } } @@ -79,7 +79,7 @@ func (m *Manager) growthOwnsVM(name string) bool { m.managed.mu.Lock() defer m.managed.mu.Unlock() for _, v := range m.managed.data.Volumes { - if v.Growth != nil && (v.Growth.fenced() || m.managed.fault != nil) && v.Growth.VMName == name { + if v.Growth != nil && (v.Growth.fenced() || m.managed.fault != nil || m.managed.writeFault != nil) && v.Growth.VMName == name { return true } } @@ -227,9 +227,6 @@ func (m *Manager) growthHolderLocked(sb *types.Sandbox, v managedVolume) (volume if !ok || m.claimed[sb.ID] != sb || h.State != holdClaimed || h.Sleep != nil || !sleepMatchesClaim(h, sb) || sb.Failed != "" || sb.HibernateSnap != "" || sb.PendingSnap != "" || m.volumeRuns[sb] != nil || outbound || removing || h.Refs[0].ID != v.ID || !h.Refs[0].Volume.RW() || h.Refs[0].Volume.Mount == "" || len(r.holders(v.ID)) != 1 || r.growthOwnsHold(sb.ID) { return h, growBusy() } - if r.verify(v) != nil { - return h, managedStorageError() - } return h, nil } @@ -240,6 +237,9 @@ func (r *volumeRegistry) growthBindingMatches(v managedVolume, h volumeHold, rec } func (m *Manager) admitGrowth(ctx context.Context, sb *types.Sandbox, v managedVolume, req types.VolumeGrow, ge volumeGrowEngine) error { + if m.managed.verify(v) != nil { + return managedStorageError() + } r := m.managed m.mu.Lock() r.mu.Lock() @@ -281,33 +281,44 @@ func (m *Manager) commitGrowthIntent(sb *types.Sandbox, v managedVolume, next ma return err } r := m.managed + if r.verify(v) != nil { + return managedStorageError() + } + r.mu.Lock() + if r.data.Version != 3 { + r.data.Version = 3 + if err := r.commitLocked(); err != nil { + r.mu.Unlock() + return err + } + } + r.mu.Unlock() m.mu.Lock() - defer m.mu.Unlock() r.mu.Lock() - defer r.mu.Unlock() current, err := r.growVolume(v.Name, string(v.ID), v.Tenant, next.Request) - if err != nil { - return err - } - if current.Growth != nil && current.Growth.Request == next.Request { + if err == nil && current.Growth != nil && current.Growth.Request == next.Request { + r.mu.Unlock() + m.mu.Unlock() return nil } - if _, err := m.growthHolderLocked(sb, current); err != nil { - return err + if err == nil { + _, err = m.growthHolderLocked(sb, current) } - if current.Size != v.Size || current.Backing != v.Backing { - return growRecovery() + if err == nil && (current.Size != v.Size || current.Backing != v.Backing) { + err = growRecovery() } - if r.data.Version != 3 { - r.data.Version = 3 - if err := r.commit(); err != nil { - return err - } + if err != nil { + r.mu.Unlock() + m.mu.Unlock() + return err } current.State = volumeGrowing current.Growth = &next r.data.Volumes[v.Name] = current - return r.commit() + m.mu.Unlock() + err = r.commitLocked() + r.mu.Unlock() + return err } func (g managedGrowth) canDispatch(i engine.DiskInspection, err error) bool { @@ -433,6 +444,14 @@ func (m *Manager) observeGrowth(ctx context.Context, v managedVolume, g managedG func (m *Manager) saveGrowth(v managedVolume, g managedGrowth, state, outcome string, guestSize int64) error { r := m.managed + var actual backingIdentity + if state == growthCompleted { + var err error + actual, err = inspectBacking(r.path(v.ID)) + if err != nil || actual.Device != v.Backing.Device || actual.Inode != v.Backing.Inode || actual.Size != g.Runtime.TargetBytes { + return managedStorageError() + } + } r.mu.Lock() defer r.mu.Unlock() current := r.data.Volumes[v.Name] @@ -445,10 +464,6 @@ func (m *Manager) saveGrowth(v managedVolume, g managedGrowth, state, outcome st if !current.Growth.fenced() && current.CapacityGeneration != g.Request.ExpectedGeneration { return managedStorageError() } - actual, err := inspectBacking(r.path(v.ID)) - if err != nil || actual.Device != v.Backing.Device || actual.Inode != v.Backing.Inode || actual.Size != g.Runtime.TargetBytes { - return managedStorageError() - } current.Size, current.Backing, current.State = actual.Size, actual, volumeReady current.CapacityGeneration++ for id, h := range r.data.Holds { @@ -464,7 +479,7 @@ func (m *Manager) saveGrowth(v managedVolume, g managedGrowth, state, outcome st current.State = volumeRecovery } r.data.Volumes[v.Name] = current - return r.commit() + return r.commitLocked() } func (r *volumeRegistry) validateGrowth(v managedVolume) error { diff --git a/sandboxd/pool/managed_grow_recovery_test.go b/sandboxd/pool/managed_grow_recovery_test.go index 5a6769e4..4d7264e5 100644 --- a/sandboxd/pool/managed_grow_recovery_test.go +++ b/sandboxd/pool/managed_grow_recovery_test.go @@ -24,12 +24,7 @@ func TestGrowthMissingVMRecordPreservesDurableOwner(t *testing.T) { e.mu.Unlock() e.processLives.Store(true) m = openGrowthTest(t, cfg, e) - if err := m.Reconcile(t.Context()); err == nil { - t.Error("reconcile must fail closed before publishing or saving claims") - } - if _, found := m.Sandbox(sb.ID); found { - t.Fatal("missing VM inventory published owner") - } + requireManagedRecoveryFence(t, m, sb.ID) claims, err := m.store.load() if err != nil { t.Fatal(err) @@ -43,11 +38,7 @@ func TestGrowthMissingVMRecordPreservesDurableOwner(t *testing.T) { m.Close() m = openGrowthTest(t, cfg, e) - if err := m.Reconcile(t.Context()); err == nil { - t.Fatal("missing inventory must keep restart closed") - } else { - t.Log("missing inventory remains fail closed without deleting the claim", err) - } + requireManagedRecoveryFence(t, m, sb.ID) } func TestGrowthChangedVMRecordCannotPublishReplacement(t *testing.T) { @@ -74,12 +65,7 @@ func TestGrowthChangedVMRecordCannotPublishReplacement(t *testing.T) { } return e.identity(pid) } - if err := m.Reconcile(t.Context()); err == nil { - t.Error("reconcile must fail closed before publishing or saving claims") - } - if _, found := m.Sandbox(sb.ID); found { - t.Error("replacement generation published as original claim") - } + requireManagedRecoveryFence(t, m, sb.ID) claims, err := m.store.load() if err != nil { t.Fatal(err) @@ -210,12 +196,7 @@ func TestGrowthRecoveryRequiresOriginalInventoryAndProcess(t *testing.T) { case "unreadable process": e.processUnknown.Store(true) } - if err = m.Reconcile(t.Context()); err == nil { - t.Fatal("mismatched owner was adopted") - } - if _, found := m.Sandbox(sb.ID); found { - t.Fatal("mismatched guest published") - } + requireManagedRecoveryFence(t, m, sb.ID) after, readErr := os.ReadFile(m.store.path) if readErr != nil || string(after) != string(original) { t.Fatalf("owner journal changed: %v", readErr) @@ -396,12 +377,12 @@ func TestGrowthMissingInventoryCannotCompleteBeforeOwnerValidation(t *testing.T) e.lag = false m = openGrowthTest(t, cfg, e) m.eng = &growthInventoryEngine{growthTestEngine: e, omit: true, change: func(*types.VMRecord) {}} - if err = m.Reconcile(t.Context()); err == nil { - t.Error("missing inventory allowed completion and adoption") - } - got, getErr := m.GetVolume("data", "t1") - if getErr != nil || got.CapacityGeneration != 0 || got.SizeBytes != 1<<30 || !m.growthOwnsHold(sb.ID) { - t.Fatalf("missing owner escaped fence: %+v %v", got, getErr) + requireManagedRecoveryFence(t, m, sb.ID) + _, getErr := m.GetVolume("data", "t1") + requireVolumeStatus(t, getErr, 503) + got := m.managed.data.Volumes["data"] + if got.CapacityGeneration != 0 || got.Size != 1<<30 || !m.growthOwnsHold(sb.ID) { + t.Fatalf("missing owner escaped fence: %+v", got) } after, readErr := os.ReadFile(m.store.path) if readErr != nil || string(after) != string(original) { @@ -421,12 +402,12 @@ func TestGrowthMissingInventoryPreservesUndispatchedIntent(t *testing.T) { m.Close() m = openGrowthTest(t, cfg, e) m.eng = &growthInventoryEngine{growthTestEngine: e, omit: true, change: func(*types.VMRecord) {}} - if err := m.Reconcile(t.Context()); err == nil { - t.Fatal("missing inventory must fail closed") - } - pending, err := m.GetVolume("data", "t1") - if err != nil || pending.Growth.State != "pending" || pending.Growth.RuntimeOutcome != "unobserved" || e.calls != 0 { - t.Fatalf("undispatched intent changed: %+v %v", pending, err) + requireManagedRecoveryFence(t, m, sb.ID) + _, err := m.GetVolume("data", "t1") + requireVolumeStatus(t, err, 503) + pending := m.managed.data.Volumes["data"] + if pending.Growth.State != "pending" || pending.Growth.Outcome != "unobserved" || e.calls != 0 { + t.Fatalf("undispatched intent changed: %+v", pending) } m.Close() m = openGrowthTest(t, cfg, e) @@ -438,3 +419,15 @@ func TestGrowthMissingInventoryPreservesUndispatchedIntent(t *testing.T) { t.Fatalf("recovered inventory cannot resume original intent: %+v %v", got, err) } } + +func requireManagedRecoveryFence(t *testing.T, m *Manager, id string) { + t.Helper() + if err := m.Reconcile(t.Context()); err != nil { + t.Fatal(err) + } + requireVolumeStatus(t, m.ManagedVolumeStatus(), 503) + sb, found := m.Sandbox(id) + if !found || sb.Failed != managedRecoveryFailure { + t.Fatalf("uncertain managed claim must remain visible and unavailable: %+v %t", sb, found) + } +} diff --git a/sandboxd/pool/managed_grow_test.go b/sandboxd/pool/managed_grow_test.go index afdb1192..8f0c0c54 100644 --- a/sandboxd/pool/managed_grow_test.go +++ b/sandboxd/pool/managed_grow_test.go @@ -560,9 +560,11 @@ func TestManagedGrowthMigratesVersionTwoBeforeIntentAndKeepsSleepValid(t *testin m, e, sb, _, v := newGrowthTest(t) r := m.managed r.data.Version = 2 - if err := r.commit(); err != nil { + r.mu.Lock() + if err := r.commitLocked(); err != nil { t.Fatal(err) } + r.mu.Unlock() write := r.commitFile var snapshots []volumeRegistryData r.commitFile = func(path string, raw []byte) error { @@ -584,9 +586,11 @@ func TestManagedGrowthMigratesVersionTwoBeforeIntentAndKeepsSleepValid(t *testin } sleeping, _, sb, _ := newSleepTest(t) sleeping.managed.data.Version = 3 - if err := sleeping.managed.commit(); err != nil { + sleeping.managed.mu.Lock() + if err := sleeping.managed.commitLocked(); err != nil { t.Fatal(err) } + sleeping.managed.mu.Unlock() requireSleep(t, sleeping, sb) if err := sleeping.managed.validate(); err != nil { t.Fatal("version three rejected existing sleep", err) diff --git a/sandboxd/pool/managed_recovery.go b/sandboxd/pool/managed_recovery.go index 0622791d..e7c93c98 100644 --- a/sandboxd/pool/managed_recovery.go +++ b/sandboxd/pool/managed_recovery.go @@ -2,9 +2,7 @@ package pool import ( "context" - "errors" "maps" - "os" "strings" "github.com/cocoonstack/sandbox/sandboxd/engine" @@ -51,7 +49,7 @@ func (m *Manager) recoverManagedOrphans(ctx context.Context, claims map[string]* continue } identity, err := r.process(rec.PID) - if (rec.PID > 0 && errors.Is(err, os.ErrNotExist)) || (rec.PID == 0 && rec.State == "stopped") { + if (rec.PID > 0 && processMissing(err)) || (rec.PID == 0 && rec.State == "stopped") { _ = m.eng.Remove(context.WithoutCancel(ctx), name) if !m.confirmGone(ctx, name) { return managedStorageError() @@ -69,7 +67,7 @@ func (m *Manager) recoverManagedOrphans(ctx context.Context, claims map[string]* } r.mu.Lock() r.data.Orphans = maps.Clone(orphans) - err := r.commit() + err := r.commitLocked() r.mu.Unlock() if err != nil { return err @@ -84,7 +82,7 @@ func (m *Manager) recoverManagedOrphans(ctx context.Context, claims map[string]* } r.mu.Lock() delete(r.data.Orphans, name) - err := r.commit() + err := r.commitLocked() r.mu.Unlock() if err != nil { return err diff --git a/sandboxd/pool/managed_recovery_regression_test.go b/sandboxd/pool/managed_recovery_regression_test.go index 8bc9d66d..cc065416 100644 --- a/sandboxd/pool/managed_recovery_regression_test.go +++ b/sandboxd/pool/managed_recovery_regression_test.go @@ -147,7 +147,10 @@ func TestManagedStartupCleansDeadOrphansAndRefusesUnknownProcess(t *testing.T) { t.Fatalf("dead orphan survived cleanup: %v %v", present, err) } } else { - requireVolumeStatus(t, reconcileErr, 503) + if reconcileErr != nil { + t.Fatal(reconcileErr) + } + requireVolumeStatus(t, m.ManagedVolumeStatus(), 503) } if tc.processError == os.ErrPermission || (!tc.stopped && tc.pid == 0) { e.mu.Lock() diff --git a/sandboxd/pool/managed_sleep_boundaries_test.go b/sandboxd/pool/managed_sleep_boundaries_test.go index 1969a2d8..08a64bf9 100644 --- a/sandboxd/pool/managed_sleep_boundaries_test.go +++ b/sandboxd/pool/managed_sleep_boundaries_test.go @@ -4,6 +4,7 @@ import ( "context" "errors" "net" + "os" "path/filepath" "slices" "testing" @@ -234,12 +235,15 @@ func TestManagedSleepExpiryHonorsRenewalWhileTransitionBusy(t *testing.T) { func TestManagedSleepReleasePersistenceFailureKeepsGuestAndHold(t *testing.T) { m, e, sb, _ := newSleepTest(t) before := slices.Clone(e.volumeOps) - path := m.store.path - m.store.path = filepath.Join(t.TempDir(), "missing", "claims.json") + if err := os.Mkdir(m.store.path+".tmp", 0o700); err != nil { + t.Fatal(err) + } if err := m.Release(t.Context(), sb.ID, Cred{Token: sb.Token}); err == nil { t.Fatal("failed durable deletion reported success") } - m.store.path = path + if err := os.Remove(m.store.path + ".tmp"); err != nil { + t.Fatal(err) + } if _, ok := m.Sandbox(sb.ID); !ok || !slices.Equal(before, e.volumeOps) { t.Fatal("failed durable deletion dropped claim or touched guest") } @@ -251,9 +255,10 @@ func TestManagedSleepReleasePersistenceFailureKeepsGuestAndHold(t *testing.T) { if release != nil { release() } - if err == nil || sock != "" { - t.Fatalf("retiring claim exposed guest with feature disabled: socket=%q err=%v", sock, err) + if err != nil || sock == "" { + t.Fatalf("failed pre-rename deletion must leave the active guest available: socket=%q err=%v", sock, err) } + waitFor(t, func() bool { return !m.store.recommitting.Load() && m.store.synced() }) if err := m.Release(t.Context(), sb.ID, Cred{Token: sb.Token}); err != nil { t.Fatal(err) } diff --git a/sandboxd/pool/managed_sleep_recovery.go b/sandboxd/pool/managed_sleep_recovery.go index aa77506a..f5c21bd6 100644 --- a/sandboxd/pool/managed_sleep_recovery.go +++ b/sandboxd/pool/managed_sleep_recovery.go @@ -32,21 +32,26 @@ func (m *Manager) releaseManagedSleepLocked(ctx context.Context, sb *types.Sandb m.mu.Unlock() return nil } - if err := m.retireManaged(sb.ID); err != nil { + if m.growthOwnsHold(sb.ID) { m.mu.Unlock() - return err + return growBusy() } + previousHold, _ := m.managedHold(sb.ID) m.dropClaimLocked(sb) js := m.store.del(sb.ID) m.mu.Unlock() - if err := m.store.commitDurable(js); err != nil { + err := m.store.commitDurable(js) + if err != nil { + m.restoreManagedClaim(previousHold, err) m.mu.Lock() m.addClaimLocked(sb) - m.store.set(sb) + rb := m.store.set(sb) m.mu.Unlock() + m.store.recommit(ctx, rb) m.closeManagedStreams(sb) return err } + _ = m.retireManaged(sb.ID) ctx = context.WithoutCancel(ctx) h, _ := m.managedHold(sb.ID) td := volumeTeardown{managedID: sb.ID} @@ -116,7 +121,7 @@ func (m *Manager) observeManagedRemoval(ctx context.Context, h volumeHold) (volu next.Restored = p h.Sleep = &next r.data.Holds[h.SandboxID] = h - return h, true, r.commit() + return h, true, r.commitLocked() } func (m *Manager) retryManagedSleepRemoval(ctx context.Context, name string, pending pendingRemoval) bool { diff --git a/sandboxd/pool/managed_sleep_registry.go b/sandboxd/pool/managed_sleep_registry.go index 687696f0..1e7f1bbd 100644 --- a/sandboxd/pool/managed_sleep_registry.go +++ b/sandboxd/pool/managed_sleep_registry.go @@ -67,12 +67,12 @@ func (r *volumeRegistry) commitManagedSleep(h volumeHold, next *managedSleep) er r.mu.Lock() defer r.mu.Unlock() current, ok := r.data.Holds[h.SandboxID] - if r.fault != nil || r.growthOwnsHold(h.SandboxID) || !ok || current.State != holdClaimed || (current.Sleep != nil && (h.Sleep == nil || current.Sleep.Operation != h.Sleep.Operation)) { + if r.fault != nil || r.writeFault != nil || r.growthOwnsHold(h.SandboxID) || !ok || current.State != holdClaimed || (current.Sleep != nil && (h.Sleep == nil || current.Sleep.Operation != h.Sleep.Operation)) { return managedStorageError() } if r.data.Version == 1 { r.data.Version = 2 - if err := r.commit(); err != nil { + if err := r.commitLocked(); err != nil { return err } } @@ -83,7 +83,7 @@ func (r *volumeRegistry) commitManagedSleep(h volumeHold, next *managedSleep) er } } r.data.Holds[h.SandboxID] = h - if err := r.commit(); err != nil { + if err := r.commitLocked(); err != nil { if next == nil { h.Sleep = current.Sleep r.data.Holds[h.SandboxID] = h @@ -101,12 +101,18 @@ func (m *Manager) verifySleepBacking(h volumeHold) error { r := m.managed r.mu.Lock() defer r.mu.Unlock() - if r.fault != nil { + if r.fault != nil || r.writeFault != nil { return managedStorageError() } ref := h.Refs[0] v, ok := r.data.Volumes[ref.Volume.Name] - if !ok || v.ID != ref.ID || v.State != volumeReady || ref.Path != r.path(v.ID) || ref.Backing != v.Backing || (h.Tenant != "" && v.Tenant != h.Tenant) || r.verify(v) != nil { + if !ok || v.ID != ref.ID || v.State != volumeReady || ref.Path != r.path(v.ID) || ref.Backing != v.Backing || (h.Tenant != "" && v.Tenant != h.Tenant) { + return managedStorageError() + } + r.mu.Unlock() + err := r.verify(v) + r.mu.Lock() + if err != nil { return managedStorageError() } return nil diff --git a/sandboxd/pool/managed_sleep_test.go b/sandboxd/pool/managed_sleep_test.go index 62a13f36..130faf28 100644 --- a/sandboxd/pool/managed_sleep_test.go +++ b/sandboxd/pool/managed_sleep_test.go @@ -770,7 +770,11 @@ func TestManagedSleepMissingClaimRetainsOwnership(t *testing.T) { } defer recovered.Close() recovered.managed.process = e.identity - if err = recovered.Reconcile(t.Context()); err == nil { + if err = recovered.Reconcile(t.Context()); err != nil { + t.Fatal(err) + } + requireVolumeStatus(t, recovered.ManagedVolumeStatus(), 503) + if _, exists := recovered.Sandbox(sb.ID); exists { t.Fatal("missing credentials were reconstructed") } if _, held := recovered.managedHold(sb.ID); !held { diff --git a/sandboxd/pool/managed_startup_boundary_test.go b/sandboxd/pool/managed_startup_boundary_test.go index e438e0f6..ca3a2167 100644 --- a/sandboxd/pool/managed_startup_boundary_test.go +++ b/sandboxd/pool/managed_startup_boundary_test.go @@ -40,7 +40,10 @@ func TestManagedStartupCollectsUnclaimedCreatingVMBeforeProcessProof(t *testing. t.Fatalf("proven stale create blocked startup: %v", err) } } else { - requireVolumeStatus(t, err, 503) + if err != nil { + t.Fatal(err) + } + requireVolumeStatus(t, m.ManagedVolumeStatus(), 503) } e.mu.Lock() defer e.mu.Unlock() diff --git a/sandboxd/pool/managed_volume.go b/sandboxd/pool/managed_volume.go index 2011bdb7..d202f717 100644 --- a/sandboxd/pool/managed_volume.go +++ b/sandboxd/pool/managed_volume.go @@ -14,10 +14,12 @@ import ( "github.com/cocoonstack/sandbox/sandboxd/types" ) -// VolumeError carries only caller-safe details. Storage paths and process IDs -// remain in the registry and never become API diagnostics. -const volumeBusyCode = "volume_busy" +const ( + volumeBusyCode = "volume_busy" + volumeBusyMessage = "volume is held by a sandbox" +) +// VolumeError carries caller-safe details without storage paths or process IDs. type VolumeError struct { Status int `json:"-"` Message string `json:"error"` @@ -42,34 +44,26 @@ func managedStorageError() *VolumeError { func managedNotFound() *VolumeError { return volumeError(404, "volume_not_found", "unknown volume") } func (m *Manager) CreateVolume(ctx context.Context, name string, req types.VolumeCreate, tenant string) (types.ManagedVolumeInfo, bool, error) { - if !types.ValidVolumeName(name) || req.SizeGiB < 1 || req.SizeGiB > 1024 { - return types.ManagedVolumeInfo{}, false, volumeError(400, "invalid_volume", "invalid volume name or size_gib (1 through 1024)") - } - if tenant != "" && req.Tenant != "" && tenant != req.Tenant { - return types.ManagedVolumeInfo{}, false, volumeError(403, "volume_owner_forbidden", "tenant cannot select another volume owner") - } - owner := req.Tenant - if tenant != "" { - owner = tenant - } - if !types.NameRe.MatchString(owner) { - return types.ManagedVolumeInfo{}, false, volumeError(400, "invalid_volume_owner", "an existing tenant owner is required") - } - if _, ok, err := m.tenants.Lookup(ctx, owner); err != nil { - return types.ManagedVolumeInfo{}, false, managedStorageError() - } else if !ok { - return types.ManagedVolumeInfo{}, false, volumeError(400, "invalid_volume_owner", "an existing tenant owner is required") + owner, err := m.volumeCreateOwner(ctx, name, req, tenant) + if err != nil { + return types.ManagedVolumeInfo{}, false, err } r := m.managed if r == nil { return types.ManagedVolumeInfo{}, false, managedStorageError() } + m.mu.Lock() + admission := m.volumeAdmission[name] + m.mu.Unlock() + r.fileMu.Lock() + defer r.fileMu.Unlock() r.mu.Lock() defer r.mu.Unlock() - if r.fault != nil { + if r.closed || r.fault != nil || r.writeFault != nil { return types.ManagedVolumeInfo{}, false, managedStorageError() } - if _, ok := m.volumes[name]; ok { + _, managedName := r.data.Volumes[name] + if _, ok := m.volumes[name]; ok || r.retainedCatalogName(name) || !managedName && (admission.readers > 0 || admission.writers > 0) { return types.ManagedVolumeInfo{}, false, volumeError(409, "volume_name_conflict", "volume name is unavailable") } v, exists := r.data.Volumes[name] @@ -83,21 +77,30 @@ func (m *Manager) CreateVolume(ctx context.Context, name string, req types.Volum if !exists { v = managedVolume{ID: volumeID("vol_" + randHex(16)), Name: name, Tenant: owner, Node: r.node, Size: size, State: volumeCreating, CreatedAt: time.Now().UTC()} r.data.Volumes[name] = v - if err := r.commit(); err != nil { + if err := r.commitLocked(); err != nil { return types.ManagedVolumeInfo{}, false, err } } switch v.State { case volumeCreating: - if err := r.completeCreate(v); err != nil { + r.mu.Unlock() + err := r.completeCreate(v) + r.mu.Lock() + if err != nil { return types.ManagedVolumeInfo{}, false, err } v = r.data.Volumes[name] case volumeGrowing: return r.info(v), false, nil case volumeReady: - if err := r.verify(v); err != nil { - return types.ManagedVolumeInfo{}, false, r.markRecovery(v) + r.mu.Unlock() + err := r.verify(v) + r.mu.Lock() + if err != nil { + if errors.Is(err, errBackingMismatch) || errors.Is(err, os.ErrNotExist) { + return types.ManagedVolumeInfo{}, false, r.markRecovery(v) + } + return types.ManagedVolumeInfo{}, false, managedStorageError() } case volumeDeleting: return types.ManagedVolumeInfo{}, false, volumeError(409, "volume_deleting", "volume deletion is pending") @@ -133,8 +136,10 @@ func (r *volumeRegistry) completeCreate(v managedVolume) error { raw := st.Sys().(*syscall.Stat_t) v.Backing = backingIdentity{Device: backingDeviceID(raw.Dev), Inode: raw.Ino, Size: st.Size()} v.State = volumeReady + r.mu.Lock() + defer r.mu.Unlock() r.data.Volumes[v.Name] = v - return r.commit() + return r.commitLocked() } func (r *volumeRegistry) markRecovery(v managedVolume) error { @@ -143,7 +148,7 @@ func (r *volumeRegistry) markRecovery(v managedVolume) error { } v.State = volumeRecovery r.data.Volumes[v.Name] = v - if err := r.commit(); err != nil { + if err := r.commitLocked(); err != nil { return err } return managedStorageError() @@ -156,26 +161,32 @@ func (m *Manager) GetVolume(name, tenant string) (types.ManagedVolumeInfo, error } r.mu.Lock() defer r.mu.Unlock() - if r.fault != nil { + if r.closed || r.fault != nil || r.writeFault != nil { return types.ManagedVolumeInfo{}, managedStorageError() } v, ok := r.data.Volumes[name] if !ok || (tenant != "" && v.Tenant != tenant) { return types.ManagedVolumeInfo{}, managedNotFound() } + info := r.info(v) + r.mu.Unlock() + defer r.mu.Lock() if v.State == volumeGrowing { actual, err := inspectBacking(r.path(v.ID)) if err != nil || r.verifyDirectories() != nil || actual.Device != v.Backing.Device || actual.Inode != v.Backing.Inode || (actual.Size != v.Size && actual.Size != v.Growth.Runtime.TargetBytes) { return types.ManagedVolumeInfo{}, managedStorageError() } } - if v.State == volumeReady && r.verify(v) != nil { - if err := r.markRecovery(v); r.fault != nil { - return types.ManagedVolumeInfo{}, err + if v.State == volumeReady { + if err := r.verify(v); err != nil { + if !errors.Is(err, errBackingMismatch) && !errors.Is(err, os.ErrNotExist) { + return types.ManagedVolumeInfo{}, managedStorageError() + } + info.State = string(volumeRecovery) + info.Available = false } - v.State = volumeRecovery } - return r.info(v), nil + return info, nil } func (m *Manager) DeleteVolume(ctx context.Context, name, expected, tenant string) error { @@ -186,9 +197,11 @@ func (m *Manager) DeleteVolume(ctx context.Context, name, expected, tenant strin if r == nil { return managedNotFound() } + r.fileMu.Lock() + defer r.fileMu.Unlock() r.mu.Lock() defer r.mu.Unlock() - if r.fault != nil { + if r.closed || r.fault != nil || r.writeFault != nil { return managedStorageError() } if expected == "" { @@ -209,13 +222,19 @@ func (m *Manager) DeleteVolume(ctx context.Context, name, expected, tenant strin return growBusy() } if holders := r.holders(v.ID); len(holders) > 0 { - return &VolumeError{Status: 409, Code: volumeBusyCode, Message: "volume is held by a sandbox", Volume: name, Holders: holders} - } - if err := r.verifyDirectories(); err != nil { - return managedStorageError() + return &VolumeError{Status: 409, Code: volumeBusyCode, Message: volumeBusyMessage, Volume: name, Holders: holders} } + r.mu.Unlock() + dirErr := r.verifyDirectories() path := r.path(id) actual, err := inspectBacking(path) + r.mu.Lock() + if dirErr != nil { + return managedStorageError() + } + if holders := r.holders(v.ID); len(holders) > 0 { + return &VolumeError{Status: 409, Code: volumeBusyCode, Message: volumeBusyMessage, Volume: name, Holders: holders} + } missing := errors.Is(err, os.ErrNotExist) if missing && v.State != volumeCreating && v.State != volumeDeleting { return r.markRecovery(v) @@ -226,24 +245,19 @@ func (m *Manager) DeleteVolume(ctx context.Context, name, expected, tenant strin if v.State != volumeDeleting { v.State = volumeDeleting r.data.Volumes[name] = v - if err := r.commit(); err != nil { + if err := r.commitLocked(); err != nil { return err } } - if !missing { - if err := os.Remove(path); err != nil { - return managedStorageError() - } - } - if err := syncDir(r.dir + "/disks"); err != nil { - return managedStorageError() - } - if err := clearVolumeDirty(path); err != nil { + r.mu.Unlock() + removeErr := removeManagedBacking(path, r.dir+"/disks", missing) + r.mu.Lock() + if removeErr != nil { return managedStorageError() } r.data.Deleted[id] = volumeTombstone{ID: id, Name: name, Tenant: v.Tenant} delete(r.data.Volumes, name) - return r.commit() + return r.commitLocked() } func (r *volumeRegistry) holders(id volumeID) []types.VolumeHolder { @@ -279,7 +293,7 @@ func (m *Manager) ManagedVolumeStatus() error { } m.managed.mu.Lock() defer m.managed.mu.Unlock() - if m.managed.fault != nil { + if m.managed.fault != nil || m.managed.writeFault != nil { return managedStorageError() } return nil @@ -291,19 +305,25 @@ func (m *Manager) managedInfos(tenant string) []types.VolumeInfo { return nil } r.mu.Lock() - defer r.mu.Unlock() - if r.fault != nil { + if r.closed || r.fault != nil || r.writeFault != nil { + r.mu.Unlock() return nil } - infos := []types.VolumeInfo{} + var volumes []managedVolume for _, v := range r.data.Volumes { if tenant == "" || tenant == v.Tenant { - info := r.info(v) - if v.State == volumeReady && r.verify(v) != nil { - info.State = string(volumeRecovery) - info.Available = false - } - infos = append(infos, info.VolumeInfo) + volumes = append(volumes, v) + } + } + infos := make([]types.VolumeInfo, len(volumes)) + for i, v := range volumes { + infos[i] = r.info(v).VolumeInfo + } + r.mu.Unlock() + for i, v := range volumes { + if v.State == volumeReady && r.verify(v) != nil { + infos[i].State = string(volumeRecovery) + infos[i].Available = false } } return infos @@ -316,7 +336,7 @@ func (m *Manager) lookupManaged(name string) (catalogVolume, bool, error) { } r.mu.Lock() defer r.mu.Unlock() - if r.fault != nil { + if r.closed || r.fault != nil || r.writeFault != nil { return catalogVolume{}, false, managedStorageError() } v, ok := r.data.Volumes[name] @@ -327,8 +347,11 @@ func (m *Manager) lookupManaged(name string) (catalogVolume, bool, error) { if v.State != volumeReady { return entry, true, managedStorageError() } - if err := r.verify(v); err != nil { - return entry, true, r.markRecovery(v) + r.mu.Unlock() + err := r.verify(v) + r.mu.Lock() + if err != nil { + return entry, true, managedStorageError() } return entry, true, nil } @@ -379,11 +402,14 @@ func (m *Manager) retireManaged(id string) error { return growBusy() } if h.State == holdRetiring { + if r.writeFault != nil { + return r.commitLocked() + } return nil } h.State = holdRetiring r.data.Holds[id] = h - return r.commit() + return r.commitLocked() } func (m *Manager) managedRefPath(id, name string) (string, bool) { @@ -420,15 +446,7 @@ func (m *Manager) managedCatalogAlias(path string) bool { identity := info.Sys().(*syscall.Stat_t) r.mu.Lock() defer r.mu.Unlock() - for _, v := range r.data.Volumes { - if v.Backing.Inode != 0 && identity.Ino == v.Backing.Inode && backingDeviceID(identity.Dev) == v.Backing.Device { - return true - } - if backing, err := os.Lstat(r.path(v.ID)); err == nil && os.SameFile(info, backing) { - return true - } - } - return false + return r.aliases[backingIdentity{Device: backingDeviceID(identity.Dev), Inode: identity.Ino}] } func growthBackingMatches(v managedVolume, actual backingIdentity) bool { @@ -437,3 +455,48 @@ func growthBackingMatches(v managedVolume, actual backingIdentity) bool { } return v.Growth.settled() && actual.Device == v.Backing.Device && actual.Inode == v.Backing.Inode && actual.Size == v.Growth.Runtime.TargetBytes } + +func (r *volumeRegistry) retainedCatalogName(name string) bool { + for _, h := range r.data.Holds { + for _, ref := range h.Refs { + if ref.ID == "" && ref.Volume.Name == name { + return true + } + } + } + return false +} + +func removeManagedBacking(path, dir string, missing bool) error { + if !missing { + if err := os.Remove(path); err != nil { + return err + } + } + if err := syncDir(dir); err != nil { + return err + } + return clearVolumeDirty(path) +} + +func (m *Manager) volumeCreateOwner(ctx context.Context, name string, req types.VolumeCreate, tenant string) (string, error) { + if !types.ValidVolumeName(name) || req.SizeGiB < 1 || req.SizeGiB > 1024 { + return "", volumeError(400, "invalid_volume", "invalid volume name or size_gib (1 through 1024)") + } + if tenant != "" && req.Tenant != "" && tenant != req.Tenant { + return "", volumeError(403, "volume_owner_forbidden", "tenant cannot select another volume owner") + } + owner := req.Tenant + if tenant != "" { + owner = tenant + } + if !types.NameRe.MatchString(owner) { + return "", volumeError(400, "invalid_volume_owner", "an existing tenant owner is required") + } + if _, ok, err := m.tenants.Lookup(ctx, owner); err != nil { + return "", ErrTenantStoreDown + } else if !ok { + return "", volumeError(400, "invalid_volume_owner", "an existing tenant owner is required") + } + return owner, nil +} diff --git a/sandboxd/pool/managed_volume_test.go b/sandboxd/pool/managed_volume_test.go index 04c6715a..c43f31b4 100644 --- a/sandboxd/pool/managed_volume_test.go +++ b/sandboxd/pool/managed_volume_test.go @@ -262,7 +262,7 @@ func TestManagedConcurrentCreateAndWriterAdmission(t *testing.T) { } } -func TestManagedRetirementSurvivesRestartAndStaleClaim(t *testing.T) { +func TestManagedRetirementWithoutDurableClaimSurvivesRestart(t *testing.T) { for _, syncClaims := range []bool{false, true} { t.Run(fmt.Sprint(syncClaims), func(t *testing.T) { cfg := managedConfig(t) @@ -275,6 +275,9 @@ func TestManagedRetirementSurvivesRestartAndStaleClaim(t *testing.T) { t.Fatal(claimErr) } e.falseRemove.Store(true) + if err := m.store.commitDurable(m.store.del(sb.ID)); err != nil { + t.Fatal(err) + } if err := m.retireManaged(sb.ID); err != nil { t.Fatal(err) } @@ -530,7 +533,7 @@ func TestManagedCorruptAndUnsafeStorage(t *testing.T) { t.Fatal(err) } case "permissions": - if err := os.Chmod(path, 0o000); err != nil { + if err := os.Chmod(path, 0o644); err != nil { t.Fatal(err) } } diff --git a/sandboxd/pool/reconcile.go b/sandboxd/pool/reconcile.go index bf5b0e23..d7fe0bf9 100644 --- a/sandboxd/pool/reconcile.go +++ b/sandboxd/pool/reconcile.go @@ -4,6 +4,7 @@ import ( "context" "encoding/json/v2" "fmt" + "maps" "os" "path/filepath" "slices" @@ -25,16 +26,17 @@ func (m *Manager) Reconcile(ctx context.Context) error { return err } vms, err := m.eng.List(ctx) - if err != nil { + inventoryErr := err + if err != nil && m.managed == nil { return fmt.Errorf("list vms: %w", err) } + clearManagedRecoveryFailures(claims) + originalClaims := maps.Clone(claims) live := make(map[string]types.VMRecord, len(vms)) for _, vm := range vms { live[vm.Config.Name] = vm } - if err = m.recoverGrowth(ctx, claims, live); err != nil { - return err - } + managedErr := inventoryErr fenced := m.fenceArchived(ctx, claims) snaps, snapsErr := m.eng.SnapshotList(ctx) imageIDs := m.imageIDs(ctx) @@ -42,12 +44,16 @@ func (m *Manager) Reconcile(ctx context.Context) error { if m.managed != nil { m.store.reset(claims) } - owned, err := m.recoverManaged(ctx, claims, live) - if err != nil { - return err + owned := map[string]bool{} + if managedErr == nil { + owned, managedErr = m.reconcileManaged(ctx, claims, live) } - if err := m.recoverManagedOrphans(ctx, claims, live, owned); err != nil { - return err + if managedErr != nil { + claims = originalClaims + m.fenceManagedRecovery(ctx, managedErr, claims, live) + for name := range live { + owned[name] = true + } } referenced := m.managedSleepSources() var adopted []*types.Sandbox @@ -57,6 +63,8 @@ func (m *Manager) Reconcile(ctx context.Context) error { rec, ok := live[sb.VMName] h, managed := m.managedHold(sb.ID) switch { + case managedErr != nil: + // Inventory uncertainty must retain the durable claim even without a VM record. case managed && h.Sleep != nil: projectSleepState(sb, h.Sleep) case sb.ArchiveCk != "": @@ -81,19 +89,21 @@ func (m *Manager) Reconcile(ctx context.Context) error { continue } m.addClaimLocked(sb) - if _, managed := m.managedHold(sb.ID); !managed { + if _, managed := m.managedHold(sb.ID); managedErr != nil || !managed { m.adoptVolumes(heldVolumes(sb)) } owned[sb.VMName] = true referenced[sb.HibernateSnap] = true } - saveErr := m.store.save(m.claimed) + js := m.store.reset(m.claimed) + m.markManagedRecoveryFailures() pooled := make(map[string]bool, len(m.pools)) for _, p := range m.pools { m.adoptGolden(p, imageIDs[p.key.Template]) pooled[p.hash] = true } m.mu.Unlock() + saveErr := m.store.commit(js) for _, sb := range adopted { m.recordHibernate(ctx, sb) } @@ -106,8 +116,10 @@ func (m *Manager) Reconcile(ctx context.Context) error { m.retryArchiveDeletes(ctx) removed := m.sweepStaleVMs(ctx, live, owned) - m.sweepOrphanSnapshots(ctx, snaps, snapsErr, referenced) - m.resyncEgress(ctx, live, removed) + if managedErr == nil { + m.sweepOrphanSnapshots(ctx, snaps, snapsErr, referenced) + m.resyncEgress(ctx, live, removed) + } m.warnDeniedClaims(ctx) logger.Infof(ctx, "adopted %d claims, %d VMs live", len(m.claimed), len(live)) return saveErr @@ -363,3 +375,75 @@ func (m *Manager) sweepOrphanSnapshots(ctx context.Context, snaps []string, snap }).Wait() } } + +func (m *Manager) fenceManagedRecovery(ctx context.Context, err error, claims map[string]*types.Sandbox, live map[string]types.VMRecord) { + r := m.managed + r.mu.Lock() + r.fault = err + r.uncertainVMs = make(map[string]bool, len(live)+len(claims)) + for name := range live { + r.uncertainVMs[name] = true + } + for _, sb := range claims { + _, held := r.data.Holds[sb.ID] + if hasVolumes(sb) || held { + r.uncertainVMs[sb.VMName] = true + } else { + delete(r.uncertainVMs, sb.VMName) + } + } + r.mu.Unlock() + m.mu.Lock() + m.volumeAdmission = map[string]volumeHolders{} + m.mu.Unlock() + log.WithFunc("pool.Reconcile").Error(ctx, err, "managed storage fenced; retaining claims and VMs") +} + +func (m *Manager) managedUncertainVM(name string) bool { + if m.managed == nil { + return false + } + m.managed.mu.Lock() + defer m.managed.mu.Unlock() + return m.managed.uncertainVMs[name] +} + +const managedRecoveryFailure = "managed storage recovery required" + +func (m *Manager) managedClaimUncertain(sb *types.Sandbox) bool { + if m.managed == nil || !hasVolumes(sb) { + return false + } + r := m.managed + r.mu.Lock() + defer r.mu.Unlock() + return r.fault != nil || r.uncertainVMs[sb.VMName] +} + +func (m *Manager) reconcileManaged(ctx context.Context, claims map[string]*types.Sandbox, live map[string]types.VMRecord) (map[string]bool, error) { + owned := map[string]bool{} + if err := m.recoverGrowth(ctx, claims, live); err != nil { + return owned, err + } + owned, err := m.recoverManaged(ctx, claims, live) + if err != nil { + return owned, err + } + return owned, m.recoverManagedOrphans(ctx, claims, live, owned) +} + +func clearManagedRecoveryFailures(claims map[string]*types.Sandbox) { + for _, sb := range claims { + if sb.Failed == managedRecoveryFailure { + sb.Failed = "" + } + } +} + +func (m *Manager) markManagedRecoveryFailures() { + for _, sb := range m.claimed { + if sb.Failed == "" && m.managedClaimUncertain(sb) { + sb.Failed = managedRecoveryFailure + } + } +} diff --git a/sandboxd/pool/remove.go b/sandboxd/pool/remove.go index f62c6dbd..ba4dc163 100644 --- a/sandboxd/pool/remove.go +++ b/sandboxd/pool/remove.go @@ -14,7 +14,7 @@ import ( ) func (m *Manager) removeVM(ctx context.Context, name string) bool { - if m.growthOwnsVM(name) { + if m.managedUncertainVM(name) || m.growthOwnsVM(name) { return false } target, allowed := m.growthTeardownTarget(ctx, name) diff --git a/sandboxd/pool/telemetry.go b/sandboxd/pool/telemetry.go index ada7fd5f..d2f5f1c2 100644 --- a/sandboxd/pool/telemetry.go +++ b/sandboxd/pool/telemetry.go @@ -23,16 +23,18 @@ const ( // Counters is a monotonic snapshot for /metrics; gauges come from Info. type Counters struct { - ClaimsWarm uint64 - ClaimsClone uint64 - ClaimsCold uint64 - Wakes uint64 - Hibernates uint64 - Forks uint64 - Checkpoints uint64 - Promotes uint64 - Releases uint64 - Reaps uint64 + ManagedWriteFailures uint64 + ManagedWriteRecoveries uint64 + ClaimsWarm uint64 + ClaimsClone uint64 + ClaimsCold uint64 + Wakes uint64 + Hibernates uint64 + Forks uint64 + Checkpoints uint64 + Promotes uint64 + Releases uint64 + Reaps uint64 Archives uint64 Unarchives uint64 @@ -95,22 +97,29 @@ type egressFrame struct { func (m *Manager) Counters() Counters { c := &m.counters + var managedFailures, managedRecoveries uint64 + if m.managed != nil { + managedFailures = m.managed.writeFailures.Load() + managedRecoveries = m.managed.writeRecoveries.Load() + } return Counters{ - ClaimsWarm: c.claimsWarm.Load(), - ClaimsClone: c.claimsClone.Load(), - ClaimsCold: c.claimsCold.Load(), - Wakes: c.wakes.Load(), - Hibernates: c.hibernates.Load(), - Forks: c.forks.Load(), - Checkpoints: c.checkpoints.Load(), - Promotes: c.promotes.Load(), - Releases: c.releases.Load(), - Reaps: c.reaps.Load(), - Archives: c.archives.Load(), - Unarchives: c.unarchives.Load(), - ArchiveDeletes: c.archiveDeletes.Load(), - ClaimNanos: c.claimNanos.Load(), - WakeNanos: c.wakeNanos.Load(), + ManagedWriteFailures: managedFailures, + ManagedWriteRecoveries: managedRecoveries, + ClaimsWarm: c.claimsWarm.Load(), + ClaimsClone: c.claimsClone.Load(), + ClaimsCold: c.claimsCold.Load(), + Wakes: c.wakes.Load(), + Hibernates: c.hibernates.Load(), + Forks: c.forks.Load(), + Checkpoints: c.checkpoints.Load(), + Promotes: c.promotes.Load(), + Releases: c.releases.Load(), + Reaps: c.reaps.Load(), + Archives: c.archives.Load(), + Unarchives: c.unarchives.Load(), + ArchiveDeletes: c.archiveDeletes.Load(), + ClaimNanos: c.claimNanos.Load(), + WakeNanos: c.wakeNanos.Load(), } } diff --git a/sandboxd/pool/volume.go b/sandboxd/pool/volume.go index de82875e..1ca6673e 100644 --- a/sandboxd/pool/volume.go +++ b/sandboxd/pool/volume.go @@ -35,6 +35,7 @@ type resolvedVolume struct { applied types.Volume managedID volumeID holdID string + backing backingIdentity } // volumeTeardown captures marker paths at quiesce time; a later config can re-point the entry. @@ -249,9 +250,10 @@ func (m *Manager) applyVolume(ctx context.Context, sb *types.Sandbox, volume res if volume.managedID != "" { m.managed.mu.Lock() v, ok := m.managed.data.Volumes[volume.applied.Name] - valid := ok && v.ID == volume.managedID && m.managed.fault == nil && m.managed.verify(v) == nil + h, held := m.managed.data.Holds[volume.holdID] + valid := ok && held && h.State == holdAttaching && h.VMName == sb.VMName && v.ID == volume.managedID && m.managed.fault == nil && m.managed.writeFault == nil m.managed.mu.Unlock() - if !valid { + if !valid || m.managed.verify(v) != nil { return managedStorageError() } } @@ -347,7 +349,7 @@ func (m *Manager) quiesceVolumes(ctx context.Context, sb *types.Sandbox) volumeT // finishVolumeTeardown runs only once the VM is gone, when its image lock is finally released. func (m *Manager) finishVolumeTeardown(ctx context.Context, td volumeTeardown) { if td.managedID != "" { - m.finishManaged(td.managedID, td.clears) + m.finishManaged(td.managedID, td.clears, td.holds) return } if len(td.holds) == 0 { diff --git a/sandboxd/pool/volume_mutation.go b/sandboxd/pool/volume_mutation.go index b47f0a3b..c301007d 100644 --- a/sandboxd/pool/volume_mutation.go +++ b/sandboxd/pool/volume_mutation.go @@ -286,6 +286,13 @@ func (m *Manager) prepareVolumeTarget(ctx context.Context, sb *types.Sandbox, ru } func (m *Manager) stageVolumeTarget(ctx context.Context, sb *types.Sandbox, run *volumeRun, target, additions []types.Volume, resolved []resolvedVolume) (claimSnapshot, bool, error) { + for i := range resolved { + identity, err := catalogBacking(resolved[i].disk.Path) + if err != nil { + return claimSnapshot{}, false, ErrVolumeUnavailable + } + resolved[i].backing = identity + } sb.Transition.Lock() defer sb.Transition.Unlock() m.mu.Lock() @@ -302,8 +309,17 @@ func (m *Manager) stageVolumeTarget(ctx context.Context, sb *types.Sandbox, run return claimSnapshot{}, false, err } run.holds = append(run.holds, additions...) - if err := m.managedLiveAdd(sb.ID, resolved); err != nil { - if m.ManagedVolumeStatus() == nil { + addErr := m.stageManagedLiveAdd(sb.ID, resolved) + _, managed := m.managedHold(sb.ID) + if addErr == nil && managed && len(resolved) > 0 { + m.mu.Unlock() + sb.Transition.Unlock() + addErr = m.commitManaged() + sb.Transition.Lock() + m.mu.Lock() + } + if err := addErr; err != nil { + if m.ownsRunLocked(sb, run) && m.ManagedVolumeStatus() == nil { m.releaseVolumes(additions) run.holds = volumeDifference(run.holds, additions) } @@ -318,7 +334,15 @@ func (m *Manager) stageVolumeTarget(ctx context.Context, sb *types.Sandbox, run return claimSnapshot{}, false, err } if cleanErr != nil { - if err := m.managedLiveRemove(sb.ID, additions); err != nil { + m.mu.Unlock() + sb.Transition.Unlock() + removeErr := m.managedLiveRemove(sb.ID, additions) + sb.Transition.Lock() + m.mu.Lock() + if err := removeErr; err != nil { + return claimSnapshot{}, false, err + } + if err := m.checkVolumeRunLocked(ctx, sb, run); err != nil { return claimSnapshot{}, false, err } m.releaseVolumes(additions) @@ -375,9 +399,15 @@ func (m *Manager) commitVolumeRun(ctx context.Context, sb *types.Sandbox, run *v return err } dropped := volumeDifference(run.holds, heldVolumes(sb)) - if err := m.managedLiveRemove(sb.ID, dropped); err != nil { + m.mu.Unlock() + removeErr := m.managedLiveRemove(sb.ID, dropped) + m.mu.Lock() + if err := removeErr; err != nil { return err } + if !m.ownsRunLocked(sb, run) { + return ErrUnknownSandbox + } m.releaseVolumes(dropped) run.holds = volumeDifference(run.holds, dropped) return nil diff --git a/sandboxd/pool/volume_registry.go b/sandboxd/pool/volume_registry.go index 19ad552a..efd742a2 100644 --- a/sandboxd/pool/volume_registry.go +++ b/sandboxd/pool/volume_registry.go @@ -1,17 +1,23 @@ package pool import ( + "context" "encoding/json/v2" "errors" "fmt" + "io" "os" "path/filepath" + "reflect" "regexp" "strings" "sync" + "sync/atomic" "syscall" "time" + "github.com/projecteru2/core/log" + "github.com/cocoonstack/sandbox/sandboxd/types" ) @@ -91,14 +97,28 @@ type volumeRegistryData struct { type volumeRegistry struct { rootDir, disksDir os.FileInfo mu sync.Mutex + writeMu sync.Mutex + fileMu sync.Mutex + seq, written uint64 + writeFault error + writeFailures atomic.Uint64 + writeRecoveries atomic.Uint64 + retrying atomic.Bool + closed bool + uncertainVMs map[string]bool + discharging map[string]bool + aliases map[backingIdentity]bool dir, node string lock *os.File data volumeRegistryData + original []byte fault error commitFile func(string, []byte) error process func(int) (processIdentity, error) } +var errBackingMismatch = errors.New("managed backing identity changed") + var managedIDPattern = regexp.MustCompile(`^vol_[0-9a-f]{32}$`) func openVolumeRegistry(dir, node string) (*volumeRegistry, error) { @@ -135,7 +155,10 @@ func openVolumeRegistry(dir, node string) (*volumeRegistry, error) { r.fault = errors.New("missing registry with existing backing files") return r, nil } - if err = r.commit(); err != nil { + r.mu.Lock() + err = r.commitLocked() + r.mu.Unlock() + if err != nil { return r, nil } return r, nil @@ -145,13 +168,20 @@ func openVolumeRegistry(dir, node string) (*volumeRegistry, error) { return r, nil } defer func() { _ = f.Close() }() - if err := json.UnmarshalRead(f, &r.data, json.RejectUnknownMembers(true)); err != nil { + raw, readErr := io.ReadAll(f) + if readErr != nil { + r.fault = readErr + return r, nil + } + r.original = raw + if err := json.Unmarshal(raw, &r.data); err != nil { r.fault = err return r, nil } if err := r.validate(); err != nil { r.fault = err } + r.indexBackings() return r, nil } @@ -240,7 +270,7 @@ func (r *volumeRegistry) verify(v managedVolume) error { return err } if v.Backing.Inode == 0 || actual != v.Backing || actual.Size != v.Size { - return errors.New("managed backing identity changed") + return errBackingMismatch } return nil } @@ -374,27 +404,117 @@ func (r *volumeRegistry) validateHoldRef(h volumeHold, ref volumeRef, names map[ return true, nil } -// Any failed snapshot poisons mutations until restart. The attempted state remains -// owned in memory because rename may already have published it. -func (r *volumeRegistry) commit() error { - if r.fault != nil { +// commitLocked is called with mu held and returns with it held. Only the writer +// allocates a snapshot, after taking writeMu without holding the state lock. +func (r *volumeRegistry) commitLocked() error { + if r.fault != nil || r.closed { return managedStorageError() } - if err := r.verifyDirectories(); err != nil { - r.fault = err + r.seq++ + seq := r.seq + r.indexBackings() + r.mu.Unlock() + r.writeMu.Lock() + r.mu.Lock() + if r.closed || r.fault != nil { + r.writeMu.Unlock() return managedStorageError() } + if r.written >= seq { + r.writeMu.Unlock() + if r.writeFault != nil { + return managedStorageError() + } + return nil + } + seq = r.seq raw, err := json.Marshal(r.data) + if err == nil && len(r.original) != 0 { + raw, err = mergeRegistryFields(raw, r.original, reflect.TypeFor[volumeRegistryData]()) + } + write := r.commitFile + r.mu.Unlock() + if err == nil { + err = r.verifyDirectories() + if err != nil { + r.mu.Lock() + r.fault = err + r.mu.Unlock() + } + } if err == nil { - err = r.commitFile(filepath.Join(r.dir, "registry.json"), raw) + err = write(filepath.Join(r.dir, "registry.json"), raw) } + r.mu.Lock() + previousFault := r.writeFault + r.writeFault = err if err != nil { - r.fault = err + r.writeFailures.Add(1) + } + if err == nil && previousFault != nil { + r.writeRecoveries.Add(1) + } + if err == nil { + r.written = seq + } + r.writeMu.Unlock() + r.mu.Unlock() + if previousFault == nil && err != nil { + log.WithFunc("pool.volumeRegistry").Error(context.Background(), err, "managed registry write failed; retaining ownership for retry") + } else if previousFault != nil && err == nil { + log.WithFunc("pool.volumeRegistry").Info(context.Background(), "managed registry persistence recovered") + } + r.mu.Lock() + if err != nil || r.writeFault != nil || r.fault != nil { return managedStorageError() } return nil } +func (r *volumeRegistry) indexBackings() { + r.aliases = make(map[backingIdentity]bool, len(r.data.Volumes)) + for _, v := range r.data.Volumes { + if v.Backing.Inode != 0 { + r.aliases[backingIdentity{Device: v.Backing.Device, Inode: v.Backing.Inode}] = true + } + } +} + +func (m *Manager) retryManagedSnapshot() { + if m.managed == nil { + return + } + r := m.managed + r.mu.Lock() + retry := (r.writeFault != nil || len(r.unboundRetirements()) > 0) && r.fault == nil && !r.closed && r.retrying.CompareAndSwap(false, true) + r.mu.Unlock() + if !retry { + return + } + go func() { + defer r.retrying.Store(false) + r.mu.Lock() + if r.writeFault != nil { + _ = r.commitLocked() + } + ids := r.unboundRetirements() + r.mu.Unlock() + for _, id := range ids { + m.finishManaged(id, nil) + } + }() +} + +func (r *volumeRegistry) unboundRetirements() []string { + var ids []string + for id, h := range r.data.Holds { + if h.State == holdRetiring && h.VMName == "" { + ids = append(ids, id) + } + } + return ids +} + func writeVolumeSnapshot(path string, raw []byte) error { tmp, err := os.CreateTemp(filepath.Dir(path), ".registry-") if err != nil { @@ -415,7 +535,12 @@ func writeVolumeSnapshot(path string, raw []byte) error { func (r *volumeRegistry) close() error { r.mu.Lock() - defer r.mu.Unlock() + r.closed = true + r.mu.Unlock() + r.fileMu.Lock() + defer r.fileMu.Unlock() + r.writeMu.Lock() + defer r.writeMu.Unlock() if r.lock == nil { return nil } diff --git a/sandboxd/pool/volume_registry_fields.go b/sandboxd/pool/volume_registry_fields.go new file mode 100644 index 00000000..e79534a8 --- /dev/null +++ b/sandboxd/pool/volume_registry_fields.go @@ -0,0 +1,103 @@ +package pool + +import ( + "encoding/json/jsontext" + "encoding/json/v2" + "reflect" + "strings" +) + +// mergeRegistryFields retains additive metadata only on objects that still exist. +// Known fields absent from the new projection stay absent, including operation fences. +func mergeRegistryFields(current, original jsontext.Value, typ reflect.Type) (jsontext.Value, error) { + for typ.Kind() == reflect.Pointer { + typ = typ.Elem() + } + if len(current) == 0 || len(original) == 0 || current[0] != original[0] { + return current, nil + } + if current[0] == '[' && typ.Kind() == reflect.Slice { + var next, old []jsontext.Value + if json.Unmarshal(current, &next) != nil || json.Unmarshal(original, &old) != nil { + return current, nil + } + for i := range next { + for j := range old { + if registryObjectKey(next[i]) != "" && registryObjectKey(next[i]) == registryObjectKey(old[j]) { + value, err := mergeRegistryFields(next[i], old[j], typ.Elem()) + if err != nil { + return nil, err + } + next[i] = value + break + } + } + } + return json.Marshal(next) + } + if current[0] != '{' || typ.Kind() != reflect.Struct && typ.Kind() != reflect.Map { + return current, nil + } + var next, old map[string]jsontext.Value + if json.Unmarshal(current, &next) != nil || json.Unmarshal(original, &old) != nil { + return current, nil + } + if a, b := registryObjectKey(current), registryObjectKey(original); typ.Kind() == reflect.Struct && a != b { + return current, nil + } + fields := map[string]reflect.Type{} + if typ.Kind() == reflect.Struct { + for field := range typ.Fields() { + name, _, _ := strings.Cut(field.Tag.Get("json"), ",") + if name == "" { + name = field.Name + } + fields[name] = field.Type + } + } + for name, previous := range old { + child, known := fields[name] + if typ.Kind() == reflect.Map { + child, known = typ.Elem(), true + } + if !known { + next[name] = previous + continue + } + if value, exists := next[name]; exists { + merged, err := mergeRegistryFields(value, previous, child) + if err != nil { + return nil, err + } + next[name] = merged + } + } + return json.Marshal(next) +} + +func registryObjectKey(raw jsontext.Value) string { + var obj map[string]jsontext.Value + if json.Unmarshal(raw, &obj) != nil { + return "" + } + if request, ok := obj["request"]; ok { + if key := registryObjectKey(request); key != "" { + return key + } + } + if boot, ok := obj["boot_id"]; ok { + return string(boot) + "/" + string(obj["pid"]) + "/" + string(obj["start"]) + } + if inode, ok := obj["inode"]; ok { + return string(obj["device"]) + "/" + string(inode) + } + for _, key := range []string{"id", "sandbox_id", "operation", "operation_id", "name", "number"} { + if value, ok := obj[key]; ok { + return key + string(value) + } + } + if volume, ok := obj["volume"]; ok { + return registryObjectKey(volume) + } + return "" +} diff --git a/sandboxd/pool/volume_registry_fields_test.go b/sandboxd/pool/volume_registry_fields_test.go new file mode 100644 index 00000000..56607885 --- /dev/null +++ b/sandboxd/pool/volume_registry_fields_test.go @@ -0,0 +1,122 @@ +package pool + +import ( + "encoding/json/v2" + "os" + "path/filepath" + "reflect" + "testing" + + "github.com/cocoonstack/sandbox/sandboxd/config" +) + +func TestManagedRegistryPreservesSupportedAdditiveFields(t *testing.T) { + for _, mode := range []string{"sleep", "growth"} { + t.Run(mode, func(t *testing.T) { + var m *Manager + var cfg *config.Config + var id string + if mode == "sleep" { + manager, _, sb, c := newSleepTest(t) + m, cfg, id = manager, c, sb.ID + requireSleep(t, m, sb) + } else { + manager, _, sb, c, v := newGrowthTest(t) + m, cfg, id = manager, c, sb.ID + if _, err := m.GrowVolume(t.Context(), "data", v.ID, growRequest(), "t1"); err != nil { + t.Fatal(err) + } + } + m.Close() + path := filepath.Join(cfg.ManagedVolumeDir, "registry.json") + raw, err := os.ReadFile(path) + if err != nil { + t.Fatal(err) + } + var doc map[string]any + if err = json.Unmarshal(raw, &doc); err != nil { + t.Fatal(err) + } + volume := doc["volumes"].(map[string]any)["data"].(map[string]any) + hold := doc["holds"].(map[string]any)[id].(map[string]any) + ref := hold["refs"].([]any)[0].(map[string]any) + objects := []map[string]any{doc, volume, volume["backing"].(map[string]any), hold, hold["process"].(map[string]any), ref, ref["volume"].(map[string]any), ref["backing"].(map[string]any)} + if mode == "sleep" { + sleep := hold["sleep"].(map[string]any) + objects = append(objects, sleep, sleep["hook"].(map[string]any), sleep["before_restore"].(map[string]any)) + } else { + growth := volume["growth"].(map[string]any) + objects = append(objects, growth, growth["request"].(map[string]any), growth["runtime"].(map[string]any), growth["guest"].(map[string]any)) + } + for _, object := range objects { + object["future_note"] = map[string]any{"value": "preserve"} + } + raw, err = json.Marshal(doc) + if err != nil { + t.Fatal(err) + } + if err = os.WriteFile(path, raw, 0o600); err != nil { + t.Fatal(err) + } + r, err := openVolumeRegistry(cfg.ManagedVolumeDir, "node") + if err != nil { + t.Fatal(err) + } + defer r.close() + if r.fault != nil { + t.Fatal(r.fault) + } + r.mu.Lock() + err = r.commitLocked() + r.mu.Unlock() + if err != nil { + t.Fatal(err) + } + after, err := os.ReadFile(path) + if err != nil { + t.Fatal(err) + } + var persisted map[string]any + if err := json.Unmarshal(after, &persisted); err != nil { + t.Fatal(err) + } + if !reflect.DeepEqual(doc, persisted) { + t.Fatalf("optional fields changed across a supported registry rewrite\nbefore=%s\nafter=%s", raw, after) + } + }) + } +} + +func TestManagedRegistryExtensionsFollowObjectIdentity(t *testing.T) { + for _, tc := range []struct { + name, old, next string + typ reflect.Type + }{ + {"removed operation", `{"state":"ready","sleep":{"operation":"old","future":true},"future":true}`, `{"state":"claimed"}`, reflect.TypeFor[volumeHold]()}, + {"new process", `{"boot_id":"boot","pid":12,"start":3,"future":true}`, `{"boot_id":"boot","pid":12,"start":4}`, reflect.TypeFor[processIdentity]()}, + {"new volume", `{"id":"old","name":"data","future":true}`, `{"id":"new","name":"data"}`, reflect.TypeFor[managedVolume]()}, + {"removed growth", `{"id":"same","growth":{"future":true}}`, `{"id":"same"}`, reflect.TypeFor[managedVolume]()}, + } { + t.Run(tc.name, func(t *testing.T) { + raw, err := mergeRegistryFields([]byte(tc.next), []byte(tc.old), tc.typ) + if err != nil { + t.Fatal(err) + } + var got map[string]any + if err := json.Unmarshal(raw, &got); err != nil { + t.Fatal(err) + } + if _, exists := got["sleep"]; exists { + t.Fatal("removed sleep was resurrected") + } + if _, exists := got["growth"]; exists { + t.Fatal("removed growth was resurrected") + } + if tc.name != "removed operation" { + if _, exists := got["future"]; exists { + t.Fatal("metadata crossed an identity boundary") + } + } + }) + } +} diff --git a/sandboxd/server/metrics.go b/sandboxd/server/metrics.go index ea60cb3d..5a3b9eab 100644 --- a/sandboxd/server/metrics.go +++ b/sandboxd/server/metrics.go @@ -68,6 +68,8 @@ func (s *Server) handleMetrics(w http.ResponseWriter, _ *http.Request) { {"forks_total", "fork operations", c.Forks}, {"checkpoints_total", "checkpoints captured", c.Checkpoints}, {"promotes_total", "templates promoted", c.Promotes}, + {"managed_volume_write_failures_total", "failed managed registry writes", c.ManagedWriteFailures}, + {"managed_volume_write_recoveries_total", "managed registry write recoveries", c.ManagedWriteRecoveries}, {"releases_total", "claims released", c.Releases}, {"reaps_total", "claims reaped at deadline", c.Reaps}, {"archives_total", "claims archived to the store", c.Archives}, From 9a51c92a34ecad316474d94f25cb832015c5cd64 Mon Sep 17 00:00:00 2001 From: minatoaquaMK2 Date: Tue, 20 Oct 2026 10:49:27 +0000 Subject: [PATCH 18/75] pool: cover stale backing observations during growth --- sandboxd/pool/managed_grow_recovery_test.go | 29 +++++++++++++++++++++ 1 file changed, 29 insertions(+) diff --git a/sandboxd/pool/managed_grow_recovery_test.go b/sandboxd/pool/managed_grow_recovery_test.go index 4d7264e5..9119cbb2 100644 --- a/sandboxd/pool/managed_grow_recovery_test.go +++ b/sandboxd/pool/managed_grow_recovery_test.go @@ -11,6 +11,35 @@ import ( "github.com/cocoonstack/sandbox/sandboxd/engine" ) +func TestManagedBackingObservationCannotDiscardGrowth(t *testing.T) { + for _, unknown := range []bool{false, true} { + t.Run(map[bool]string{false: "completed", true: "unknown"}[unknown], func(t *testing.T) { + m, e, _, _, v := newGrowthTest(t) + m.managed.mu.Lock() + observed := m.managed.data.Volumes["data"] + m.managed.mu.Unlock() + e.unknown = unknown + _, growErr := m.GrowVolume(t.Context(), "data", v.ID, growRequest(), "t1") + if unknown { + requireVolumeStatus(t, growErr, 409) + } else if growErr != nil { + t.Fatal(growErr) + } + before, err := m.GetVolume("data", "t1") + if err != nil || before.Growth == nil { + t.Fatalf("growth result unavailable: %+v %v", before, err) + } + m.managed.mu.Lock() + _ = m.managed.markRecovery(observed) + m.managed.mu.Unlock() + after, err := m.GetVolume("data", "t1") + if err != nil || !reflect.DeepEqual(after, before) { + t.Fatalf("stale backing observation discarded growth: before=%+v after=%+v error=%v", before, after, err) + } + }) + } +} + func TestGrowthMissingVMRecordPreservesDurableOwner(t *testing.T) { m, e, sb, cfg, v := newGrowthTest(t) e.unknown = true From 0b01150b25596539f1e51fcc87cc2730eed0b4ee Mon Sep 17 00:00:00 2001 From: minatoaquaMK2 Date: Tue, 20 Oct 2026 10:50:27 +0000 Subject: [PATCH 19/75] pool: preserve newer state after backing observations --- sandboxd/pool/managed_volume.go | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/sandboxd/pool/managed_volume.go b/sandboxd/pool/managed_volume.go index d202f717..2c4c3d2f 100644 --- a/sandboxd/pool/managed_volume.go +++ b/sandboxd/pool/managed_volume.go @@ -143,6 +143,11 @@ func (r *volumeRegistry) completeCreate(v managedVolume) error { } func (r *volumeRegistry) markRecovery(v managedVolume) error { + current, ok := r.data.Volumes[v.Name] + if !ok || current.ID != v.ID || current.State != v.State || current.Size != v.Size || + current.Backing != v.Backing || current.CapacityGeneration != v.CapacityGeneration || current.Growth != v.Growth { + return managedStorageError() + } if r.sleepOwnsVolume(v.ID) || v.Growth.fenced() { return managedStorageError() } From 0b4cf98cfe6224676a51f692b28e2e623fa8c155 Mon Sep 17 00:00:00 2001 From: minatoaquaMK2 Date: Tue, 20 Oct 2026 10:51:16 +0000 Subject: [PATCH 20/75] pool: make snapshot lock handoff explicit --- sandboxd/pool/managed_attachment.go | 18 +++++++++--------- sandboxd/pool/managed_grow.go | 6 +++--- sandboxd/pool/managed_grow_test.go | 4 ++-- sandboxd/pool/managed_recovery.go | 4 ++-- sandboxd/pool/managed_sleep_recovery.go | 2 +- sandboxd/pool/managed_sleep_registry.go | 4 ++-- sandboxd/pool/managed_volume.go | 14 +++++++------- sandboxd/pool/volume_registry.go | 8 +++----- sandboxd/pool/volume_registry_fields_test.go | 2 +- 9 files changed, 30 insertions(+), 32 deletions(-) diff --git a/sandboxd/pool/managed_attachment.go b/sandboxd/pool/managed_attachment.go index 4638d557..48f7d19f 100644 --- a/sandboxd/pool/managed_attachment.go +++ b/sandboxd/pool/managed_attachment.go @@ -113,7 +113,7 @@ func (m *Manager) admitManaged(tenant string, volumes []resolvedVolume) ([]types if cleanErr != nil { delete(r.data.Holds, h.SandboxID) } - err := r.commitLocked() + err := r.commitAndRelock() r.mu.Unlock() m.mu.Lock() r.mu.Lock() @@ -173,7 +173,7 @@ func (m *Manager) releaseUnbound(volumes []resolvedVolume, reserved []types.Volu } delete(r.data.Holds, id) m.mu.Unlock() - err := r.commitLocked() + err := r.commitAndRelock() r.mu.Unlock() m.mu.Lock() r.mu.Lock() @@ -212,7 +212,7 @@ func (m *Manager) bindManaged(ctx context.Context, sb *types.Sandbox, volumes [] h.VMName = sb.VMName h.Process = p r.data.Holds[id] = h - return r.commitLocked() + return r.commitAndRelock() } func (m *Manager) activateManaged(id string) error { @@ -234,7 +234,7 @@ func (m *Manager) activateManaged(id string) error { } h.State = holdClaimed r.data.Holds[id] = h - return r.commitLocked() + return r.commitAndRelock() } func (m *Manager) finishManaged(id string, clears []string, charges ...[]types.Volume) bool { @@ -280,7 +280,7 @@ func (m *Manager) finishManaged(id string, clears []string, charges ...[]types.V } r.discharging[id] = true delete(r.data.Holds, id) - if r.commitLocked() != nil { + if r.commitAndRelock() != nil { r.data.Holds[id] = h delete(r.discharging, id) return false @@ -320,7 +320,7 @@ func (m *Manager) restoreManagedClaim(previous volumeHold, err error) { if ok && h.State == holdRetiring { h.State = previous.State r.data.Holds[h.SandboxID] = h - _ = r.commitLocked() + _ = r.commitAndRelock() } } @@ -365,7 +365,7 @@ func (m *Manager) recoverManaged(ctx context.Context, claims map[string]*types.S r.mu.Lock() h.State = holdClaimed r.data.Holds[h.SandboxID] = h - err := r.commitLocked() + err := r.commitAndRelock() r.mu.Unlock() if err != nil { return owned, err @@ -503,7 +503,7 @@ func (m *Manager) managedLiveRemove(id string, volumes []types.Volume) error { } h.Refs = refs r.data.Holds[id] = h - return r.commitLocked() + return r.commitAndRelock() } func catalogBacking(path string) (backingIdentity, error) { @@ -525,7 +525,7 @@ func (m *Manager) commitManaged() error { r := m.managed r.mu.Lock() defer r.mu.Unlock() - return r.commitLocked() + return r.commitAndRelock() } func processMissing(err error) bool { diff --git a/sandboxd/pool/managed_grow.go b/sandboxd/pool/managed_grow.go index 6f3daf60..53fe108f 100644 --- a/sandboxd/pool/managed_grow.go +++ b/sandboxd/pool/managed_grow.go @@ -287,7 +287,7 @@ func (m *Manager) commitGrowthIntent(sb *types.Sandbox, v managedVolume, next ma r.mu.Lock() if r.data.Version != 3 { r.data.Version = 3 - if err := r.commitLocked(); err != nil { + if err := r.commitAndRelock(); err != nil { r.mu.Unlock() return err } @@ -316,7 +316,7 @@ func (m *Manager) commitGrowthIntent(sb *types.Sandbox, v managedVolume, next ma current.Growth = &next r.data.Volumes[v.Name] = current m.mu.Unlock() - err = r.commitLocked() + err = r.commitAndRelock() r.mu.Unlock() return err } @@ -479,7 +479,7 @@ func (m *Manager) saveGrowth(v managedVolume, g managedGrowth, state, outcome st current.State = volumeRecovery } r.data.Volumes[v.Name] = current - return r.commitLocked() + return r.commitAndRelock() } func (r *volumeRegistry) validateGrowth(v managedVolume) error { diff --git a/sandboxd/pool/managed_grow_test.go b/sandboxd/pool/managed_grow_test.go index 8f0c0c54..50da32ff 100644 --- a/sandboxd/pool/managed_grow_test.go +++ b/sandboxd/pool/managed_grow_test.go @@ -561,7 +561,7 @@ func TestManagedGrowthMigratesVersionTwoBeforeIntentAndKeepsSleepValid(t *testin r := m.managed r.data.Version = 2 r.mu.Lock() - if err := r.commitLocked(); err != nil { + if err := r.commitAndRelock(); err != nil { t.Fatal(err) } r.mu.Unlock() @@ -587,7 +587,7 @@ func TestManagedGrowthMigratesVersionTwoBeforeIntentAndKeepsSleepValid(t *testin sleeping, _, sb, _ := newSleepTest(t) sleeping.managed.data.Version = 3 sleeping.managed.mu.Lock() - if err := sleeping.managed.commitLocked(); err != nil { + if err := sleeping.managed.commitAndRelock(); err != nil { t.Fatal(err) } sleeping.managed.mu.Unlock() diff --git a/sandboxd/pool/managed_recovery.go b/sandboxd/pool/managed_recovery.go index e7c93c98..d3e3fddc 100644 --- a/sandboxd/pool/managed_recovery.go +++ b/sandboxd/pool/managed_recovery.go @@ -67,7 +67,7 @@ func (m *Manager) recoverManagedOrphans(ctx context.Context, claims map[string]* } r.mu.Lock() r.data.Orphans = maps.Clone(orphans) - err := r.commitLocked() + err := r.commitAndRelock() r.mu.Unlock() if err != nil { return err @@ -82,7 +82,7 @@ func (m *Manager) recoverManagedOrphans(ctx context.Context, claims map[string]* } r.mu.Lock() delete(r.data.Orphans, name) - err := r.commitLocked() + err := r.commitAndRelock() r.mu.Unlock() if err != nil { return err diff --git a/sandboxd/pool/managed_sleep_recovery.go b/sandboxd/pool/managed_sleep_recovery.go index f5c21bd6..8f2e923b 100644 --- a/sandboxd/pool/managed_sleep_recovery.go +++ b/sandboxd/pool/managed_sleep_recovery.go @@ -121,7 +121,7 @@ func (m *Manager) observeManagedRemoval(ctx context.Context, h volumeHold) (volu next.Restored = p h.Sleep = &next r.data.Holds[h.SandboxID] = h - return h, true, r.commitLocked() + return h, true, r.commitAndRelock() } func (m *Manager) retryManagedSleepRemoval(ctx context.Context, name string, pending pendingRemoval) bool { diff --git a/sandboxd/pool/managed_sleep_registry.go b/sandboxd/pool/managed_sleep_registry.go index 1e7f1bbd..2b4c3d39 100644 --- a/sandboxd/pool/managed_sleep_registry.go +++ b/sandboxd/pool/managed_sleep_registry.go @@ -72,7 +72,7 @@ func (r *volumeRegistry) commitManagedSleep(h volumeHold, next *managedSleep) er } if r.data.Version == 1 { r.data.Version = 2 - if err := r.commitLocked(); err != nil { + if err := r.commitAndRelock(); err != nil { return err } } @@ -83,7 +83,7 @@ func (r *volumeRegistry) commitManagedSleep(h volumeHold, next *managedSleep) er } } r.data.Holds[h.SandboxID] = h - if err := r.commitLocked(); err != nil { + if err := r.commitAndRelock(); err != nil { if next == nil { h.Sleep = current.Sleep r.data.Holds[h.SandboxID] = h diff --git a/sandboxd/pool/managed_volume.go b/sandboxd/pool/managed_volume.go index 2c4c3d2f..35c1306d 100644 --- a/sandboxd/pool/managed_volume.go +++ b/sandboxd/pool/managed_volume.go @@ -77,7 +77,7 @@ func (m *Manager) CreateVolume(ctx context.Context, name string, req types.Volum if !exists { v = managedVolume{ID: volumeID("vol_" + randHex(16)), Name: name, Tenant: owner, Node: r.node, Size: size, State: volumeCreating, CreatedAt: time.Now().UTC()} r.data.Volumes[name] = v - if err := r.commitLocked(); err != nil { + if err := r.commitAndRelock(); err != nil { return types.ManagedVolumeInfo{}, false, err } } @@ -139,7 +139,7 @@ func (r *volumeRegistry) completeCreate(v managedVolume) error { r.mu.Lock() defer r.mu.Unlock() r.data.Volumes[v.Name] = v - return r.commitLocked() + return r.commitAndRelock() } func (r *volumeRegistry) markRecovery(v managedVolume) error { @@ -153,7 +153,7 @@ func (r *volumeRegistry) markRecovery(v managedVolume) error { } v.State = volumeRecovery r.data.Volumes[v.Name] = v - if err := r.commitLocked(); err != nil { + if err := r.commitAndRelock(); err != nil { return err } return managedStorageError() @@ -250,7 +250,7 @@ func (m *Manager) DeleteVolume(ctx context.Context, name, expected, tenant strin if v.State != volumeDeleting { v.State = volumeDeleting r.data.Volumes[name] = v - if err := r.commitLocked(); err != nil { + if err := r.commitAndRelock(); err != nil { return err } } @@ -262,7 +262,7 @@ func (m *Manager) DeleteVolume(ctx context.Context, name, expected, tenant strin } r.data.Deleted[id] = volumeTombstone{ID: id, Name: name, Tenant: v.Tenant} delete(r.data.Volumes, name) - return r.commitLocked() + return r.commitAndRelock() } func (r *volumeRegistry) holders(id volumeID) []types.VolumeHolder { @@ -408,13 +408,13 @@ func (m *Manager) retireManaged(id string) error { } if h.State == holdRetiring { if r.writeFault != nil { - return r.commitLocked() + return r.commitAndRelock() } return nil } h.State = holdRetiring r.data.Holds[id] = h - return r.commitLocked() + return r.commitAndRelock() } func (m *Manager) managedRefPath(id, name string) (string, bool) { diff --git a/sandboxd/pool/volume_registry.go b/sandboxd/pool/volume_registry.go index efd742a2..024637d9 100644 --- a/sandboxd/pool/volume_registry.go +++ b/sandboxd/pool/volume_registry.go @@ -156,7 +156,7 @@ func openVolumeRegistry(dir, node string) (*volumeRegistry, error) { return r, nil } r.mu.Lock() - err = r.commitLocked() + err = r.commitAndRelock() r.mu.Unlock() if err != nil { return r, nil @@ -404,9 +404,7 @@ func (r *volumeRegistry) validateHoldRef(h volumeHold, ref volumeRef, names map[ return true, nil } -// commitLocked is called with mu held and returns with it held. Only the writer -// allocates a snapshot, after taking writeMu without holding the state lock. -func (r *volumeRegistry) commitLocked() error { +func (r *volumeRegistry) commitAndRelock() error { if r.fault != nil || r.closed { return managedStorageError() } @@ -495,7 +493,7 @@ func (m *Manager) retryManagedSnapshot() { defer r.retrying.Store(false) r.mu.Lock() if r.writeFault != nil { - _ = r.commitLocked() + _ = r.commitAndRelock() } ids := r.unboundRetirements() r.mu.Unlock() diff --git a/sandboxd/pool/volume_registry_fields_test.go b/sandboxd/pool/volume_registry_fields_test.go index 56607885..7e54d3c4 100644 --- a/sandboxd/pool/volume_registry_fields_test.go +++ b/sandboxd/pool/volume_registry_fields_test.go @@ -67,7 +67,7 @@ func TestManagedRegistryPreservesSupportedAdditiveFields(t *testing.T) { t.Fatal(r.fault) } r.mu.Lock() - err = r.commitLocked() + err = r.commitAndRelock() r.mu.Unlock() if err != nil { t.Fatal(err) From 0ee0292b0ee7083e4307fb616bd3cd51ec076266 Mon Sep 17 00:00:00 2001 From: minatoaquaMK2 Date: Tue, 20 Oct 2026 11:04:53 +0000 Subject: [PATCH 21/75] pool: preserve recovery and durability fences on retries --- sandboxd/pool/egress_test.go | 65 +++++++++++++++++++- sandboxd/pool/liveness_test.go | 2 +- sandboxd/pool/managed_foundation_test.go | 20 ++++++ sandboxd/pool/managed_grow.go | 2 +- sandboxd/pool/managed_grow_test.go | 47 ++++++++++++++ sandboxd/pool/managed_sleep_registry.go | 2 +- sandboxd/pool/reconcile.go | 14 +++-- sandboxd/pool/volume_registry_fields.go | 35 ++++++++--- sandboxd/pool/volume_registry_fields_test.go | 58 +++++++++++++++++ 9 files changed, 226 insertions(+), 19 deletions(-) diff --git a/sandboxd/pool/egress_test.go b/sandboxd/pool/egress_test.go index 12f05bf2..b1ab6b57 100644 --- a/sandboxd/pool/egress_test.go +++ b/sandboxd/pool/egress_test.go @@ -313,7 +313,7 @@ func TestQuarantineFailedRemoveStaysUnswept(t *testing.T) { var gotKeep map[string]bool withOutbound(t, m, func(o *outbound.Options) { o.Sweep = func(keep map[string]bool) error { gotKeep = keep; return nil } }) - m.resyncEgress(t.Context(), live, removed) + m.resyncEgress(t.Context(), live, removed, true) if removed["sbx-q1"] { t.Error("failed remove marked the VM gone; the sweep would drop its lock table") @@ -700,3 +700,66 @@ func withOutbound(t *testing.T, m *Manager, edit func(*outbound.Options)) { edit(&o) m.out = outbound.New(o) } + +func TestManagedFaultPreservesOrdinaryEgressRecovery(t *testing.T) { + for _, corrupt := range []bool{false, true} { + name := "healthy" + if corrupt { + name = "corrupt" + } + t.Run(name, func(t *testing.T) { + cfg := managedConfig(t) + e := newManagedEngine() + m := openManagedTest(t, cfg, e) + sb := &types.Sandbox{ID: "sb_ordinary", VMName: "sbx-ordinary", Key: egKey, VsockSocket: "/fake/vsock", Token: "token"} + e.vms[sb.VMName] = sb.VsockSocket + if err := m.store.commitDurable(m.store.set(sb)); err != nil { + t.Fatal(err) + } + if err := m.Close(); err != nil { + t.Fatal(err) + } + if corrupt { + if err := os.WriteFile(filepath.Join(cfg.ManagedVolumeDir, "registry.json"), []byte("{"), 0o600); err != nil { + t.Fatal(err) + } + } + m = openManagedTest(t, cfg, e) + withOutbound(t, m, func(o *outbound.Options) { o.LockNIC = true; o.Sweep = func(map[string]bool) error { return nil } }) + if err := m.Reconcile(t.Context()); err != nil { + t.Fatal(err) + } + live, present := m.Sandbox(sb.ID) + failed := "" + if present { + failed = live.Failed + } + t.Logf("corrupt=%v claim_present=%v failed=%q locks_nic=%v uncertain_vm=%v", corrupt, present, failed, m.out.LocksNIC(egKey), m.managedUncertainVM(sb.VMName)) + if present && failed == "" { + t.Error("ordinary egress claim with no lockable TAP remains active after recovery") + } + }) + } +} + +func TestUnknownInventoryPreservesEgressTables(t *testing.T) { + e := newManagedEngine() + m := openManagedTest(t, managedConfig(t), e) + sb := &types.Sandbox{ID: "sb_unknown", VMName: "sbx-unknown", Key: egKey, Token: "token", VsockSocket: "/fake/socket"} + e.vms[sb.VMName] = sb.VsockSocket + m.mu.Lock() + m.addClaimLocked(sb) + m.mu.Unlock() + swept := false + withOutbound(t, m, func(o *outbound.Options) { + o.LockNIC = true + o.Sweep = func(map[string]bool) error { swept = true; return nil } + }) + m.resyncEgress(t.Context(), map[string]types.VMRecord{}, map[string]bool{}, false) + if live, present := m.Sandbox(sb.ID); present && live.Failed == "" { + t.Fatal("unknown inventory left an egress claim active without a verified lock") + } + if swept { + t.Fatal("unknown inventory swept potentially live egress locks") + } +} diff --git a/sandboxd/pool/liveness_test.go b/sandboxd/pool/liveness_test.go index 2864c615..83090a4a 100644 --- a/sandboxd/pool/liveness_test.go +++ b/sandboxd/pool/liveness_test.go @@ -409,7 +409,7 @@ func TestResyncRecordsADownEgressLaneTap(t *testing.T) { o.Sweep = func(map[string]bool) error { return nil } }) - m.resyncEgress(t.Context(), live, map[string]bool{}) + m.resyncEgress(t.Context(), live, map[string]bool{}, true) m.mu.Lock() kept := m.claimed[sb.ID] == sb diff --git a/sandboxd/pool/managed_foundation_test.go b/sandboxd/pool/managed_foundation_test.go index 520d3248..1a891b84 100644 --- a/sandboxd/pool/managed_foundation_test.go +++ b/sandboxd/pool/managed_foundation_test.go @@ -701,3 +701,23 @@ func TestManagedMissingBootMetadataCannotProveProcessGone(t *testing.T) { t.Fatal("missing boot metadata was treated as an absent VMM") } } + +func TestManagedRecoveryFencesAgentSocket(t *testing.T) { + m, e, sb, _, _ := newGrowthTest(t) + ordinary := &types.Sandbox{ID: "sb_ordinary", VMName: "sbx-ordinary", Token: "ordinary-token", VsockSocket: "/ordinary/socket"} + e.vms[ordinary.VMName] = ordinary.VsockSocket + m.mu.Lock() + m.addClaimLocked(ordinary) + m.mu.Unlock() + m.fenceManagedRecovery(t.Context(), errors.New("inventory unavailable"), map[string]*types.Sandbox{sb.ID: sb, ordinary.ID: ordinary}, nil) + if socket, err := m.AgentSocket(sb.ID, sb.Token); err == nil || socket != "" { + t.Fatalf("uncertain managed claim exposes socket %q, error=%v", socket, err) + } + if socket, err := m.AgentSocket(ordinary.ID, ordinary.Token); err != nil || socket != ordinary.VsockSocket { + t.Fatalf("ordinary claim affected: socket=%q error=%v", socket, err) + } + sb.Volumes = nil + if socket, err := m.AgentSocket(sb.ID, sb.Token); err == nil || socket != "" { + t.Fatalf("recorded uncertain VM exposes socket after empty volume projection: %q, %v", socket, err) + } +} diff --git a/sandboxd/pool/managed_grow.go b/sandboxd/pool/managed_grow.go index 53fe108f..600918bf 100644 --- a/sandboxd/pool/managed_grow.go +++ b/sandboxd/pool/managed_grow.go @@ -114,7 +114,7 @@ func (m *Manager) growthTeardownTarget(ctx context.Context, name string) (string } func (r *volumeRegistry) growVolume(name, expectedID, tenant string, req types.VolumeGrow) (managedVolume, error) { - if r.fault != nil { + if r.closed || r.fault != nil || r.writeFault != nil { return managedVolume{}, managedStorageError() } v, ok := r.data.Volumes[name] diff --git a/sandboxd/pool/managed_grow_test.go b/sandboxd/pool/managed_grow_test.go index 50da32ff..faf6be85 100644 --- a/sandboxd/pool/managed_grow_test.go +++ b/sandboxd/pool/managed_grow_test.go @@ -7,6 +7,7 @@ import ( "os" "path/filepath" "sync" + "syscall" "testing" "time" @@ -596,3 +597,49 @@ func TestManagedGrowthMigratesVersionTwoBeforeIntentAndKeepsSleepValid(t *testin t.Fatal("version three rejected existing sleep", err) } } + +func TestManagedGrowthRetryRequiresDurableIntent(t *testing.T) { + for _, phase := range []string{"intent", "completion"} { + t.Run(phase, func(t *testing.T) { + m, e, _, _, v := newGrowthTest(t) + r := m.managed + r.commitFile = func(path string, raw []byte) error { + var snapshot volumeRegistryData + if err := json.Unmarshal(raw, &snapshot); err != nil { + return err + } + growth := snapshot.Volumes["data"].Growth + if growth != nil && (phase == "intent" || growth.State == growthCompleted) { + return syscall.ENOSPC + } + return writeVolumeSnapshot(path, raw) + } + _, err := m.GrowVolume(t.Context(), "data", v.ID, growRequest(), "t1") + if err == nil { + t.Fatal("injected persistence failure did not fail") + } + wantCalls := 0 + if phase == "completion" { + wantCalls = 1 + } + if e.calls != wantCalls { + t.Fatalf("runtime calls=%d, want %d", e.calls, wantCalls) + } + _, err = m.GrowVolume(t.Context(), "data", v.ID, growRequest(), "t1") + requireVolumeStatus(t, err, 503) + if e.calls != wantCalls { + t.Fatal("retry dispatched undurable operation") + } + r.commitFile = writeVolumeSnapshot + m.retryManagedSnapshot() + waitFor(t, func() bool { return !r.retrying.Load() }) + result, err := m.GrowVolume(t.Context(), "data", v.ID, growRequest(), "t1") + if err != nil { + t.Fatal(err) + } + if e.calls != 1 || !e.intentSeen || result.SizeBytes != 2<<30 || result.CapacityGeneration != 1 { + t.Fatalf("recovered operation did not converge: calls=%d intent=%v result=%+v", e.calls, e.intentSeen, result) + } + }) + } +} diff --git a/sandboxd/pool/managed_sleep_registry.go b/sandboxd/pool/managed_sleep_registry.go index 2b4c3d39..2917c2c0 100644 --- a/sandboxd/pool/managed_sleep_registry.go +++ b/sandboxd/pool/managed_sleep_registry.go @@ -127,7 +127,7 @@ func validSleepProcess(p processIdentity) bool { return p.PID > 0 && p.Start > 0 func (m *Manager) managedPaused(sb *types.Sandbox) bool { h, held := m.managedHold(sb.ID) - return held && (h.Sleep != nil || h.State == holdRetiring) + return m.managedClaimUncertain(sb) || held && (h.Sleep != nil || h.State == holdRetiring) } func (r *volumeRegistry) sleepOwnsVolume(id volumeID) bool { diff --git a/sandboxd/pool/reconcile.go b/sandboxd/pool/reconcile.go index d7fe0bf9..2cb0489e 100644 --- a/sandboxd/pool/reconcile.go +++ b/sandboxd/pool/reconcile.go @@ -118,8 +118,8 @@ func (m *Manager) Reconcile(ctx context.Context) error { if managedErr == nil { m.sweepOrphanSnapshots(ctx, snaps, snapsErr, referenced) - m.resyncEgress(ctx, live, removed) } + m.resyncEgress(ctx, live, removed, inventoryErr == nil) m.warnDeniedClaims(ctx) logger.Infof(ctx, "adopted %d claims, %d VMs live", len(m.claimed), len(live)) return saveErr @@ -224,7 +224,7 @@ func (m *Manager) holdStaleVolumes(ctx context.Context, name string, rec types.V } // resyncEgress re-locks adopted egress claims, quarantining any it cannot lock. -func (m *Manager) resyncEgress(ctx context.Context, live map[string]types.VMRecord, removed map[string]bool) { +func (m *Manager) resyncEgress(ctx context.Context, live map[string]types.VMRecord, removed map[string]bool, inventoryKnown bool) { logger := log.WithFunc("pool.resyncEgress") now := time.Now() locked, lockedErr := m.out.LockedTaps() @@ -238,7 +238,7 @@ func (m *Manager) resyncEgress(ctx context.Context, live map[string]types.VMReco if m.out.LocksNIC(sb.Key) { tap := m.readoptEgressTap(sb, live) // a down egress-lane VM has no guest to lock or mark; the liveness pass fails it - if live[sb.VMName].State != vmStateRunning { + if rec, known := live[sb.VMName]; known && rec.State != vmStateRunning { continue } if tap == "" { @@ -269,6 +269,9 @@ func (m *Manager) resyncEgress(ctx context.Context, live map[string]types.VMReco keep[sb.TAP] = true } } + if !inventoryKnown { + return + } // keep every still-running VM's tap; sweep only a confirmed-gone VM's table. for name, rec := range live { if tap := rec.TapDevice(); tap != "" && !removed[name] { @@ -411,13 +414,14 @@ func (m *Manager) managedUncertainVM(name string) bool { const managedRecoveryFailure = "managed storage recovery required" func (m *Manager) managedClaimUncertain(sb *types.Sandbox) bool { - if m.managed == nil || !hasVolumes(sb) { + if m.managed == nil { return false } r := m.managed r.mu.Lock() defer r.mu.Unlock() - return r.fault != nil || r.uncertainVMs[sb.VMName] + _, held := r.data.Holds[sb.ID] + return r.uncertainVMs[sb.VMName] || r.fault != nil && (hasVolumes(sb) || held) } func (m *Manager) reconcileManaged(ctx context.Context, claims map[string]*types.Sandbox, live map[string]types.VMRecord) (map[string]bool, error) { diff --git a/sandboxd/pool/volume_registry_fields.go b/sandboxd/pool/volume_registry_fields.go index e79534a8..fa7dc7ed 100644 --- a/sandboxd/pool/volume_registry_fields.go +++ b/sandboxd/pool/volume_registry_fields.go @@ -23,7 +23,7 @@ func mergeRegistryFields(current, original jsontext.Value, typ reflect.Type) (js } for i := range next { for j := range old { - if registryObjectKey(next[i]) != "" && registryObjectKey(next[i]) == registryObjectKey(old[j]) { + if registryObjectKey(next[i], typ.Elem()) != "" && registryObjectKey(next[i], typ.Elem()) == registryObjectKey(old[j], typ.Elem()) { value, err := mergeRegistryFields(next[i], old[j], typ.Elem()) if err != nil { return nil, err @@ -42,7 +42,7 @@ func mergeRegistryFields(current, original jsontext.Value, typ reflect.Type) (js if json.Unmarshal(current, &next) != nil || json.Unmarshal(original, &old) != nil { return current, nil } - if a, b := registryObjectKey(current), registryObjectKey(original); typ.Kind() == reflect.Struct && a != b { + if a, b := registryObjectKey(current, typ), registryObjectKey(original, typ); typ.Kind() == reflect.Struct && a != b { return current, nil } fields := map[string]reflect.Type{} @@ -75,29 +75,44 @@ func mergeRegistryFields(current, original jsontext.Value, typ reflect.Type) (js return json.Marshal(next) } -func registryObjectKey(raw jsontext.Value) string { +func registryObjectKey(raw jsontext.Value, typ reflect.Type) string { + for typ.Kind() == reflect.Pointer { + typ = typ.Elem() + } + if typ.Kind() != reflect.Struct { + return "" + } + fields := map[string]reflect.Type{} + for field := range typ.Fields() { + name, _, _ := strings.Cut(field.Tag.Get("json"), ",") + if name == "" { + name = field.Name + } + fields[name] = field.Type + } + var obj map[string]jsontext.Value if json.Unmarshal(raw, &obj) != nil { return "" } - if request, ok := obj["request"]; ok { - if key := registryObjectKey(request); key != "" { + if request, ok := obj["request"]; ok && fields["request"] != nil { + if key := registryObjectKey(request, fields["request"]); key != "" { return key } } - if boot, ok := obj["boot_id"]; ok { + if boot, ok := obj["boot_id"]; ok && fields["boot_id"] != nil { return string(boot) + "/" + string(obj["pid"]) + "/" + string(obj["start"]) } - if inode, ok := obj["inode"]; ok { + if inode, ok := obj["inode"]; ok && fields["inode"] != nil { return string(obj["device"]) + "/" + string(inode) } for _, key := range []string{"id", "sandbox_id", "operation", "operation_id", "name", "number"} { - if value, ok := obj[key]; ok { + if value, ok := obj[key]; ok && fields[key] != nil { return key + string(value) } } - if volume, ok := obj["volume"]; ok { - return registryObjectKey(volume) + if volume, ok := obj["volume"]; ok && fields["volume"] != nil { + return registryObjectKey(volume, fields["volume"]) } return "" } diff --git a/sandboxd/pool/volume_registry_fields_test.go b/sandboxd/pool/volume_registry_fields_test.go index 7e54d3c4..d17812b4 100644 --- a/sandboxd/pool/volume_registry_fields_test.go +++ b/sandboxd/pool/volume_registry_fields_test.go @@ -50,6 +50,11 @@ func TestManagedRegistryPreservesSupportedAdditiveFields(t *testing.T) { } for _, object := range objects { object["future_note"] = map[string]any{"value": "preserve"} + for _, field := range []string{"id", "name", "operation", "request"} { + if _, known := object[field]; !known { + object[field] = "future-extension" + } + } } raw, err = json.Marshal(doc) if err != nil { @@ -120,3 +125,56 @@ func TestManagedRegistryExtensionsFollowObjectIdentity(t *testing.T) { }) } } + +func TestManagedAdditiveMetadataCannotBecomeObjectIdentity(t *testing.T) { + cfg := managedConfig(t) + m := openManagedTest(t, cfg, newManagedEngine()) + createManagedTest(t, m, "data") + if err := m.Close(); err != nil { + t.Fatal(err) + } + path := filepath.Join(cfg.ManagedVolumeDir, "registry.json") + raw, err := os.ReadFile(path) + if err != nil { + t.Fatal(err) + } + var doc map[string]any + if err = json.Unmarshal(raw, &doc); err != nil { + t.Fatal(err) + } + doc["id"] = "future-registry-instance" + doc["future_note"] = "preserve" + raw, err = json.Marshal(doc) + if err != nil { + t.Fatal(err) + } + if err = os.WriteFile(path, raw, 0o600); err != nil { + t.Fatal(err) + } + r, err := openVolumeRegistry(cfg.ManagedVolumeDir, "node") + if err != nil { + t.Fatal(err) + } + defer r.close() + if r.fault != nil { + t.Fatalf("extension was not accepted: %v", r.fault) + } + r.mu.Lock() + err = r.commitAndRelock() + r.mu.Unlock() + if err != nil { + t.Fatal(err) + } + raw, err = os.ReadFile(path) + if err != nil { + t.Fatal(err) + } + var got map[string]any + if err = json.Unmarshal(raw, &got); err != nil { + t.Fatal(err) + } + t.Logf("persisted id=%v future_note=%v", got["id"], got["future_note"]) + if got["id"] != doc["id"] || got["future_note"] != doc["future_note"] { + t.Error("accepted additive registry metadata was discarded by identity heuristic") + } +} From 56f09a3be26a05676c84e457d69abc46391df763 Mon Sep 17 00:00:00 2001 From: minatoaquaMK2 Date: Tue, 20 Oct 2026 11:12:31 +0000 Subject: [PATCH 22/75] pool: retain stale catalog holds during managed recovery faults --- sandboxd/pool/managed_foundation_test.go | 56 +++++++++++++++++++++++- sandboxd/pool/reconcile.go | 48 +++++++++++++++++--- 2 files changed, 96 insertions(+), 8 deletions(-) diff --git a/sandboxd/pool/managed_foundation_test.go b/sandboxd/pool/managed_foundation_test.go index 1a891b84..edbbc2e5 100644 --- a/sandboxd/pool/managed_foundation_test.go +++ b/sandboxd/pool/managed_foundation_test.go @@ -709,7 +709,7 @@ func TestManagedRecoveryFencesAgentSocket(t *testing.T) { m.mu.Lock() m.addClaimLocked(ordinary) m.mu.Unlock() - m.fenceManagedRecovery(t.Context(), errors.New("inventory unavailable"), map[string]*types.Sandbox{sb.ID: sb, ordinary.ID: ordinary}, nil) + m.fenceManagedRecovery(t.Context(), errors.New("inventory unavailable"), map[string]*types.Sandbox{sb.ID: sb, ordinary.ID: ordinary}, nil, true) if socket, err := m.AgentSocket(sb.ID, sb.Token); err == nil || socket != "" { t.Fatalf("uncertain managed claim exposes socket %q, error=%v", socket, err) } @@ -721,3 +721,57 @@ func TestManagedRecoveryFencesAgentSocket(t *testing.T) { t.Fatalf("recorded uncertain VM exposes socket after empty volume projection: %q, %v", socket, err) } } + +func TestManagedFaultRetainsStaleCatalogWriter(t *testing.T) { + for _, unknown := range []bool{false, true} { + name := "observed" + if unknown { + name = "unknown" + } + t.Run(name, func(t *testing.T) { + cfg := managedConfig(t) + path := filepath.Join(filepath.Dir(cfg.DataDir), "catalog.img") + if err := os.WriteFile(path, []byte("catalog"), 0o600); err != nil { + t.Fatal(err) + } + freePath := filepath.Join(filepath.Dir(cfg.DataDir), "other.img") + if err := os.WriteFile(freePath, []byte("other"), 0o600); err != nil { + t.Fatal(err) + } + cfg.Volumes = []config.VolumeSpec{{Name: "catalog", Path: path, Writable: true}, {Name: "other", Path: freePath, Writable: true}} + e := newManagedEngine() + m := openManagedTest(t, cfg, e) + old := foundationClaim(t, m, "catalog") + e.disks = map[string][]types.VMDisk{old.VMName: {{Name: "catalog", Path: path, ReadOnly: false}}} + m.releaseDelay = time.Hour + if err := m.Release(t.Context(), old.ID, Cred{Operator: true}); err != nil { + t.Fatal(err) + } + if _, present := e.vms[old.VMName]; !present { + t.Fatal("delayed release removed VM") + } + if err := m.Close(); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(cfg.ManagedVolumeDir, "registry.json"), []byte("{"), 0o600); err != nil { + t.Fatal(err) + } + e.missingInventory.Store(unknown) + m = openManagedTest(t, cfg, e) + if err := m.Reconcile(t.Context()); err != nil { + t.Fatal(err) + } + if _, present := e.vms[old.VMName]; !present { + t.Fatal("unknown stale writer was swept") + } + + if newer, err := m.ClaimProvision(t.Context(), testKey, managedOptions("catalog")); err == nil { + t.Fatalf("second writer %s admitted while old VM %s remains", newer.VMName, old.VMName) + } + if _, err := m.ClaimProvision(t.Context(), testKey, managedOptions("other")); (err != nil) != unknown { + t.Fatalf("unrelated catalog availability: unknown=%v error=%v", unknown, err) + } + foundationClaim(t, m) + }) + } +} diff --git a/sandboxd/pool/reconcile.go b/sandboxd/pool/reconcile.go index 2cb0489e..258329df 100644 --- a/sandboxd/pool/reconcile.go +++ b/sandboxd/pool/reconcile.go @@ -50,7 +50,7 @@ func (m *Manager) Reconcile(ctx context.Context) error { } if managedErr != nil { claims = originalClaims - m.fenceManagedRecovery(ctx, managedErr, claims, live) + m.fenceManagedRecovery(ctx, managedErr, claims, live, inventoryErr == nil) for name := range live { owned[name] = true } @@ -205,11 +205,25 @@ func (m *Manager) holdStaleVolumes(ctx context.Context, name string, rec types.V if rec.State != vmStateRunning { return nil } - inspected, _, err := m.eng.Inspect(ctx, name) + holds, err := m.inspectStaleVolumes(ctx, name) if err != nil { log.WithFunc("pool.holdStaleVolumes").Warnf(ctx, "inspect stale vm %s, its volumes stay unheld: %v", name, err) return nil } + m.mu.Lock() + m.adoptVolumes(holds) + m.mu.Unlock() + return holds +} + +func (m *Manager) inspectStaleVolumes(ctx context.Context, name string) ([]types.Volume, error) { + inspected, found, err := m.eng.Inspect(ctx, name) + if err != nil { + return nil, err + } + if !found { + return nil, fmt.Errorf("stale VM %s absent from inspection", name) + } holds := make([]types.Volume, len(inspected.AttachedDevices.Disks)) for i, disk := range inspected.AttachedDevices.Disks { holds[i].Name = disk.Name @@ -217,10 +231,7 @@ func (m *Manager) holdStaleVolumes(ctx context.Context, name string, rec types.V holds[i].Mode = types.VolumeModeRW } } - m.mu.Lock() - m.adoptVolumes(holds) - m.mu.Unlock() - return holds + return holds, nil } // resyncEgress re-locks adopted egress claims, quarantining any it cannot lock. @@ -379,7 +390,7 @@ func (m *Manager) sweepOrphanSnapshots(ctx context.Context, snaps []string, snap } } -func (m *Manager) fenceManagedRecovery(ctx context.Context, err error, claims map[string]*types.Sandbox, live map[string]types.VMRecord) { +func (m *Manager) fenceManagedRecovery(ctx context.Context, err error, claims map[string]*types.Sandbox, live map[string]types.VMRecord, inventoryKnown bool) { r := m.managed r.mu.Lock() r.fault = err @@ -396,8 +407,31 @@ func (m *Manager) fenceManagedRecovery(ctx context.Context, err error, claims ma } } r.mu.Unlock() + known := make(map[string]bool, len(claims)) + for _, sb := range claims { + known[sb.VMName] = true + } + var retained []types.Volume + for name, rec := range live { + if known[name] || !strings.HasPrefix(name, vmPrefix) || rec.State != vmStateRunning { + continue + } + holds, inspectErr := m.inspectStaleVolumes(ctx, name) + if inspectErr != nil { + inventoryKnown = false + log.WithFunc("pool.Reconcile").Error(ctx, inspectErr, "stale volume inventory unavailable; catalog admissions fenced") + continue + } + retained = append(retained, holds...) + } + if !inventoryKnown { + for name := range m.volumes { + retained = append(retained, types.Volume{Name: name, Mode: types.VolumeModeRW}) + } + } m.mu.Lock() m.volumeAdmission = map[string]volumeHolders{} + m.adoptVolumes(retained) m.mu.Unlock() log.WithFunc("pool.Reconcile").Error(ctx, err, "managed storage fenced; retaining claims and VMs") } From cc2c6d7fa46cd773fa2025d061602d30485f0b94 Mon Sep 17 00:00:00 2001 From: minatoaquaMK2 Date: Tue, 20 Oct 2026 11:18:13 +0000 Subject: [PATCH 23/75] pool: contain uncertain managed egress guests during recovery --- docs/managed-volumes.md | 8 +++- sandboxd/pool/egress_test.go | 72 +++++++++++++++++++++++++++++++++- sandboxd/pool/liveness_test.go | 2 +- sandboxd/pool/reconcile.go | 65 ++++++++++++++++++++++-------- 4 files changed, 126 insertions(+), 21 deletions(-) diff --git a/docs/managed-volumes.md b/docs/managed-volumes.md index 0d270cf6..fc0a66ac 100644 --- a/docs/managed-volumes.md +++ b/docs/managed-volumes.md @@ -296,6 +296,10 @@ managed retirement persistence fails. A stubborn VM or unavailable registry can keep the hold for operator recovery. An uncooperative VMM may still retain the hold. +An uncertain managed claim still requires host NIC-lock recovery. If that lock +cannot be established, the daemon stops the VM while retaining its claim and +backing. Startup recovery fails if the VM cannot be confirmed stopped. + After restart, the persisted egress-quarantine failure marker resumes retirement and removal. Ordinary failed claims keep their existing ownership until release. Unclaimed stopped VM records are removed only after the recorded process is @@ -304,7 +308,9 @@ while the node continues serving ordinary volume-free claims. It preserves uncertain claims, VM records, snapshots, and backing files. A matching healthy durable claim takes precedence over a legacy retiring hold. Unsafe configured directories and failure to acquire the daemon's directory lock still prevent -startup. +startup. Catalog holds from unclaimed running VMs are reconstructed from runtime +disk inventory. If that inventory is unavailable, catalog admissions remain +fenced too; ordinary volume-free claims can continue. An interrupted allocation with no recorded backing inode requires operator quarantine. The API does not delete or adopt that unproven file. Preserve the diff --git a/sandboxd/pool/egress_test.go b/sandboxd/pool/egress_test.go index b1ab6b57..d6334331 100644 --- a/sandboxd/pool/egress_test.go +++ b/sandboxd/pool/egress_test.go @@ -1,6 +1,7 @@ package pool import ( + "context" "crypto/tls" "errors" "net" @@ -313,7 +314,7 @@ func TestQuarantineFailedRemoveStaysUnswept(t *testing.T) { var gotKeep map[string]bool withOutbound(t, m, func(o *outbound.Options) { o.Sweep = func(keep map[string]bool) error { gotKeep = keep; return nil } }) - m.resyncEgress(t.Context(), live, removed, true) + _ = m.resyncEgress(t.Context(), live, removed, true) if removed["sbx-q1"] { t.Error("failed remove marked the VM gone; the sweep would drop its lock table") @@ -755,7 +756,7 @@ func TestUnknownInventoryPreservesEgressTables(t *testing.T) { o.LockNIC = true o.Sweep = func(map[string]bool) error { swept = true; return nil } }) - m.resyncEgress(t.Context(), map[string]types.VMRecord{}, map[string]bool{}, false) + _ = m.resyncEgress(t.Context(), map[string]types.VMRecord{}, map[string]bool{}, false) if live, present := m.Sandbox(sb.ID); present && live.Failed == "" { t.Fatal("unknown inventory left an egress claim active without a verified lock") } @@ -763,3 +764,70 @@ func TestUnknownInventoryPreservesEgressTables(t *testing.T) { t.Fatal("unknown inventory swept potentially live egress locks") } } + +func TestManagedFaultContainsRunningEgressVM(t *testing.T) { + for _, outcome := range []string{"stopped", "stop error", "false acknowledgement"} { + t.Run(outcome, func(t *testing.T) { + cfg := managedConfig(t) + cfg.Bridges = []string{"sbxbr0"} + cfg.Pools = []config.PoolSpec{{PoolKey: egKey, Warm: 1}} + e := newManagedEngine() + m := openManagedTest(t, cfg, e) + withOutbound(t, m, func(o *outbound.Options) { o.LockNIC = false }) + v := createManagedTest(t, m, "data") + backing := filepath.Join(cfg.ManagedVolumeDir, "disks", v.ID+".raw") + before, err := os.Stat(backing) + if err != nil { + t.Fatal(err) + } + sb, err := m.ClaimProvision(t.Context(), egKey, managedOptions("data")) + if err != nil { + t.Fatal(err) + } + if err = m.Close(); err != nil { + t.Fatal(err) + } + if err = os.WriteFile(filepath.Join(cfg.ManagedVolumeDir, "registry.json"), []byte("{"), 0o600); err != nil { + t.Fatal(err) + } + m = openManagedTest(t, cfg, e) + m.eng = &egressRecoveryEngine{managedTestEngine: e, outcome: outcome} + withOutbound(t, m, func(o *outbound.Options) { + o.LockNIC = true + o.Lock = func(string) error { return errors.New("lock unavailable") } + o.Sweep = func(map[string]bool) error { return nil } + }) + err = m.Reconcile(t.Context()) + if (err != nil) != (outcome != "stopped") { + t.Fatalf("containment outcome=%s error=%v", outcome, err) + } + rec, found, err := e.Inspect(t.Context(), sb.VMName) + if err != nil || !found || (rec.State == "stopped") != (outcome == "stopped") { + t.Fatalf("containment=%s found=%v state=%s error=%v", outcome, found, rec.State, err) + } + if current, present := m.Sandbox(sb.ID); !present || current.Failed == "" { + t.Fatal("uncertain owner claim was lost or served") + } + after, err := os.Stat(backing) + if err != nil || !os.SameFile(before, after) || before.Size() != after.Size() { + t.Fatal("containment changed volume backing", err) + } + }) + } +} + +type egressRecoveryEngine struct { + *managedTestEngine + outcome string +} + +func (e *egressRecoveryEngine) Stop(ctx context.Context, name string) error { + switch e.outcome { + case "stop error": + return errors.New("stop failed") + case "false acknowledgement": + return nil + default: + return e.managedTestEngine.Stop(ctx, name) + } +} diff --git a/sandboxd/pool/liveness_test.go b/sandboxd/pool/liveness_test.go index 83090a4a..cee7e288 100644 --- a/sandboxd/pool/liveness_test.go +++ b/sandboxd/pool/liveness_test.go @@ -409,7 +409,7 @@ func TestResyncRecordsADownEgressLaneTap(t *testing.T) { o.Sweep = func(map[string]bool) error { return nil } }) - m.resyncEgress(t.Context(), live, map[string]bool{}, true) + _ = m.resyncEgress(t.Context(), live, map[string]bool{}, true) m.mu.Lock() kept := m.claimed[sb.ID] == sb diff --git a/sandboxd/pool/reconcile.go b/sandboxd/pool/reconcile.go index 258329df..89cadb24 100644 --- a/sandboxd/pool/reconcile.go +++ b/sandboxd/pool/reconcile.go @@ -3,6 +3,7 @@ package pool import ( "context" "encoding/json/v2" + "errors" "fmt" "maps" "os" @@ -119,10 +120,10 @@ func (m *Manager) Reconcile(ctx context.Context) error { if managedErr == nil { m.sweepOrphanSnapshots(ctx, snaps, snapsErr, referenced) } - m.resyncEgress(ctx, live, removed, inventoryErr == nil) + egressErr := m.resyncEgress(ctx, live, removed, inventoryErr == nil) m.warnDeniedClaims(ctx) logger.Infof(ctx, "adopted %d claims, %d VMs live", len(m.claimed), len(live)) - return saveErr + return errors.Join(saveErr, egressErr) } func (m *Manager) sweepLocalStaging(ctx context.Context, pooled map[string]bool) { @@ -235,37 +236,39 @@ func (m *Manager) inspectStaleVolumes(ctx context.Context, name string) ([]types } // resyncEgress re-locks adopted egress claims, quarantining any it cannot lock. -func (m *Manager) resyncEgress(ctx context.Context, live map[string]types.VMRecord, removed map[string]bool, inventoryKnown bool) { +func (m *Manager) resyncEgress(ctx context.Context, live map[string]types.VMRecord, removed map[string]bool, inventoryKnown bool) error { logger := log.WithFunc("pool.resyncEgress") + var containmentErr error now := time.Now() locked, lockedErr := m.out.LockedTaps() var quarantine []*types.Sandbox for _, sb := range m.claimed { + uncertain := m.managedClaimUncertain(sb) sb.TouchAt(now) // a hibernated or archived claim has no guest to serve; its wake arms the door - if sb.HibernateSnap != "" || sb.PendingSnap != "" || sb.ArchiveCk != "" || sb.Failed != "" { + if !uncertain && (sb.HibernateSnap != "" || sb.PendingSnap != "" || sb.ArchiveCk != "" || sb.Failed != "") { continue } + var lockErr error if m.out.LocksNIC(sb.Key) { tap := m.readoptEgressTap(sb, live) // a down egress-lane VM has no guest to lock or mark; the liveness pass fails it if rec, known := live[sb.VMName]; known && rec.State != vmStateRunning { continue } - if tap == "" { - logger.Errorf(ctx, errNoEgressTap, "egress claim %s has no lockable tap; quarantining", sb.ID) - quarantine = append(quarantine, sb) - continue - } - err := lockedErr - if err == nil && !locked[tap] { - err = m.out.Relock(ctx, sb, tap) - } - if err != nil { - logger.Errorf(ctx, err, "ensure egress lock %s; quarantining", sb.ID) + lockErr = m.ensureEgressLock(ctx, sb, tap, locked, lockedErr) + } + if lockErr != nil { + logger.Errorf(ctx, lockErr, "ensure egress lock %s; quarantining", sb.ID) + if uncertain { + containmentErr = errors.Join(containmentErr, m.stopUncertainEgress(ctx, sb)) + } else { quarantine = append(quarantine, sb) - continue } + continue + } + if sb.Failed != "" { + continue } if proxyErr := m.out.ArmProxy(ctx, sb); proxyErr != nil { logger.Errorf(ctx, proxyErr, "arm egress proxy %s", sb.ID) @@ -281,7 +284,7 @@ func (m *Manager) resyncEgress(ctx context.Context, live map[string]types.VMReco } } if !inventoryKnown { - return + return containmentErr } // keep every still-running VM's tap; sweep only a confirmed-gone VM's table. for name, rec := range live { @@ -292,6 +295,34 @@ func (m *Manager) resyncEgress(ctx context.Context, live map[string]types.VMReco if sweepErr := m.out.Sweep(keep); sweepErr != nil { logger.Warnf(ctx, "sweep orphan egress tables: %v", sweepErr) } + return containmentErr +} + +func (m *Manager) ensureEgressLock(ctx context.Context, sb *types.Sandbox, tap string, locked map[string]bool, inventoryErr error) error { + if tap == "" { + return errNoEgressTap + } + if inventoryErr != nil { + return inventoryErr + } + if locked[tap] { + return nil + } + return m.out.Relock(ctx, sb, tap) +} + +func (m *Manager) stopUncertainEgress(ctx context.Context, sb *types.Sandbox) error { + if err := m.eng.Stop(ctx, sb.VMName); err != nil { + return fmt.Errorf("stop uncontained egress VM %s: %w", sb.VMName, err) + } + rec, found, err := m.eng.Inspect(ctx, sb.VMName) + if err != nil { + return fmt.Errorf("confirm egress VM %s stopped: %w", sb.VMName, err) + } + if found && rec.State != "stopped" { + return fmt.Errorf("egress VM %s is not confirmed stopped", sb.VMName) + } + return nil } const egressQuarantineFailure = "sandbox quarantined after egress recovery failure" From 25625b0d936495a0fa937aae1028d5d746923ed3 Mon Sep 17 00:00:00 2001 From: minatoaquaMK2 Date: Tue, 20 Oct 2026 11:31:00 +0000 Subject: [PATCH 24/75] pool: contain unclaimed VMs during managed recovery faults --- docs/managed-volumes.md | 14 +++- sandboxd/pool/egress_test.go | 95 ++++++++++++++++++++++++ sandboxd/pool/managed_foundation_test.go | 39 ++++++++++ sandboxd/pool/reconcile.go | 27 ++++--- 4 files changed, 163 insertions(+), 12 deletions(-) diff --git a/docs/managed-volumes.md b/docs/managed-volumes.md index fc0a66ac..e5f82cbe 100644 --- a/docs/managed-volumes.md +++ b/docs/managed-volumes.md @@ -298,13 +298,21 @@ the hold. An uncertain managed claim still requires host NIC-lock recovery. If that lock cannot be established, the daemon stops the VM while retaining its claim and -backing. Startup recovery fails if the VM cannot be confirmed stopped. +backing. Startup recovery fails if the VM cannot be confirmed stopped. During +managed recovery faults, an unclaimed running `sbx-` VM has no durable network +policy. +The daemon stops it and confirms the result through runtime inspection, even +when its disk inventory is empty. It retains the VM record, volume charges, +registry, and backing files. A failed stop or inspection prevents startup. +A VM that rejects stopping can keep running. A startup error does not provide +network isolation. After restart, the persisted egress-quarantine failure marker resumes retirement and removal. Ordinary failed claims keep their existing ownership until release. Unclaimed stopped VM records are removed only after the recorded process is -proved gone. Managed metadata or inventory uncertainty fences managed work -while the node continues serving ordinary volume-free claims. It preserves +proved gone. Managed metadata uncertainty fences managed work while the node +continues serving ordinary volume-free claims. A failed VM listing prevents startup, +because it can hide unclaimed running VMs that require containment. It preserves uncertain claims, VM records, snapshots, and backing files. A matching healthy durable claim takes precedence over a legacy retiring hold. Unsafe configured directories and failure to acquire the daemon's directory lock still prevent diff --git a/sandboxd/pool/egress_test.go b/sandboxd/pool/egress_test.go index d6334331..08608074 100644 --- a/sandboxd/pool/egress_test.go +++ b/sandboxd/pool/egress_test.go @@ -831,3 +831,98 @@ func (e *egressRecoveryEngine) Stop(ctx context.Context, name string) error { return e.managedTestEngine.Stop(ctx, name) } } + +func TestManagedFaultContainsUnclaimedVM(t *testing.T) { + for _, tc := range []struct { + name string + managed bool + outcome string + }{ + {name: "managed", managed: true}, + {name: "volume-free"}, + {name: "stop error", managed: true, outcome: "stop error"}, + {name: "false acknowledgement", managed: true, outcome: "false acknowledgement"}, + {name: "inspection error", managed: true, outcome: "inspection error"}, + } { + t.Run(tc.name, func(t *testing.T) { + cfg := managedConfig(t) + cfg.Bridges = []string{"sbxbr0"} + cfg.Pools = []config.PoolSpec{{PoolKey: egKey, Warm: 1}, {PoolKey: testKey, Warm: 1}} + e := newManagedEngine() + e.tap = "orphan-tap" + m := openManagedTest(t, cfg, e) + withOutbound(t, m, func(o *outbound.Options) { o.LockNIC = false }) + v := createManagedTest(t, m, "data") + backing := filepath.Join(cfg.ManagedVolumeDir, "disks", v.ID+".raw") + before, err := os.Stat(backing) + if err != nil { + t.Fatal(err) + } + opts := managedOptions() + if tc.managed { + opts = managedOptions("data") + } + sb, err := m.ClaimProvision(t.Context(), egKey, opts) + if err != nil { + t.Fatal(err) + } + if tc.managed { + e.disks = map[string][]types.VMDisk{sb.VMName: {{Name: "data", Path: backing}}} + } + ordinary := foundationClaim(t, m) + e.vms["foreign-vm"] = "/foreign/socket" + m.releaseDelay = time.Hour + if err = m.Release(t.Context(), sb.ID, Cred{Operator: true}); err != nil { + t.Fatal(err) + } + if err = m.Close(); err != nil { + t.Fatal(err) + } + registry := filepath.Join(cfg.ManagedVolumeDir, "registry.json") + if err = os.WriteFile(registry, []byte("{"), 0o600); err != nil { + t.Fatal(err) + } + m = openManagedTest(t, cfg, e) + m.eng = &egressRecoveryEngine{managedTestEngine: e, outcome: tc.outcome} + withOutbound(t, m, func(o *outbound.Options) { + o.LockNIC = true + o.Lock = func(string) error { return nil } + o.Sweep = func(map[string]bool) error { return nil } + }) + err = m.Reconcile(t.Context()) + if (err != nil) != (tc.outcome != "") { + t.Errorf("containment outcome=%q startup error=%v", tc.outcome, err) + } + rec, found, err := e.Inspect(t.Context(), sb.VMName) + if err != nil || !found || (rec.State == "stopped") != (tc.outcome == "" || tc.outcome == "inspection error") { + t.Errorf("unclaimed containment outcome=%q found=%v state=%s error=%v", tc.outcome, found, rec.State, err) + } + if _, present := m.Sandbox(sb.ID); present { + t.Error("released claim was recreated") + } + for _, name := range []string{ordinary.VMName, "foreign-vm"} { + rec, found, err = e.Inspect(t.Context(), name) + if err != nil || !found || rec.State != vmStateRunning { + t.Errorf("unaffected VM %s found=%v state=%s error=%v", name, found, rec.State, err) + } + } + after, err := os.Stat(backing) + if err != nil || !os.SameFile(before, after) || before.Size() != after.Size() { + t.Error("containment changed volume backing", err) + } + if raw, err := os.ReadFile(registry); err != nil || string(raw) != "{" { + t.Errorf("containment rewrote fenced registry: %q %v", raw, err) + } + if len(m.pendingRemovals) != 0 { + t.Error("uncertain owner queued for destructive removal") + } + }) + } +} + +func (e *egressRecoveryEngine) Inspect(ctx context.Context, name string) (types.VMRecord, bool, error) { + if e.outcome == "inspection error" { + return types.VMRecord{}, false, errors.New("inspection unavailable") + } + return e.managedTestEngine.Inspect(ctx, name) +} diff --git a/sandboxd/pool/managed_foundation_test.go b/sandboxd/pool/managed_foundation_test.go index edbbc2e5..f3a514d1 100644 --- a/sandboxd/pool/managed_foundation_test.go +++ b/sandboxd/pool/managed_foundation_test.go @@ -2,6 +2,7 @@ package pool import ( "bytes" + "context" "errors" "os" "os/exec" @@ -775,3 +776,41 @@ func TestManagedFaultRetainsStaleCatalogWriter(t *testing.T) { }) } } + +func TestManagedListFailureRetainsStateAndRejectsStartup(t *testing.T) { + cfg := managedConfig(t) + e := newManagedEngine() + m := openManagedTest(t, cfg, e) + createManagedTest(t, m, "data") + sb := foundationClaim(t, m, "data") + ordinary := foundationClaim(t, m) + if err := m.Close(); err != nil { + t.Fatal(err) + } + m = openManagedTest(t, cfg, e) + inventoryErr := errors.New("VM inventory unavailable") + m.eng = &failedListEngine{managedTestEngine: e, err: inventoryErr} + if err := m.Reconcile(t.Context()); !errors.Is(err, inventoryErr) { + t.Errorf("incomplete VM inventory permitted startup: %v", err) + } + for _, id := range []string{sb.ID, ordinary.ID} { + if _, present := m.Sandbox(id); !present { + t.Errorf("inventory failure lost durable claim %s", id) + } + } + if _, held := m.managedHold(sb.ID); !held { + t.Error("inventory failure discharged managed reservation") + } + if rec, found, err := e.Inspect(t.Context(), sb.VMName); err != nil || !found || rec.State != vmStateRunning { + t.Errorf("inventory failure destroyed retained VM: found=%v state=%s error=%v", found, rec.State, err) + } +} + +type failedListEngine struct { + *managedTestEngine + err error +} + +func (e *failedListEngine) List(context.Context) ([]types.VMRecord, error) { + return nil, e.err +} diff --git a/sandboxd/pool/reconcile.go b/sandboxd/pool/reconcile.go index 89cadb24..d6999d12 100644 --- a/sandboxd/pool/reconcile.go +++ b/sandboxd/pool/reconcile.go @@ -123,7 +123,7 @@ func (m *Manager) Reconcile(ctx context.Context) error { egressErr := m.resyncEgress(ctx, live, removed, inventoryErr == nil) m.warnDeniedClaims(ctx) logger.Infof(ctx, "adopted %d claims, %d VMs live", len(m.claimed), len(live)) - return errors.Join(saveErr, egressErr) + return errors.Join(saveErr, egressErr, inventoryErr) } func (m *Manager) sweepLocalStaging(ctx context.Context, pooled map[string]bool) { @@ -235,14 +235,16 @@ func (m *Manager) inspectStaleVolumes(ctx context.Context, name string) ([]types return holds, nil } -// resyncEgress re-locks adopted egress claims, quarantining any it cannot lock. +// resyncEgress re-locks adopted egress claims and contains uncertain retained VMs. func (m *Manager) resyncEgress(ctx context.Context, live map[string]types.VMRecord, removed map[string]bool, inventoryKnown bool) error { logger := log.WithFunc("pool.resyncEgress") var containmentErr error now := time.Now() locked, lockedErr := m.out.LockedTaps() var quarantine []*types.Sandbox + claimedVMs := make(map[string]bool, len(m.claimed)) for _, sb := range m.claimed { + claimedVMs[sb.VMName] = true uncertain := m.managedClaimUncertain(sb) sb.TouchAt(now) // a hibernated or archived claim has no guest to serve; its wake arms the door @@ -261,7 +263,7 @@ func (m *Manager) resyncEgress(ctx context.Context, live map[string]types.VMReco if lockErr != nil { logger.Errorf(ctx, lockErr, "ensure egress lock %s; quarantining", sb.ID) if uncertain { - containmentErr = errors.Join(containmentErr, m.stopUncertainEgress(ctx, sb)) + containmentErr = errors.Join(containmentErr, m.stopUncertainEgress(ctx, sb.VMName)) } else { quarantine = append(quarantine, sb) } @@ -274,6 +276,13 @@ func (m *Manager) resyncEgress(ctx context.Context, live map[string]types.VMReco logger.Errorf(ctx, proxyErr, "arm egress proxy %s", sb.ID) } } + for name, rec := range live { + if claimedVMs[name] || !strings.HasPrefix(name, vmPrefix) || rec.State != vmStateRunning || !m.managedUncertainVM(name) { + continue + } + // Without a durable claim, the retained VM's network lane is unknown. + containmentErr = errors.Join(containmentErr, m.stopUncertainEgress(ctx, name)) + } // quarantine before the sweep so a failed remove's still-running tap is kept. keep := make(map[string]bool, len(live)) for _, sb := range quarantine { @@ -311,16 +320,16 @@ func (m *Manager) ensureEgressLock(ctx context.Context, sb *types.Sandbox, tap s return m.out.Relock(ctx, sb, tap) } -func (m *Manager) stopUncertainEgress(ctx context.Context, sb *types.Sandbox) error { - if err := m.eng.Stop(ctx, sb.VMName); err != nil { - return fmt.Errorf("stop uncontained egress VM %s: %w", sb.VMName, err) +func (m *Manager) stopUncertainEgress(ctx context.Context, name string) error { + if err := m.eng.Stop(ctx, name); err != nil { + return fmt.Errorf("stop uncontained egress VM %s: %w", name, err) } - rec, found, err := m.eng.Inspect(ctx, sb.VMName) + rec, found, err := m.eng.Inspect(ctx, name) if err != nil { - return fmt.Errorf("confirm egress VM %s stopped: %w", sb.VMName, err) + return fmt.Errorf("confirm egress VM %s stopped: %w", name, err) } if found && rec.State != "stopped" { - return fmt.Errorf("egress VM %s is not confirmed stopped", sb.VMName) + return fmt.Errorf("egress VM %s is not confirmed stopped", name) } return nil } From 7983fea566a5790457f300fd8e55a884d05edc58 Mon Sep 17 00:00:00 2001 From: minatoaquaMK2 Date: Tue, 20 Oct 2026 11:35:10 +0000 Subject: [PATCH 25/75] pool: remove redundant recovery summary --- sandboxd/pool/reconcile.go | 1 - 1 file changed, 1 deletion(-) diff --git a/sandboxd/pool/reconcile.go b/sandboxd/pool/reconcile.go index d6999d12..9e09ba67 100644 --- a/sandboxd/pool/reconcile.go +++ b/sandboxd/pool/reconcile.go @@ -235,7 +235,6 @@ func (m *Manager) inspectStaleVolumes(ctx context.Context, name string) ([]types return holds, nil } -// resyncEgress re-locks adopted egress claims and contains uncertain retained VMs. func (m *Manager) resyncEgress(ctx context.Context, live map[string]types.VMRecord, removed map[string]bool, inventoryKnown bool) error { logger := log.WithFunc("pool.resyncEgress") var containmentErr error From c408de9f3369b9642456111ca1f52e170a8dca8f Mon Sep 17 00:00:00 2001 From: minatoaquaMK2 Date: Tue, 20 Oct 2026 11:44:34 +0000 Subject: [PATCH 26/75] pool: wait for managed growth test expiry workers --- sandboxd/pool/managed_grow_test.go | 58 +++++++++++++++++++++++++----- 1 file changed, 50 insertions(+), 8 deletions(-) diff --git a/sandboxd/pool/managed_grow_test.go b/sandboxd/pool/managed_grow_test.go index faf6be85..94b5ce62 100644 --- a/sandboxd/pool/managed_grow_test.go +++ b/sandboxd/pool/managed_grow_test.go @@ -178,6 +178,20 @@ func growRequest() types.VolumeGrow { return types.VolumeGrow{SizeGiB: 2, OperationID: "b51b3d2b-4ccc-41d5-b430-3046340c30b0"} } +func startGrowthTestReap(t *testing.T, m *Manager) func() { + t.Helper() + maint, refill := m.maintSem, m.refillSem + m.maintSem, m.refillSem = make(chan struct{}), make(chan struct{}) + m.reapOnce(t.Context()) + <-m.maintSem + <-m.refillSem + return sync.OnceFunc(func() { + m.refillSem <- struct{}{} + m.maintSem <- struct{}{} + m.maintSem, m.refillSem = maint, refill + }) +} + func TestManagedGrowthCommitReplayCASAndRestart(t *testing.T) { m, e, sb, cfg, v := newGrowthTest(t) req := growRequest() @@ -272,10 +286,15 @@ func TestManagedGrowthUnknownFencesLifecycleUntilExplicitExactStop(t *testing.T) m.mu.Lock() sb.Deadline = time.Now().Add(-time.Second) m.mu.Unlock() - m.reapOnce(t.Context()) + finishReap := startGrowthTestReap(t, m) + defer finishReap() + finishReap() if _, ok := m.Sandbox(sb.ID); !ok { t.Fatal("ordinary reap dropped grow owner") } + if _, renewErr := m.Renew(t.Context(), sb.ID, Cred{Operator: true}, time.Hour, ""); renewErr != nil { + t.Fatal(renewErr) + } if m.removeVM(t.Context(), sb.VMName) || m.finishManaged(sb.ID, nil) { t.Fatal("destructive fallback escaped guard") } @@ -334,12 +353,22 @@ func TestManagedGrowthAdmissionSerializesOrdinaryReleaseAndReap(t *testing.T) { m, e, sb, _, v := newGrowthTest(t) e.entered = make(chan struct{}) e.resume = make(chan struct{}) - done := make(chan error, 1) - go func() { _, err := m.GrowVolume(t.Context(), "data", v.ID, growRequest(), "t1"); done <- err }() + done := make(chan struct{}) + var growErr error + go func() { + _, growErr = m.GrowVolume(t.Context(), "data", v.ID, growRequest(), "t1") + close(done) + }() + resumeGrowth := sync.OnceValue(func() error { + close(e.resume) + <-done + return growErr + }) + defer resumeGrowth() select { case <-e.entered: - case err := <-done: - t.Fatalf("grow did not dispatch: %v", err) + case <-done: + t.Fatalf("grow did not dispatch: %v", growErr) } if m.managedLifecycle(sb) { t.Fatal("test must cover release without Transition") @@ -348,14 +377,27 @@ func TestManagedGrowthAdmissionSerializesOrdinaryReleaseAndReap(t *testing.T) { m.mu.Lock() sb.Deadline = time.Now().Add(-time.Second) m.mu.Unlock() - m.reapOnce(t.Context()) + finishReap := startGrowthTestReap(t, m) + defer func() { + resumeGrowth() + finishReap() + }() if _, ok := m.Sandbox(sb.ID); !ok { t.Fatal("reap dropped grow owner") } - close(e.resume) - if err := <-done; err != nil { + if err := resumeGrowth(); err != nil { t.Fatal(err) } + finishReap() + if _, ok := m.Sandbox(sb.ID); ok { + t.Fatal("expiry retained claim after growth completed") + } + if _, held := m.managedHold(sb.ID); held { + t.Fatal("expiry retained volume hold after growth completed") + } + if _, present, err := e.Inspect(t.Context(), sb.VMName); err != nil || present { + t.Fatalf("expiry retained VM after growth completed: present=%t err=%v", present, err) + } } func TestManagedGrowthRejectsChangedGuestAndClosingRuntime(t *testing.T) { From 1dbec2be1018da20001adcb924d5b90cf48d5b5e Mon Sep 17 00:00:00 2001 From: minatoaquaMK2 Date: Tue, 20 Oct 2026 13:13:38 +0000 Subject: [PATCH 27/75] pool: Keep stopped managed claim recovery covered. --- sandboxd/pool/managed_foundation_test.go | 33 ++-- .../pool/managed_recovery_regression_test.go | 157 ++++++++++++++++++ 2 files changed, 179 insertions(+), 11 deletions(-) diff --git a/sandboxd/pool/managed_foundation_test.go b/sandboxd/pool/managed_foundation_test.go index f3a514d1..d88670cb 100644 --- a/sandboxd/pool/managed_foundation_test.go +++ b/sandboxd/pool/managed_foundation_test.go @@ -7,7 +7,6 @@ import ( "os" "os/exec" "path/filepath" - "strconv" "sync" "sync/atomic" "syscall" @@ -578,35 +577,47 @@ func TestManagedClaimPostRenameFailureFencesWithoutTeardown(t *testing.T) { } func TestManagedStoppedZeroPIDRequiresOriginalProcessGone(t *testing.T) { - for _, gone := range []bool{false, true} { - t.Run(strconv.FormatBool(gone), func(t *testing.T) { + for _, unknown := range []bool{false, true} { + name := "live" + if unknown { + name = "unreadable" + } + t.Run(name, func(t *testing.T) { cfg := managedConfig(t) e := newManagedEngine() m := openManagedTest(t, cfg, e) createManagedTest(t, m, "data") sb := foundationClaim(t, m, "data") h, _ := m.managedHold(sb.ID) - m.Close() + if err := m.Close(); err != nil { + t.Fatal(err) + } e.mu.Lock() e.stopped[sb.VMName] = true e.pids[sb.VMName] = 0 e.mu.Unlock() m = openManagedTest(t, cfg, e) m.managed.process = func(int) (processIdentity, error) { - if gone { - return processIdentity{}, os.ErrNotExist + if unknown { + return processIdentity{}, os.ErrPermission } return h.Process, nil } if err := m.Reconcile(t.Context()); err != nil { t.Fatal(err) } - _, held := m.managedHold(sb.ID) - if held == gone { - t.Fatalf("original process gone=%t but hold retained=%t", gone, held) + retained, held := m.managedHold(sb.ID) + if !held || retained.Process != h.Process || retained.State != holdClaimed { + t.Fatalf("uncertain original process lost ownership: %+v", retained) } - if !gone { - requireVolumeStatus(t, m.ManagedVolumeStatus(), 503) + requireVolumeStatus(t, m.ManagedVolumeStatus(), 503) + if _, live := m.Sandbox(sb.ID); !live { + t.Fatal("uncertain original process lost its durable claim") + } + e.mu.Lock() + defer e.mu.Unlock() + if len(e.removes) != 0 || len(e.starts) != 0 { + t.Fatalf("uncertain original process changed VM: removes=%v starts=%v", e.removes, e.starts) } }) } diff --git a/sandboxd/pool/managed_recovery_regression_test.go b/sandboxd/pool/managed_recovery_regression_test.go index cc065416..f59c2f30 100644 --- a/sandboxd/pool/managed_recovery_regression_test.go +++ b/sandboxd/pool/managed_recovery_regression_test.go @@ -5,6 +5,7 @@ import ( "os" "path/filepath" "testing" + "time" "github.com/cocoonstack/sandbox/sandboxd/types" ) @@ -226,3 +227,159 @@ func TestManagedStorePreservesBackingWhenCleanupAliasesDisks(t *testing.T) { }) } } + +func TestManagedStoppedClaimRetainsOwnershipOnRestart(t *testing.T) { + for _, failed := range []bool{false, true} { + name := "new-failure-release" + if failed { + name = "previous-failure-expiry" + } + t.Run(name, func(t *testing.T) { + cfg := managedConfig(t) + e := newManagedEngine() + m := openManagedTest(t, cfg, e) + v := createManagedTest(t, m, "data") + sb := foundationClaim(t, m, "data") + h, _ := m.managedHold(sb.ID) + backing := m.managed.path(volumeID(v.ID)) + identity, err := inspectBacking(backing) + if err != nil { + t.Fatal(err) + } + if failed { + sb.Failed = "ordinary failure requiring explicit release" + if err := m.store.commit(m.store.set(sb)); err != nil { + t.Fatal(err) + } + } + if err := m.Close(); err != nil { + t.Fatal(err) + } + e.mu.Lock() + e.stopped[sb.VMName] = true + e.pids[sb.VMName] = 0 + e.mu.Unlock() + for restart := range 2 { + m = openManagedTest(t, cfg, e) + if err := m.Reconcile(t.Context()); err != nil { + t.Fatal(err) + } + if _, live := m.Sandbox(sb.ID); !live { + t.Fatalf("restart %d discarded the durable stopped claim", restart) + } + runLiveness(t, m) + listed := m.Sandboxes("t1", "", nil) + if len(listed) != 1 || listed[0].ID != sb.ID || listed[0].Failed == "" { + t.Fatalf("restart %d did not list the failed owner: %+v", restart, listed) + } + retained, held := m.managedHold(sb.ID) + if !held || retained.State != holdClaimed || retained.Process != h.Process || retained.VMName != h.VMName || len(retained.Refs) != 1 || retained.Refs[0] != h.Refs[0] { + t.Fatalf("restart %d changed managed ownership: %+v", restart, retained) + } + if rec, present, err := e.Inspect(t.Context(), sb.VMName); err != nil || !present || rec.State != vmStateStopped || rec.PID != 0 { + t.Fatalf("restart %d changed stopped VM record: %+v present=%t err=%v", restart, rec, present, err) + } + if current, err := inspectBacking(backing); err != nil || current != identity { + t.Fatalf("restart %d changed backing identity: %+v %v", restart, current, err) + } + if err := m.ManagedVolumeStatus(); err != nil { + t.Fatalf("dead original process fenced healthy storage: %v", err) + } + _, err := m.ClaimProvision(t.Context(), testKey, managedOptions("data")) + requireVolumeStatus(t, err, 409) + if _, _, _, err := m.WakeAgentSocket(t.Context(), sb.ID, sb.Token); !errors.Is(err, ErrFailed) { + t.Fatalf("new data plane request returned %v, want ErrFailed", err) + } + if err := m.Wake(t.Context(), sb.ID, Cred{Token: sb.Token}); !errors.Is(err, ErrFailed) { + t.Fatalf("explicit wake returned %v, want ErrFailed", err) + } + e.mu.Lock() + changed := len(e.removes) != 0 || len(e.starts) != 0 + e.mu.Unlock() + if changed { + t.Fatal("recovery cold-booted or removed the stopped owner") + } + persisted, err := m.store.load() + if err != nil || persisted[sb.ID] == nil || persisted[sb.ID].Failed == "" || persisted[sb.ID].VMName != sb.VMName { + t.Fatalf("failed claim identity not durable: %+v %v", persisted[sb.ID], err) + } + if !failed { + if events := usageEventsOf(t, m, sb.ID); events != "claim,vmm_exit,vmm_failed" { + t.Fatalf("restart %d lost or repeated VMM exit audit: %s", restart, events) + } + } + if restart == 0 { + if err := m.Close(); err != nil { + t.Fatal(err) + } + } + } + if failed { + m.mu.Lock() + m.claimed[sb.ID].Deadline = time.Now().Add(-time.Minute) + m.mu.Unlock() + m.reapOnce(t.Context()) + } else if err := m.Release(t.Context(), sb.ID, Cred{Operator: true}); err != nil { + t.Fatal(err) + } + waitFor(t, func() bool { _, held := m.managedHold(sb.ID); return !held }) + if _, live := m.Sandbox(sb.ID); live { + t.Fatal("release or expiry retained the failed claim") + } + if _, present, err := e.Inspect(t.Context(), sb.VMName); err != nil || present { + t.Fatalf("release or expiry retained VM record: present=%t err=%v", present, err) + } + if current, err := inspectBacking(backing); err != nil || current != identity { + t.Fatalf("release or expiry changed backing identity: %+v %v", current, err) + } + if _, err := m.ClaimProvision(t.Context(), testKey, managedOptions("data")); err != nil { + t.Fatalf("release or expiry did not free writer admission: %v", err) + } + }) + } +} + +func TestManagedStoppedUnownedClaimsStillRetire(t *testing.T) { + for _, name := range []string{"absent-claim", "quarantined-claim", "retiring-hold"} { + t.Run(name, func(t *testing.T) { + cfg := managedConfig(t) + e := newManagedEngine() + m := openManagedTest(t, cfg, e) + createManagedTest(t, m, "data") + sb := foundationClaim(t, m, "data") + if name == "quarantined-claim" { + sb.Failed = egressQuarantineFailure + if err := m.store.commit(m.store.set(sb)); err != nil { + t.Fatal(err) + } + } else if name == "retiring-hold" { + if err := m.retireManaged(sb.ID); err != nil { + t.Fatal(err) + } + } else if err := m.store.commit(m.store.del(sb.ID)); err != nil { + t.Fatal(err) + } + if err := m.Close(); err != nil { + t.Fatal(err) + } + e.mu.Lock() + e.stopped[sb.VMName] = true + e.pids[sb.VMName] = 0 + e.mu.Unlock() + m = openManagedTest(t, cfg, e) + if err := m.Reconcile(t.Context()); err != nil { + t.Fatal(err) + } + m.retryRemovals(t.Context()).Wait() + if _, held := m.managedHold(sb.ID); held { + t.Fatal("unowned stopped VM retained its hold") + } + if _, present, err := e.Inspect(t.Context(), sb.VMName); err != nil || present { + t.Fatalf("unowned stopped VM survived teardown: present=%t err=%v", present, err) + } + if _, err := m.ClaimProvision(t.Context(), testKey, managedOptions("data")); err != nil { + t.Fatalf("unowned stopped VM still holds writer admission: %v", err) + } + }) + } +} From 2b7b95c305945091474c4aabd0cde71b69c6c1dd Mon Sep 17 00:00:00 2001 From: minatoaquaMK2 Date: Tue, 20 Oct 2026 13:15:37 +0000 Subject: [PATCH 28/75] pool: Retain stopped managed claims until release. --- sandboxd/pool/managed_attachment.go | 21 ++++++++++++--------- 1 file changed, 12 insertions(+), 9 deletions(-) diff --git a/sandboxd/pool/managed_attachment.go b/sandboxd/pool/managed_attachment.go index 48f7d19f..e7581eb9 100644 --- a/sandboxd/pool/managed_attachment.go +++ b/sandboxd/pool/managed_attachment.go @@ -425,17 +425,20 @@ func (r *volumeRegistry) recoverableManagedClaim(h volumeHold, sb *types.Sandbox return false, nil } if rec.PID == 0 && rec.State == vmStateStopped { - if r.processGone(h.Process) { + if !r.processGone(h.Process) { + return false, managedStorageError() + } + if h.State != holdClaimed { + return false, nil + } + } else { + identity, err := r.process(rec.PID) + if err != nil && !processMissing(err) { + return false, managedStorageError() + } + if err != nil || identity != h.Process { return false, nil } - return false, managedStorageError() - } - identity, err := r.process(rec.PID) - if err != nil && !processMissing(err) { - return false, managedStorageError() - } - if err != nil || identity != h.Process { - return false, nil } for _, volume := range heldVolumes(sb) { if !slices.ContainsFunc(h.Refs, func(ref volumeRef) bool { return ref.Volume == volume }) { From 27e5fa52ef7c5091e93642cb734befda12c2ac67 Mon Sep 17 00:00:00 2001 From: minatoaquaMK2 Date: Tue, 20 Oct 2026 13:13:55 +0000 Subject: [PATCH 29/75] pool: cover final managed wake commit availability --- .../pool/managed_sleep_boundaries_test.go | 155 ++++++++++++++++++ 1 file changed, 155 insertions(+) diff --git a/sandboxd/pool/managed_sleep_boundaries_test.go b/sandboxd/pool/managed_sleep_boundaries_test.go index 08a64bf9..0c030459 100644 --- a/sandboxd/pool/managed_sleep_boundaries_test.go +++ b/sandboxd/pool/managed_sleep_boundaries_test.go @@ -3,10 +3,13 @@ package pool import ( "context" "errors" + "fmt" "net" "os" "path/filepath" "slices" + "strings" + "sync" "testing" "time" @@ -325,3 +328,155 @@ func TestManagedSleepPolicyReloadDuringResumeKeepsGuestPaused(t *testing.T) { t.Fatal("policy retry replayed memory restore") } } + +func gateManagedSleepReadyCommit(m *Manager, fail bool) (<-chan struct{}, func()) { + entered, resume := make(chan struct{}), make(chan struct{}) + var once sync.Once + commit := m.managed.commitFile + m.managed.commitFile = func(path string, raw []byte) error { + if !strings.Contains(string(raw), `"sleep":`) { + close(entered) + <-resume + if fail { + return errors.New("ready journal failed") + } + } + return commit(path, raw) + } + return entered, func() { once.Do(func() { close(resume) }) } +} + +func TestManagedSleepFinalWakeCommitKeepsUnrelatedRenewalLive(t *testing.T) { + for _, fail := range []bool{false, true} { + t.Run(map[bool]string{false: "success", true: "write failure"}[fail], func(t *testing.T) { + m, _, sb, _ := newSleepTest(t) + ordinary, err := m.ClaimProvision(t.Context(), testKey, ClaimOptions{}) + if err != nil { + t.Fatal(err) + } + requireSleep(t, m, sb) + entered, unblock := gateManagedSleepReadyCommit(m, fail) + var workers sync.WaitGroup + t.Cleanup(func() { unblock(); workers.Wait() }) + wake := make(chan error, 1) + workers.Go(func() { wake <- m.Wake(t.Context(), sb.ID, Cred{Token: sb.Token}) }) + select { + case <-entered: + case <-time.After(time.Second): + t.Fatal("wake did not reach final journal commit") + } + renew := make(chan error, 1) + workers.Go(func() { + _, err := m.Renew(t.Context(), ordinary.ID, Cred{Token: ordinary.Token}, time.Hour, "") + renew <- err + }) + select { + case err := <-renew: + if err != nil { + t.Errorf("unrelated renewal failed during final wake commit: %v", err) + } + case <-time.After(time.Second): + t.Error("unrelated Renew blocked by final wake journal commit") + } + passive := make(chan error, 1) + workers.Go(func() { + sock, err := m.AgentSocket(sb.ID, sb.Token) + if sock != "" { + passive <- fmt.Errorf("passive socket exposed before final commit: %q", sock) + return + } + passive <- err + }) + select { + case err := <-passive: + if !errors.Is(err, ErrPaused) { + t.Errorf("final commit did not fence passive readiness: %v", err) + } + case <-time.After(time.Second): + t.Error("passive readiness check blocked by final wake journal commit") + } + type socketResult struct { + sock string + err error + } + active := make(chan socketResult, 1) + started := make(chan struct{}) + workers.Go(func() { + close(started) + sock, _, release, err := m.WakeAgentSocket(t.Context(), sb.ID, sb.Token) + if release != nil { + release() + } + active <- socketResult{sock, err} + }) + <-started + var result socketResult + completed := false + select { + case result = <-active: + completed = true + t.Errorf("active socket completed before final commit: %+v", result) + case <-time.After(50 * time.Millisecond): + } + unblock() + if err := <-wake; (err != nil) != fail { + t.Errorf("wake after final commit: %v", err) + } + if !completed { + result = <-active + } + if fail { + if result.err == nil || result.sock != "" { + t.Errorf("failed ready journal exposed active socket: %+v", result) + } + if sock, err := m.AgentSocket(sb.ID, sb.Token); sock != "" || !errors.Is(err, ErrPaused) { + t.Errorf("failed ready journal exposed passive socket: %q %v", sock, err) + } + } else if result.err != nil || result.sock == "" { + t.Errorf("successful ready journal did not serve active socket: %+v", result) + } + }) + } +} + +func TestManagedSleepPolicyReloadWaitsForFinalWakeCommit(t *testing.T) { + m, _, sb, cfg := newSleepTest(t) + sb.EgressClass = "desk" + requireSleep(t, m, sb) + entered, unblock := gateManagedSleepReadyCommit(m, false) + var workers sync.WaitGroup + t.Cleanup(func() { unblock(); workers.Wait() }) + wake := make(chan error, 1) + workers.Go(func() { wake <- m.Wake(t.Context(), sb.ID, Cred{Token: sb.Token}) }) + select { + case <-entered: + case <-time.After(time.Second): + t.Fatal("wake did not reach final journal commit") + } + next := *cfg + next.Pools = slices.Clone(cfg.Pools) + next.Pools[0].Egress = &egress.Policy{Allow: []egress.Rule{{Host: "example.com"}}} + next.EgressClasses = []config.EgressClass{{Name: "desk", Egress: next.Pools[0].Egress}} + reload, started := make(chan error, 1), make(chan struct{}) + workers.Go(func() { + close(started) + _, err := m.ReloadConfig(t.Context(), &next) + reload <- err + }) + <-started + select { + case err := <-reload: + t.Fatalf("policy reload overtook final wake commit: %v", err) + case <-time.After(50 * time.Millisecond): + } + unblock() + if err := <-wake; err != nil { + t.Fatal(err) + } + if err := <-reload; err != nil { + t.Fatal(err) + } + if sock, err := m.AgentSocket(sb.ID, sb.Token); err != nil || sock == "" { + t.Fatalf("reload after ready commit hid running guest: %q %v", sock, err) + } +} From de438a2288df8dee370637d8acc747e02e9fd6bb Mon Sep 17 00:00:00 2001 From: minatoaquaMK2 Date: Tue, 20 Oct 2026 13:17:24 +0000 Subject: [PATCH 30/75] pool: release the manager lock during final managed wake commit --- sandboxd/pool/claim.go | 12 ++++++++++-- sandboxd/pool/managed_sleep.go | 7 +++++-- sandboxd/pool/managed_sleep_boundaries_test.go | 6 +++--- 3 files changed, 18 insertions(+), 7 deletions(-) diff --git a/sandboxd/pool/claim.go b/sandboxd/pool/claim.go index 2416dd19..ad6a0658 100644 --- a/sandboxd/pool/claim.go +++ b/sandboxd/pool/claim.go @@ -207,10 +207,18 @@ func (m *Manager) PreviewDial(ctx context.Context, id string, port uint16) (net. // AgentSocket resolves a claimed sandbox's vsock UDS without waking it. func (m *Manager) AgentSocket(id, token string) (string, error) { + sb, ok := m.claim(id, token) + if !ok { + return "", ErrUnknownSandbox + } + if !sb.Transition.TryLock() { + return "", ErrPaused + } + defer sb.Transition.Unlock() m.mu.Lock() defer m.mu.Unlock() - sb, ok := m.authed(id, token) - if !ok { + current, ok := m.authed(id, token) + if !ok || current != sb { return "", ErrUnknownSandbox } if sb.Replicated() && m.fence.Load().fenced { diff --git a/sandboxd/pool/managed_sleep.go b/sandboxd/pool/managed_sleep.go index 8324fdfd..f83eca39 100644 --- a/sandboxd/pool/managed_sleep.go +++ b/sandboxd/pool/managed_sleep.go @@ -344,13 +344,16 @@ func (m *Manager) resumeManagedSleep(ctx context.Context, sb *types.Sandbox, h v m.pauseManagedSleep(sb, h) return err } + m.reloadMu.Lock() m.mu.Lock() - if _, outbound := m.view.Load().out.Resolve(sb, m.claimPooledLocked); outbound { + _, outbound := m.view.Load().out.Resolve(sb, m.claimPooledLocked) + m.mu.Unlock() + if outbound { err = ErrVolumeCapture } else { err = m.managed.commitManagedSleep(h, nil) } - m.mu.Unlock() + m.reloadMu.Unlock() if err != nil { m.pauseManagedSleep(sb, h) return err diff --git a/sandboxd/pool/managed_sleep_boundaries_test.go b/sandboxd/pool/managed_sleep_boundaries_test.go index 0c030459..cd48452a 100644 --- a/sandboxd/pool/managed_sleep_boundaries_test.go +++ b/sandboxd/pool/managed_sleep_boundaries_test.go @@ -331,11 +331,11 @@ func TestManagedSleepPolicyReloadDuringResumeKeepsGuestPaused(t *testing.T) { func gateManagedSleepReadyCommit(m *Manager, fail bool) (<-chan struct{}, func()) { entered, resume := make(chan struct{}), make(chan struct{}) - var once sync.Once + var enteredOnce, resumeOnce sync.Once commit := m.managed.commitFile m.managed.commitFile = func(path string, raw []byte) error { if !strings.Contains(string(raw), `"sleep":`) { - close(entered) + enteredOnce.Do(func() { close(entered) }) <-resume if fail { return errors.New("ready journal failed") @@ -343,7 +343,7 @@ func gateManagedSleepReadyCommit(m *Manager, fail bool) (<-chan struct{}, func() } return commit(path, raw) } - return entered, func() { once.Do(func() { close(resume) }) } + return entered, func() { resumeOnce.Do(func() { close(resume) }) } } func TestManagedSleepFinalWakeCommitKeepsUnrelatedRenewalLive(t *testing.T) { From ac2bee4a3be0fc5b0a27a4323d5aea26c39cb759 Mon Sep 17 00:00:00 2001 From: minatoaquaMK2 Date: Tue, 20 Oct 2026 13:37:08 +0000 Subject: [PATCH 31/75] pool: Keep stopped claim recovery lint clean. --- sandboxd/pool/managed_attachment.go | 16 ++++++++-------- .../pool/managed_recovery_regression_test.go | 8 ++++---- 2 files changed, 12 insertions(+), 12 deletions(-) diff --git a/sandboxd/pool/managed_attachment.go b/sandboxd/pool/managed_attachment.go index e7581eb9..32f8cc98 100644 --- a/sandboxd/pool/managed_attachment.go +++ b/sandboxd/pool/managed_attachment.go @@ -424,14 +424,14 @@ func (r *volumeRegistry) recoverableManagedClaim(h volumeHold, sb *types.Sandbox if sb == nil || sb.VMName != h.VMName || sb.Tenant != h.Tenant || sb.Failed == egressQuarantineFailure || !present { return false, nil } - if rec.PID == 0 && rec.State == vmStateStopped { - if !r.processGone(h.Process) { - return false, managedStorageError() - } - if h.State != holdClaimed { - return false, nil - } - } else { + stopped := rec.PID == 0 && rec.State == vmStateStopped + if stopped && !r.processGone(h.Process) { + return false, managedStorageError() + } + if stopped && h.State != holdClaimed { + return false, nil + } + if !stopped { identity, err := r.process(rec.PID) if err != nil && !processMissing(err) { return false, managedStorageError() diff --git a/sandboxd/pool/managed_recovery_regression_test.go b/sandboxd/pool/managed_recovery_regression_test.go index f59c2f30..0291b1fc 100644 --- a/sandboxd/pool/managed_recovery_regression_test.go +++ b/sandboxd/pool/managed_recovery_regression_test.go @@ -287,11 +287,11 @@ func TestManagedStoppedClaimRetainsOwnershipOnRestart(t *testing.T) { } _, err := m.ClaimProvision(t.Context(), testKey, managedOptions("data")) requireVolumeStatus(t, err, 409) - if _, _, _, err := m.WakeAgentSocket(t.Context(), sb.ID, sb.Token); !errors.Is(err, ErrFailed) { - t.Fatalf("new data plane request returned %v, want ErrFailed", err) + if _, _, _, wakeErr := m.WakeAgentSocket(t.Context(), sb.ID, sb.Token); !errors.Is(wakeErr, ErrFailed) { + t.Fatalf("new data plane request returned %v, want ErrFailed", wakeErr) } - if err := m.Wake(t.Context(), sb.ID, Cred{Token: sb.Token}); !errors.Is(err, ErrFailed) { - t.Fatalf("explicit wake returned %v, want ErrFailed", err) + if wakeErr := m.Wake(t.Context(), sb.ID, Cred{Token: sb.Token}); !errors.Is(wakeErr, ErrFailed) { + t.Fatalf("explicit wake returned %v, want ErrFailed", wakeErr) } e.mu.Lock() changed := len(e.removes) != 0 || len(e.starts) != 0 From 586dd7b2b263c55bed045a02c20dcd276faae397 Mon Sep 17 00:00:00 2001 From: minatoaquaMK2 Date: Tue, 20 Oct 2026 13:32:34 +0000 Subject: [PATCH 32/75] test(pool): cover wake reload cycles and passive availability --- sandboxd/pool/claim_socket_test.go | 51 ++++++++++ .../pool/managed_sleep_boundaries_test.go | 33 +++++-- sandboxd/pool/managed_sleep_reload_test.go | 93 +++++++++++++++++++ 3 files changed, 170 insertions(+), 7 deletions(-) create mode 100644 sandboxd/pool/claim_socket_test.go create mode 100644 sandboxd/pool/managed_sleep_reload_test.go diff --git a/sandboxd/pool/claim_socket_test.go b/sandboxd/pool/claim_socket_test.go new file mode 100644 index 00000000..c31d9055 --- /dev/null +++ b/sandboxd/pool/claim_socket_test.go @@ -0,0 +1,51 @@ +package pool + +import ( + "context" + "sync" + "testing" + "time" +) + +type metadataWriteGate struct { + *fakeEngine + entered chan struct{} + resume chan struct{} +} + +func (e *metadataWriteGate) WriteInstanceMetadata(ctx context.Context, socket string, doc []byte) error { + close(e.entered) + <-e.resume + return e.fakeEngine.WriteInstanceMetadata(ctx, socket, doc) +} + +func TestAgentSocketAvailableDuringMetadataUpdate(t *testing.T) { + e := newFakeEngine() + m := newTestManager(t, e) + t.Cleanup(func() { _ = m.Close() }) + sb, err := m.ClaimProvision(t.Context(), testKey, ClaimOptions{}) + if err != nil { + t.Fatal(err) + } + gate := &metadataWriteGate{fakeEngine: e, entered: make(chan struct{}), resume: make(chan struct{})} + m.eng = gate + done := make(chan error, 1) + go func() { done <- m.SetInstanceMetadata(t.Context(), sb.ID, []byte(`{"role":"test"}`)) }() + finish := sync.OnceFunc(func() { + close(gate.resume) + if err := <-done; err != nil { + t.Errorf("metadata update: %v", err) + } + }) + t.Cleanup(finish) + select { + case <-gate.entered: + case <-time.After(time.Second): + t.Fatal("metadata update did not enter engine") + } + socket, err := m.AgentSocket(sb.ID, sb.Token) + finish() + if err != nil || socket != sb.VsockSocket { + t.Fatalf("running claim unavailable during metadata update: socket=%q err=%v", socket, err) + } +} diff --git a/sandboxd/pool/managed_sleep_boundaries_test.go b/sandboxd/pool/managed_sleep_boundaries_test.go index cd48452a..a2d67547 100644 --- a/sandboxd/pool/managed_sleep_boundaries_test.go +++ b/sandboxd/pool/managed_sleep_boundaries_test.go @@ -439,7 +439,7 @@ func TestManagedSleepFinalWakeCommitKeepsUnrelatedRenewalLive(t *testing.T) { } } -func TestManagedSleepPolicyReloadWaitsForFinalWakeCommit(t *testing.T) { +func TestManagedSleepPolicyReloadDuringFinalWakeCommitKeepsPaused(t *testing.T) { m, _, sb, cfg := newSleepTest(t) sb.EgressClass = "desk" requireSleep(t, m, sb) @@ -466,17 +466,36 @@ func TestManagedSleepPolicyReloadWaitsForFinalWakeCommit(t *testing.T) { <-started select { case err := <-reload: - t.Fatalf("policy reload overtook final wake commit: %v", err) - case <-time.After(50 * time.Millisecond): + if err != nil { + t.Fatal(err) + } + case <-time.After(time.Second): + t.Fatal("policy reload blocked by final wake journal commit") } unblock() - if err := <-wake; err != nil { + if err := <-wake; !errors.Is(err, ErrVolumeCapture) { + t.Fatalf("wake accepted policy published during final commit: %v", err) + } + if sock, err := m.AgentSocket(sb.ID, sb.Token); sock != "" || !errors.Is(err, ErrPaused) { + t.Fatalf("policy-blocked ready commit exposed guest: %q %v", sock, err) + } + if err := m.Close(); err != nil { t.Fatal(err) } - if err := <-reload; err != nil { + registry, err := openVolumeRegistry(cfg.ManagedVolumeDir, m.nodeID) + if err != nil { + t.Fatal(err) + } + defer registry.close() + h := registry.data.Holds[sb.ID] + if h.Sleep == nil || h.Sleep.Phase != sleepResuming { + t.Fatalf("policy refusal lost durable resume fence: %+v", h.Sleep) + } + claims, err := m.store.load() + if err != nil { t.Fatal(err) } - if sock, err := m.AgentSocket(sb.ID, sb.Token); err != nil || sock == "" { - t.Fatalf("reload after ready commit hid running guest: %q %v", sock, err) + if claim := claims[sb.ID]; claim == nil || claim.HibernateSnap != h.Sleep.Snapshot || claim.ReaderSafe { + t.Fatalf("policy refusal lost durable paused claim: %+v", claim) } } diff --git a/sandboxd/pool/managed_sleep_reload_test.go b/sandboxd/pool/managed_sleep_reload_test.go new file mode 100644 index 00000000..ec2a50b3 --- /dev/null +++ b/sandboxd/pool/managed_sleep_reload_test.go @@ -0,0 +1,93 @@ +package pool + +import ( + "context" + "slices" + "sync" + "testing" + "time" + + "github.com/cocoonstack/sandbox/sandboxd/config" + "github.com/cocoonstack/sandbox/sandboxd/tenants" +) + +func TestManagedSleepReloadAndExpiryCompleteDuringWake(t *testing.T) { + cfg := managedConfig(t) + cfg.RefillConcurrency = 2 + cfg.Pools[0].IdleHibernateSeconds = 1 + cfg.ManagedVolumeHibernate = &config.ManagedVolumeHibernate{HookArgv: []string{"/hook"}, HookTimeoutSeconds: 1} + e := &sleepTestEngine{managedTestEngine: newManagedEngine(), generation: 2, reason: "boot", vmID: "immutable-vm", mounted: true} + m, err := NewManager(t.Context(), cfg, e, testSecrets(t), testBackends(t, cfg)) + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { _ = m.Close() }) + m.managed.process = e.identity + if err := m.PutTenant(t.Context(), tenants.TenantSpec{Name: "t1", Token: "t1-token"}); err != nil { + t.Fatal(err) + } + createManagedTest(t, m, "data") + opts := managedOptions("data") + opts.Volumes[0].AttachOnly = false + sb, err := m.ClaimProvision(t.Context(), testKey, opts) + if err != nil { + t.Fatal(err) + } + waitFor(t, func() bool { return refilledTo(t, m, testKey, 1) }) + waitPoolSettled(t, m, testKey) + requireSleep(t, m, sb) + e.hookEntered = make(chan struct{}, 2) + e.hookContinue = make(chan struct{}) + releaseHook := sync.OnceFunc(func() { close(e.hookContinue) }) + t.Cleanup(releaseHook) + wake := make(chan error, 1) + go func() { wake <- m.Wake(context.Background(), sb.ID, Cred{Token: sb.Token}) }() + select { + case <-e.hookEntered: + case <-time.After(3 * time.Second): + t.Fatal("wake did not enter resume hook") + } + m.mu.Lock() + sb.Deadline = time.Now().Add(-time.Minute) + m.mu.Unlock() + m.reapOnce(t.Context()) + waitFor(t, func() bool { return len(m.maintSem) == 1 }) + if cap(m.maintSem) != 1 { + t.Fatal("unexpected configured maintenance capacity") + } + if _, err := m.Renew(t.Context(), sb.ID, Cred{Token: sb.Token}, time.Hour, ""); err != nil { + t.Fatal(err) + } + before := m.view.Load() + next := *cfg + next.Pools = slices.Clone(cfg.Pools) + next.Pools[0].Storage = "30G" + reload := make(chan error, 1) + go func() { _, err := m.ReloadConfig(context.Background(), &next); reload <- err }() + waitFor(t, func() bool { return m.view.Load() != before }) + releaseHook() + completed := false + select { + case err := <-reload: + if err != nil { + t.Fatal(err) + } + completed = true + case <-time.After(time.Second): + } + if !completed { + <-m.maintSem + err := <-reload + m.maintSem <- struct{}{} + if err != nil { + t.Errorf("reload rescue failed: %v", err) + } + } + if err := <-wake; err != nil { + t.Errorf("wake after cleanup: %v", err) + } + waitFor(t, func() bool { return len(m.maintSem) == 0 }) + if !completed { + t.Fatal("reload and managed wake did not complete until the blocked expiry worker's maintenance permit was borrowed") + } +} From 4ab5c5c7db17df8920bb24fc3805a854f5cf11db Mon Sep 17 00:00:00 2001 From: minatoaquaMK2 Date: Tue, 20 Oct 2026 13:34:33 +0000 Subject: [PATCH 33/75] test(pool): cover wake publication crash and rollback failures --- .../pool/managed_sleep_boundaries_test.go | 130 ++++++++++++++++++ 1 file changed, 130 insertions(+) diff --git a/sandboxd/pool/managed_sleep_boundaries_test.go b/sandboxd/pool/managed_sleep_boundaries_test.go index a2d67547..90721788 100644 --- a/sandboxd/pool/managed_sleep_boundaries_test.go +++ b/sandboxd/pool/managed_sleep_boundaries_test.go @@ -2,6 +2,7 @@ package pool import ( "context" + "encoding/json/v2" "errors" "fmt" "net" @@ -499,3 +500,132 @@ func TestManagedSleepPolicyReloadDuringFinalWakeCommitKeepsPaused(t *testing.T) t.Fatalf("policy refusal lost durable paused claim: %+v", claim) } } + +func TestManagedSleepReadyCommitCrashRearmsReloadedPolicy(t *testing.T) { + m, e, sb, cfg := newSleepTest(t) + sb.EgressClass = "desk" + requireSleep(t, m, sb) + commit := m.managed.commitFile + m.managed.commitFile = func(path string, raw []byte) error { + if err := commit(path, raw); err != nil { + return err + } + if !strings.Contains(string(raw), `"sleep":`) { + return errors.New("crash after durable ready registry") + } + return nil + } + if err := m.Wake(t.Context(), sb.ID, Cred{Token: sb.Token}); err == nil { + t.Fatal("lost ready commit acknowledgement was hidden") + } + if err := m.Close(); err != nil { + t.Fatal(err) + } + next := *cfg + next.Pools = slices.Clone(cfg.Pools) + next.Pools[0].Egress = &egress.Policy{Allow: []egress.Rule{{Host: "example.com"}}} + next.EgressClasses = []config.EgressClass{{Name: "desk", Egress: next.Pools[0].Egress}} + sock := filepath.Join(outboundtest.SockRoot(t), "v") + e.mu.Lock() + e.vms[sb.VMName] = sock + e.mu.Unlock() + recovered, err := NewManager(t.Context(), &next, e, testSecrets(t), testBackends(t, &next)) + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { recovered.out.Disarm(sb.ID, false); _ = recovered.Close() }) + recovered.managed.process = e.identity + if h, _ := recovered.managedHold(sb.ID); h.Sleep != nil { + t.Fatal("crash fixture did not reach durable ready registry") + } + if err := recovered.Reconcile(t.Context()); err != nil { + t.Fatal(err) + } + if got, err := recovered.AgentSocket(sb.ID, sb.Token); err != nil || got != sock { + t.Fatalf("completed wake did not recover running guest: %q %v", got, err) + } + requireProxyDoor(t, sock) + if len(e.restores) != 1 { + t.Fatal("ready-commit crash replayed memory restore") + } +} + +func TestManagedSleepLatePolicyRollbackWriteFailureStaysPaused(t *testing.T) { + for _, journal := range []string{"registry", "claims"} { + t.Run(journal, func(t *testing.T) { + m, _, sb, cfg := newSleepTest(t) + sb.EgressClass = "desk" + requireSleep(t, m, sb) + commit := m.managed.commitFile + entered, resume := make(chan struct{}), make(chan struct{}) + unblock := sync.OnceFunc(func() { close(resume) }) + var readyWritten bool + m.managed.commitFile = func(path string, raw []byte) error { + if readyWritten && journal == "registry" { + return errors.New("resume fence write failed") + } + if !strings.Contains(string(raw), `"sleep":`) { + close(entered) + <-resume + if journal == "claims" { + if err := os.Mkdir(m.store.path+".tmp", 0o700); err != nil { + return err + } + } + readyWritten = true + } + return commit(path, raw) + } + var workers sync.WaitGroup + t.Cleanup(func() { unblock(); workers.Wait() }) + wake := make(chan error, 1) + workers.Go(func() { wake <- m.Wake(t.Context(), sb.ID, Cred{Token: sb.Token}) }) + select { + case <-entered: + case <-time.After(time.Second): + t.Fatal("wake did not reach ready commit") + } + next := *cfg + next.Pools = slices.Clone(cfg.Pools) + next.Pools[0].Egress = &egress.Policy{Allow: []egress.Rule{{Host: "example.com"}}} + next.EgressClasses = []config.EgressClass{{Name: "desk", Egress: next.Pools[0].Egress}} + reload := make(chan error, 1) + workers.Go(func() { _, err := m.ReloadConfig(t.Context(), &next); reload <- err }) + select { + case err := <-reload: + if err != nil { + t.Fatal(err) + } + case <-time.After(time.Second): + t.Fatal("policy reload blocked by ready commit") + } + unblock() + if err := <-wake; !errors.Is(err, ErrVolumeCapture) || err == ErrVolumeCapture { + t.Fatalf("late policy rollback did not report its write failure: %v", err) + } + if got, err := m.AgentSocket(sb.ID, sb.Token); got != "" || !errors.Is(err, ErrPaused) { + t.Fatalf("failed rollback exposed passive guest: %q %v", got, err) + } + got, _, done, err := m.WakeAgentSocket(t.Context(), sb.ID, sb.Token) + if done != nil { + done() + } + if got != "" || err == nil { + t.Fatalf("failed rollback exposed active guest: %q %v", got, err) + } + if journal == "claims" { + raw, err := os.ReadFile(filepath.Join(cfg.ManagedVolumeDir, "registry.json")) + if err != nil { + t.Fatal(err) + } + var data volumeRegistryData + if err := json.Unmarshal(raw, &data); err != nil { + t.Fatal(err) + } + if h := data.Holds[sb.ID]; h.Sleep == nil || h.Sleep.Phase != sleepResuming { + t.Fatalf("claim projection failure lost durable resume fence: %+v", h.Sleep) + } + } + }) + } +} From 69a53ca4a74367c25cefff6246283dad34398065 Mon Sep 17 00:00:00 2001 From: minatoaquaMK2 Date: Tue, 20 Oct 2026 13:37:22 +0000 Subject: [PATCH 34/75] fix(pool): publish managed wake without holding reload lock --- sandboxd/pool/claim.go | 17 ++++++--------- sandboxd/pool/claim_socket_test.go | 4 ++-- sandboxd/pool/managed_sleep.go | 25 +++++++++++----------- sandboxd/pool/managed_sleep_reload_test.go | 2 +- 4 files changed, 22 insertions(+), 26 deletions(-) diff --git a/sandboxd/pool/claim.go b/sandboxd/pool/claim.go index ad6a0658..b19a5451 100644 --- a/sandboxd/pool/claim.go +++ b/sandboxd/pool/claim.go @@ -207,24 +207,21 @@ func (m *Manager) PreviewDial(ctx context.Context, id string, port uint16) (net. // AgentSocket resolves a claimed sandbox's vsock UDS without waking it. func (m *Manager) AgentSocket(id, token string) (string, error) { - sb, ok := m.claim(id, token) - if !ok { - return "", ErrUnknownSandbox - } - if !sb.Transition.TryLock() { - return "", ErrPaused - } - defer sb.Transition.Unlock() m.mu.Lock() defer m.mu.Unlock() - current, ok := m.authed(id, token) - if !ok || current != sb { + sb, ok := m.authed(id, token) + if !ok { return "", ErrUnknownSandbox } if sb.Replicated() && m.fence.Load().fenced { return "", ErrLeaseLost } // no activity stamp: a control-plane poll must not keep an idle sandbox awake + select { + case <-m.managedStreams[sb]: + return "", ErrPaused + default: + } if m.managedPaused(sb) { return "", ErrPaused } diff --git a/sandboxd/pool/claim_socket_test.go b/sandboxd/pool/claim_socket_test.go index c31d9055..8c895acd 100644 --- a/sandboxd/pool/claim_socket_test.go +++ b/sandboxd/pool/claim_socket_test.go @@ -33,8 +33,8 @@ func TestAgentSocketAvailableDuringMetadataUpdate(t *testing.T) { go func() { done <- m.SetInstanceMetadata(t.Context(), sb.ID, []byte(`{"role":"test"}`)) }() finish := sync.OnceFunc(func() { close(gate.resume) - if err := <-done; err != nil { - t.Errorf("metadata update: %v", err) + if updateErr := <-done; updateErr != nil { + t.Errorf("metadata update: %v", updateErr) } }) t.Cleanup(finish) diff --git a/sandboxd/pool/managed_sleep.go b/sandboxd/pool/managed_sleep.go index f83eca39..f4f0c7a4 100644 --- a/sandboxd/pool/managed_sleep.go +++ b/sandboxd/pool/managed_sleep.go @@ -336,6 +336,7 @@ func (m *Manager) resumeManagedSleep(ctx context.Context, sb *types.Sandbox, h v if err = m.eng.VerifyMountedVolume(ctx, sock, h.Refs[0].Volume.Name, h.Refs[0].Volume.Mount); err != nil { return err } + m.closeManagedStreams(sb) m.mu.Lock() sb.VsockSocket, sb.HibernateSnap, sb.PendingSnap, sb.ReaderSafe = sock, "", "", false js := m.store.set(sb) @@ -344,21 +345,25 @@ func (m *Manager) resumeManagedSleep(ctx context.Context, sb *types.Sandbox, h v m.pauseManagedSleep(sb, h) return err } - m.reloadMu.Lock() m.mu.Lock() _, outbound := m.view.Load().out.Resolve(sb, m.claimPooledLocked) m.mu.Unlock() if outbound { - err = ErrVolumeCapture - } else { - err = m.managed.commitManagedSleep(h, nil) + return errors.Join(ErrVolumeCapture, m.projectManagedSleep(sb, h)) } - m.reloadMu.Unlock() - if err != nil { + if err = m.managed.commitManagedSleep(h, nil); err != nil { m.pauseManagedSleep(sb, h) return err } - m.rotateManagedStreams(sb) + m.mu.Lock() + _, outbound = m.view.Load().out.Resolve(sb, m.claimPooledLocked) + if !outbound { + m.managedStreams[sb] = make(chan struct{}) + } + m.mu.Unlock() + if outbound { + return errors.Join(ErrVolumeCapture, m.managed.commitManagedSleep(h, h.Sleep), m.projectManagedSleep(sb, h)) + } if restored { m.counters.wakes.Add(1) m.recordUsage(ctx, usageEvent{Event: "wake", ID: sb.ID, VMName: sb.VMName}) @@ -391,9 +396,3 @@ func (m *Manager) closeManagedStreams(sb *types.Sandbox) { } } } - -func (m *Manager) rotateManagedStreams(sb *types.Sandbox) { - m.mu.Lock() - defer m.mu.Unlock() - m.managedStreams[sb] = make(chan struct{}) -} diff --git a/sandboxd/pool/managed_sleep_reload_test.go b/sandboxd/pool/managed_sleep_reload_test.go index ec2a50b3..65aeced6 100644 --- a/sandboxd/pool/managed_sleep_reload_test.go +++ b/sandboxd/pool/managed_sleep_reload_test.go @@ -23,7 +23,7 @@ func TestManagedSleepReloadAndExpiryCompleteDuringWake(t *testing.T) { } t.Cleanup(func() { _ = m.Close() }) m.managed.process = e.identity - if err := m.PutTenant(t.Context(), tenants.TenantSpec{Name: "t1", Token: "t1-token"}); err != nil { + if err = m.PutTenant(t.Context(), tenants.TenantSpec{Name: "t1", Token: "t1-token"}); err != nil { t.Fatal(err) } createManagedTest(t, m, "data") From 1baa003ce37e0577a23f54a19823fc9c984efab6 Mon Sep 17 00:00:00 2001 From: minatoaquaMK2 Date: Tue, 20 Oct 2026 13:33:55 +0000 Subject: [PATCH 35/75] engine: Cover additive growth replies and failed command output. --- sandboxd/engine/volume_grow_test.go | 46 +++++++++++++++++++++++++++++ 1 file changed, 46 insertions(+) diff --git a/sandboxd/engine/volume_grow_test.go b/sandboxd/engine/volume_grow_test.go index c48ce123..d424cad1 100644 --- a/sandboxd/engine/volume_grow_test.go +++ b/sandboxd/engine/volume_grow_test.go @@ -112,3 +112,49 @@ func TestGrowthRejectsMalformedWireEvenWithAValidReceiptPrefix(t *testing.T) { t.Fatalf("partial malformed stdout accepted: %+v %v", got, err) } } + +func TestGrowthAcceptsAdditiveRuntimeFields(t *testing.T) { + inspect, err := os.ReadFile("testdata/disk-inspect.json") + if err != nil { + t.Fatal(err) + } + grow, err := os.ReadFile("testdata/disk-grow.json") + if err != nil { + t.Fatal(err) + } + var expected DiskInspection + var result DiskGrowResult + if err := json.Unmarshal(inspect, &expected); err != nil { + t.Fatal(err) + } + if err := json.Unmarshal(grow, &result); err != nil { + t.Fatal(err) + } + t.Setenv("GROW_INSPECT", strings.ReplaceAll(string(inspect), "{", `{"future_field":{"nested":true},`)) + t.Setenv("GROW_RESULT", strings.ReplaceAll(string(grow), "{", `{"future_field":{"nested":true},`)) + fakeCocoon(t, `#!/bin/sh +if [ "$3" = inspect ]; then printf '%s' "$GROW_INSPECT"; else printf '%s' "$GROW_RESULT"; fi +`) + e := New("cocoon", nil, nil, false, false, "") + got, err := e.InspectDisk(t.Context(), expected.Requested) + if err != nil || got.Live.Binding == nil || !got.Live.Binding.Equal(*expected.Live.Binding) { + t.Errorf("additive inspection fields rejected: %+v %v", got, err) + } + ack, err := e.GrowDisk(t.Context(), result.Receipt.Request) + if err != nil || ack.Receipt == nil || !ack.Receipt.Request.Equal(result.Receipt.Request) { + t.Errorf("additive growth fields rejected: %+v %v", ack, err) + } + t.Setenv("GROW_INSPECT", strings.Replace(string(inspect), `"guest_block_status": "unobserved"`, `"guest_block_status": "unsupported"`, 1)) + if _, err := e.InspectDisk(t.Context(), expected.Requested); err == nil { + t.Fatal("unsupported known inspection field accepted") + } +} + +func TestRunDiscardsFailedOutputOutsideGrowth(t *testing.T) { + fakeCocoon(t, "#!/bin/sh\nprintf 'partial output'\nexit 1\n") + e := New("cocoon", nil, nil, false, false, "") + raw, err := e.run(t.Context(), "vm", "inspect", "test-vm") + if err == nil || len(raw) != 0 { + t.Fatalf("ordinary command returned failed output: %q %v", raw, err) + } +} From 38587e7c6102cae3ed26e1379e962068d9f04ed7 Mon Sep 17 00:00:00 2001 From: minatoaquaMK2 Date: Tue, 20 Oct 2026 13:34:42 +0000 Subject: [PATCH 36/75] engine: Accept additive growth replies and scope failed output. --- sandboxd/engine/engine.go | 8 ++++++++ sandboxd/engine/volume_grow.go | 8 ++++---- 2 files changed, 12 insertions(+), 4 deletions(-) diff --git a/sandboxd/engine/engine.go b/sandboxd/engine/engine.go index 8610b02e..82adba83 100644 --- a/sandboxd/engine/engine.go +++ b/sandboxd/engine/engine.go @@ -407,6 +407,14 @@ func (e *Engine) netArgs(name string, key types.PoolKey, cold bool) []string { } func (e *Engine) run(ctx context.Context, args ...string) ([]byte, error) { + out, err := e.runOutput(ctx, args...) + if err != nil { + return nil, err + } + return out, nil +} + +func (e *Engine) runOutput(ctx context.Context, args ...string) ([]byte, error) { ctx, cancel := context.WithTimeout(ctx, cmdTimeout) defer cancel() log.WithFunc("engine.run").Debugf(ctx, "cocoon %s", strings.Join(args, " ")) diff --git a/sandboxd/engine/volume_grow.go b/sandboxd/engine/volume_grow.go index 96a8ed1b..ec632d0d 100644 --- a/sandboxd/engine/volume_grow.go +++ b/sandboxd/engine/volume_grow.go @@ -112,8 +112,8 @@ func (r DiskGrowRequest) Equal(other DiskGrowRequest) bool { func (e *Engine) InspectDisk(ctx context.Context, ref DiskRef) (DiskInspection, error) { var result DiskInspection - raw, err := e.run(ctx, "vm", "disk", "inspect", ref.VMID, "--disk-id", ref.DiskID, "--output", "json") - if parseErr := json.Unmarshal(raw, &result, json.RejectUnknownMembers(true)); parseErr != nil { + raw, err := e.runOutput(ctx, "vm", "disk", "inspect", ref.VMID, "--disk-id", ref.DiskID, "--output", "json") + if parseErr := json.Unmarshal(raw, &result); parseErr != nil { return DiskInspection{}, errors.Join(err, parseErr) } if result.Requested != ref || result.GuestBlockStatus != diskUnobserved { @@ -147,8 +147,8 @@ func (e *Engine) GrowDisk(ctx context.Context, request DiskGrowRequest) (DiskGro return DiskGrowResult{}, err } var result DiskGrowResult - raw, err = e.run(ctx, "vm", "disk", "grow", "--expect", f.Name(), "--request-id", request.ID, "--target-bytes", strconv.FormatInt(request.TargetBytes, 10), "--output", "json") - if parseErr := json.Unmarshal(raw, &result, json.RejectUnknownMembers(true)); parseErr != nil { + raw, err = e.runOutput(ctx, "vm", "disk", "grow", "--expect", f.Name(), "--request-id", request.ID, "--target-bytes", strconv.FormatInt(request.TargetBytes, 10), "--output", "json") + if parseErr := json.Unmarshal(raw, &result); parseErr != nil { return DiskGrowResult{}, errors.Join(err, parseErr) } if result.GuestBlockStatus != diskUnobserved { From 8ba86081edec0da504810d5b2d269b47efc23a83 Mon Sep 17 00:00:00 2001 From: minatoaquaMK2 Date: Tue, 20 Oct 2026 13:35:00 +0000 Subject: [PATCH 37/75] engine: Keep growth regression parsing lint clean. --- sandboxd/engine/volume_grow_test.go | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/sandboxd/engine/volume_grow_test.go b/sandboxd/engine/volume_grow_test.go index d424cad1..0b4b7e42 100644 --- a/sandboxd/engine/volume_grow_test.go +++ b/sandboxd/engine/volume_grow_test.go @@ -124,11 +124,11 @@ func TestGrowthAcceptsAdditiveRuntimeFields(t *testing.T) { } var expected DiskInspection var result DiskGrowResult - if err := json.Unmarshal(inspect, &expected); err != nil { - t.Fatal(err) + if parseErr := json.Unmarshal(inspect, &expected); parseErr != nil { + t.Fatal(parseErr) } - if err := json.Unmarshal(grow, &result); err != nil { - t.Fatal(err) + if parseErr := json.Unmarshal(grow, &result); parseErr != nil { + t.Fatal(parseErr) } t.Setenv("GROW_INSPECT", strings.ReplaceAll(string(inspect), "{", `{"future_field":{"nested":true},`)) t.Setenv("GROW_RESULT", strings.ReplaceAll(string(grow), "{", `{"future_field":{"nested":true},`)) From 03272db0a14a6a40ce5ae0534e8a34a771b11efd Mon Sep 17 00:00:00 2001 From: minatoaquaMK2 Date: Tue, 20 Oct 2026 13:37:25 +0000 Subject: [PATCH 38/75] test(pool): reproduce growth admission and replay races --- .../pool/managed_grow_concurrency_test.go | 491 ++++++++++++++++++ 1 file changed, 491 insertions(+) create mode 100644 sandboxd/pool/managed_grow_concurrency_test.go diff --git a/sandboxd/pool/managed_grow_concurrency_test.go b/sandboxd/pool/managed_grow_concurrency_test.go new file mode 100644 index 00000000..4510a9a7 --- /dev/null +++ b/sandboxd/pool/managed_grow_concurrency_test.go @@ -0,0 +1,491 @@ +package pool + +import ( + "context" + "encoding/json/v2" + "errors" + "fmt" + "io" + "net" + "os" + "path/filepath" + "strings" + "sync" + "testing" + "time" + + "github.com/cocoonstack/sandbox/sandboxd/config" + "github.com/cocoonstack/sandbox/sandboxd/engine" + "github.com/cocoonstack/sandbox/sandboxd/types" +) + +type growthBarrierEngine struct { + *growthTestEngine + barrierMu sync.Mutex + method string + occurrence int + seen int + entered, resume chan struct{} + unblock func() + socket string + connections sync.WaitGroup +} + +func (e *growthBarrierEngine) pause(method string) { + e.barrierMu.Lock() + block := false + if method == e.method { + e.seen++ + block = e.seen == e.occurrence + } + e.barrierMu.Unlock() + if block { + close(e.entered) + <-e.resume + } +} + +func (e *growthBarrierEngine) Inspect(ctx context.Context, name string) (types.VMRecord, bool, error) { + e.pause("Inspect") + return e.growthTestEngine.Inspect(ctx, name) +} + +func (e *growthBarrierEngine) InspectDisk(ctx context.Context, ref engine.DiskRef) (engine.DiskInspection, error) { + e.pause("InspectDisk") + return e.growthTestEngine.InspectDisk(ctx, ref) +} + +func (e *growthBarrierEngine) GrowDisk(ctx context.Context, req engine.DiskGrowRequest) (engine.DiskGrowResult, error) { + e.pause("GrowDisk") + return e.growthTestEngine.GrowDisk(ctx, req) +} + +func (e *growthBarrierEngine) ObserveVolume(ctx context.Context, socket, serial, mount string) (engine.GuestVolumeObservation, error) { + e.pause("ObserveVolume") + return e.growthTestEngine.ObserveVolume(ctx, socket, serial, mount) +} + +func (e *growthBarrierEngine) DialGuestPort(_ context.Context, socket string, port uint16) (net.Conn, error) { + if socket != e.socket || port != 8080 { + return nil, fmt.Errorf("dialed %q:%d, want original guest %q:8080", socket, port, e.socket) + } + client, server := net.Pipe() + e.connections.Go(func() { + defer server.Close() + _, _ = io.Copy(server, server) + }) + return client, nil +} + +type growthCallResult struct { + info types.ManagedVolumeInfo + err error +} + +func (r growthCallResult) String() string { + return fmt.Sprintf("size=%d generation=%d growth=%+v err=%v", r.info.SizeBytes, r.info.CapacityGeneration, r.info.Growth, r.err) +} + +type growthConcurrencyTest struct { + m *Manager + e *growthBarrierEngine + sb *types.Sandbox + volume types.ManagedVolumeInfo + workers sync.WaitGroup +} + +func newGrowthConcurrencyTest(t *testing.T, method string, occurrence int) *growthConcurrencyTest { + t.Helper() + m, engine, sb, _, volume := newGrowthTest(t) + e := &growthBarrierEngine{ + growthTestEngine: engine, method: method, occurrence: occurrence, + entered: make(chan struct{}), resume: make(chan struct{}), socket: sb.VsockSocket, + } + e.unblock = sync.OnceFunc(func() { close(e.resume) }) + m.eng = e + f := &growthConcurrencyTest{m: m, e: e, sb: sb, volume: volume} + t.Cleanup(func() { + e.unblock() + done := make(chan struct{}) + go func() { + f.workers.Wait() + e.connections.Wait() + close(done) + }() + select { + case <-done: + case <-time.After(5 * time.Second): + t.Error("growth test workers did not stop after releasing the engine") + } + }) + return f +} + +func (f *growthConcurrencyTest) grow(ctx context.Context, req types.VolumeGrow) <-chan growthCallResult { + done := make(chan growthCallResult, 1) + f.workers.Go(func() { + info, err := f.m.GrowVolume(ctx, "data", f.volume.ID, req, "t1") + done <- growthCallResult{info, err} + }) + return done +} + +func (f *growthConcurrencyTest) call(fn func() error) <-chan error { + done := make(chan error, 1) + f.workers.Go(func() { done <- fn() }) + return done +} + +func (f *growthConcurrencyTest) waitEntered(t *testing.T, owner <-chan growthCallResult) { + t.Helper() + select { + case <-f.e.entered: + case result := <-owner: + t.Fatalf("growth returned before %s barrier: %+v", f.e.method, result) + case <-time.After(5 * time.Second): + t.Fatalf("growth did not enter %s", f.e.method) + } +} + +func growthPrompt(t *testing.T, name string, done <-chan error) (error, bool) { + t.Helper() + select { + case err := <-done: + return err, true + case <-time.After(time.Second): + t.Errorf("%s waited for the blocked growth engine", name) + return nil, false + } +} + +func waitGrowthResult(t *testing.T, done <-chan growthCallResult) growthCallResult { + t.Helper() + select { + case result := <-done: + return result + case <-time.After(5 * time.Second): + t.Fatal("growth did not return after releasing the engine") + return growthCallResult{} + } +} + +func (f *growthConcurrencyTest) guestAccess(t *testing.T) { + t.Helper() + checks := []struct { + name string + call func() error + }{ + {"AgentSocket", func() error { + socket, err := f.m.AgentSocket(f.sb.ID, f.sb.Token) + if err == nil && socket != f.e.socket { + return fmt.Errorf("socket %q, want %q", socket, f.e.socket) + } + return err + }}, + {"operator DialPort", func() error { return f.dial(t.Context(), Cred{Operator: true}) }}, + {"WakeAgentSocket", func() error { + socket, fenced, release, err := f.m.WakeAgentSocket(t.Context(), f.sb.ID, f.sb.Token) + if release != nil { + defer release() + } + if err != nil { + return err + } + if socket != f.e.socket { + return fmt.Errorf("socket %q, want %q", socket, f.e.socket) + } + select { + case <-fenced: + return errors.New("running guest stream was fenced") + default: + return nil + } + }}, + {"token DialPort", func() error { return f.dial(t.Context(), Cred{Token: f.sb.Token}) }}, + } + for _, check := range checks { + if err, returned := growthPrompt(t, check.name, f.call(check.call)); returned && err != nil { + t.Errorf("%s refused the running guest during growth: %v", check.name, err) + } + } +} + +func (f *growthConcurrencyTest) dial(ctx context.Context, cred Cred) error { + conn, err := f.m.DialPort(ctx, f.sb.ID, cred, 8080) + if err != nil { + return err + } + defer conn.Close() + if err := conn.SetDeadline(time.Now().Add(time.Second)); err != nil { + return err + } + if _, err := conn.Write([]byte("guest")); err != nil { + return err + } + got := make([]byte, 5) + if _, err := io.ReadFull(conn, got); err != nil { + return err + } + if string(got) != "guest" { + return fmt.Errorf("guest reply %q", got) + } + return nil +} + +func (f *growthConcurrencyTest) lifecycleRefused(t *testing.T) { + t.Helper() + for _, call := range []struct { + name string + fn func() error + }{ + {"Hibernate", func() error { return f.m.Hibernate(t.Context(), f.sb.ID, Cred{Operator: true}) }}, + {"Release", func() error { return f.m.Release(t.Context(), f.sb.ID, Cred{Operator: true}) }}, + } { + if err, returned := growthPrompt(t, call.name, f.call(call.fn)); returned { + if call.name == "Hibernate" && f.m.managedSleepHook == nil && errors.Is(err, ErrVolumeCapture) { + continue + } + var conflict *VolumeError + if !errors.As(err, &conflict) || conflict.Status != 409 { + t.Errorf("%s returned %v, want volume status 409", call.name, err) + } + } + } + if _, ok := f.m.Sandbox(f.sb.ID); !ok { + t.Error("growth owner lost its claim") + } + if _, ok := f.m.managedHold(f.sb.ID); !ok { + t.Error("growth owner lost its volume hold") + } + f.e.mu.Lock() + defer f.e.mu.Unlock() + if f.e.stopped[f.sb.VMName] || len(f.e.removes) != 0 { + t.Errorf("growth owner was stopped or removed: stopped=%t removes=%v", f.e.stopped[f.sb.VMName], f.e.removes) + } + for _, op := range f.e.volumeOps { + if strings.HasPrefix(op, "umount:") { + t.Errorf("growth owner was unmounted: %s", op) + } + } +} + +func TestManagedGrowthGuestConnectionsDuringEngineStalls(t *testing.T) { + for _, barrier := range []struct { + name, method string + occurrence int + }{ + {"admission VM inspection", "Inspect", 1}, + {"admission disk inspection", "InspectDisk", 1}, + {"admission guest observation", "ObserveVolume", 1}, + {"dispatch disk inspection", "InspectDisk", 2}, + {"disk growth", "GrowDisk", 1}, + {"acknowledged guest observation", "ObserveVolume", 2}, + {"closing disk inspection", "InspectDisk", 3}, + } { + t.Run(barrier.name, func(t *testing.T) { + f := newGrowthConcurrencyTest(t, barrier.method, barrier.occurrence) + owner := f.grow(t.Context(), growRequest()) + f.waitEntered(t, owner) + f.guestAccess(t) + f.e.unblock() + result := waitGrowthResult(t, owner) + if result.err != nil || result.info.Growth == nil || result.info.Growth.State != "completed" { + t.Fatalf("growth after releasing engine: %+v", result) + } + }) + } +} + +func TestManagedGrowthAdmissionAndDispatchRefuseLifecyclePromptly(t *testing.T) { + for _, scenario := range []struct { + method string + sleep bool + }{ + {"Inspect", false}, {"Inspect", true}, {"GrowDisk", false}, {"GrowDisk", true}, + } { + t.Run(fmt.Sprintf("%s managed sleep %t", scenario.method, scenario.sleep), func(t *testing.T) { + method := scenario.method + f := newGrowthConcurrencyTest(t, method, 1) + if scenario.sleep { + f.m.managedSleepHook = &config.ManagedVolumeHibernate{HookArgv: []string{"/volume-hook"}, HookTimeoutSeconds: 1} + } + owner := f.grow(t.Context(), growRequest()) + f.waitEntered(t, owner) + if method == "Inspect" { + current, err := f.m.GetVolume("data", "t1") + if err != nil || current.Growth != nil { + t.Fatalf("admission barrier must precede durable intent: %+v %v", current, err) + } + } + f.lifecycleRefused(t) + f.guestAccess(t) + f.e.unblock() + _ = waitGrowthResult(t, owner) + }) + } +} + +func TestManagedGrowthCanceledOwnerRetainsExclusionUntilEngineReturns(t *testing.T) { + for _, method := range []string{"Inspect", "GrowDisk"} { + t.Run(method, func(t *testing.T) { + f := newGrowthConcurrencyTest(t, method, 1) + f.m.managedSleepHook = &config.ManagedVolumeHibernate{HookArgv: []string{"/volume-hook"}, HookTimeoutSeconds: 1} + ctx, cancel := context.WithCancel(t.Context()) + defer cancel() + owner := f.grow(ctx, growRequest()) + f.waitEntered(t, owner) + cancel() + waiterCtx, cancelWaiter := context.WithCancel(t.Context()) + waiter := f.grow(waiterCtx, growRequest()) + cancelWaiter() + if err, returned := growthPrompt(t, "canceled identical waiter", f.call(func() error { + return (<-waiter).err + })); returned && !errors.Is(err, context.Canceled) { + t.Errorf("canceled waiter returned %v", err) + } + for _, changed := range []types.VolumeGrow{ + {SizeGiB: 3, OperationID: growRequest().OperationID}, + {SizeGiB: 2, OperationID: "c51b3d2b-4ccc-41d5-b430-3046340c30b0"}, + } { + done := f.grow(t.Context(), changed) + if err, returned := growthPrompt(t, "conflicting growth", f.call(func() error { return (<-done).err })); returned { + requireVolumeStatus(t, err, 409) + } + } + f.lifecycleRefused(t) + f.e.unblock() + result := waitGrowthResult(t, owner) + if !errors.Is(result.err, context.Canceled) { + t.Errorf("canceled owner returned %+v", result) + } + f.e.growMu.Lock() + calls := f.e.calls + f.e.growMu.Unlock() + current, err := f.m.GetVolume("data", "t1") + if err != nil { + t.Fatal(err) + } + if method == "Inspect" { + if calls != 0 || current.Growth != nil { + t.Errorf("canceled admission published intent or dispatched: calls=%d volume=%+v", calls, current) + } + } else if calls != 1 || current.Growth == nil || current.Growth.RuntimeOutcome != "acknowledged" { + t.Errorf("canceled dispatch lost exact outcome: calls=%d volume=%+v", calls, current) + } else { + retry := waitGrowthResult(t, f.grow(t.Context(), growRequest())) + if retry.err != nil || retry.info.Growth == nil || retry.info.Growth.State != "completed" { + t.Errorf("exact retry after canceled dispatch: %+v", retry) + } + f.e.growMu.Lock() + defer f.e.growMu.Unlock() + if f.e.calls != 1 { + t.Errorf("exact retry repeated the dispatched effect: %d calls", f.e.calls) + } + } + }) + } +} + +func TestManagedGrowthIdenticalRequestsJoinOneDispatch(t *testing.T) { + f := newGrowthConcurrencyTest(t, "Inspect", 1) + owner := f.grow(t.Context(), growRequest()) + f.waitEntered(t, owner) + joiner := f.grow(t.Context(), growRequest()) + select { + case result := <-joiner: + t.Fatalf("identical request returned before its owner: %+v", result) + case <-time.After(100 * time.Millisecond): + } + f.e.unblock() + for _, done := range []<-chan growthCallResult{owner, joiner} { + result := waitGrowthResult(t, done) + if result.err != nil || result.info.Growth == nil || result.info.Growth.State != "completed" || result.info.CapacityGeneration != 1 { + t.Errorf("joined growth: %+v", result) + } + } + f.e.growMu.Lock() + defer f.e.growMu.Unlock() + if f.e.calls != 1 { + t.Errorf("identical requests dispatched %d times", f.e.calls) + } +} + +func TestManagedGrowthCompletedReplayWaitsForDurableCommit(t *testing.T) { + for _, fail := range []bool{false, true} { + t.Run(fmt.Sprintf("write failure %t", fail), func(t *testing.T) { + f := newGrowthConcurrencyTest(t, "", 0) + write := f.m.managed.commitFile + entered, resume := make(chan struct{}), make(chan struct{}) + unblock := sync.OnceFunc(func() { close(resume) }) + t.Cleanup(unblock) + var once sync.Once + f.m.managed.commitFile = func(path string, raw []byte) error { + var saved volumeRegistryData + if err := json.Unmarshal(raw, &saved); err != nil { + return err + } + if v := saved.Volumes["data"]; v.Growth != nil && v.Growth.State == "completed" { + once.Do(func() { close(entered) }) + <-resume + if fail { + return errors.New("completion snapshot failed") + } + } + return write(path, raw) + } + owner := f.grow(t.Context(), growRequest()) + select { + case <-entered: + case result := <-owner: + t.Fatalf("growth returned before completion write: %+v", result) + case <-time.After(5 * time.Second): + t.Fatal("completion snapshot was not attempted") + } + raw, err := os.ReadFile(filepath.Join(f.m.managed.dir, "registry.json")) + if err != nil { + t.Fatal(err) + } + var persisted volumeRegistryData + if err := json.Unmarshal(raw, &persisted); err != nil { + t.Fatal(err) + } + if persisted.Volumes["data"].Growth.State == "completed" { + t.Fatal("completion barrier did not precede durable completion") + } + replay := f.grow(t.Context(), growRequest()) + var early *growthCallResult + select { + case result := <-replay: + early = &result + t.Errorf("exact replay returned before completion was durable: %+v", result) + case <-time.After(100 * time.Millisecond): + } + f.guestAccess(t) + unblock() + first := waitGrowthResult(t, owner) + second := growthCallResult{} + if early != nil { + second = *early + } else { + second = waitGrowthResult(t, replay) + } + if fail { + if first.err == nil || second.err == nil { + t.Errorf("failed completion reported success: owner=%+v replay=%+v", first, second) + } + requireVolumeStatus(t, f.m.Release(t.Context(), f.sb.ID, Cred{Operator: true}), 409) + } else { + for _, result := range []growthCallResult{first, second} { + if result.err != nil || result.info.Growth == nil || result.info.Growth.State != "completed" || result.info.CapacityGeneration != 1 { + t.Errorf("durable completed response: %+v", result) + } + } + } + f.e.growMu.Lock() + defer f.e.growMu.Unlock() + if f.e.calls != 1 { + t.Errorf("completion replay dispatched %d times", f.e.calls) + } + }) + } +} From a96ef7b60dcdb77a17897f250ba09b3fc78f0ea4 Mon Sep 17 00:00:00 2001 From: minatoaquaMK2 Date: Tue, 20 Oct 2026 14:01:57 +0000 Subject: [PATCH 39/75] fix(pool): reserve growth without blocking guest access Keep an exact in-flight growth owner through admission, engine calls, cancellation, and completion persistence. Fence lifecycle changes without holding the guest connection mutex, and expose committed capacity only after the completion snapshot is durable. --- sandboxd/pool/failover.go | 2 +- sandboxd/pool/managed_attachment.go | 2 +- sandboxd/pool/managed_grow.go | 478 ++++++++++++++---- .../pool/managed_grow_concurrency_test.go | 239 ++++++++- sandboxd/pool/managed_grow_recovery_test.go | 10 +- sandboxd/pool/managed_grow_test.go | 2 + sandboxd/pool/managed_volume.go | 4 + sandboxd/pool/volume_registry.go | 1 + 8 files changed, 615 insertions(+), 123 deletions(-) diff --git a/sandboxd/pool/failover.go b/sandboxd/pool/failover.go index 602d66d6..351e4d08 100644 --- a/sandboxd/pool/failover.go +++ b/sandboxd/pool/failover.go @@ -407,7 +407,7 @@ func (m *Manager) forgetAdopted(ctx context.Context, sb *types.Sandbox) func() { func (m *Manager) forgetClaim(ctx context.Context, sb *types.Sandbox) bool { m.mu.Lock() - if m.claimed[sb.ID] != sb { + if m.claimed[sb.ID] != sb || m.growthOwnsHold(sb.ID) { m.mu.Unlock() return false } diff --git a/sandboxd/pool/managed_attachment.go b/sandboxd/pool/managed_attachment.go index 32f8cc98..eb70b4ea 100644 --- a/sandboxd/pool/managed_attachment.go +++ b/sandboxd/pool/managed_attachment.go @@ -352,7 +352,7 @@ func (m *Manager) recoverManaged(ctx context.Context, claims map[string]*types.S r.mu.Lock() g := r.data.Volumes[h.Refs[0].Volume.Name].Growth r.mu.Unlock() - if !exists || !sleepMatchesClaim(h, sb) || g == nil || !m.growthRecordMatches(*g, sb, live[h.VMName]) { + if !exists || !sleepMatchesClaim(h, sb) || g == nil || !m.growthRecordMatches(*g, sb.VsockSocket, live[h.VMName]) { return owned, managedStorageError() } continue diff --git a/sandboxd/pool/managed_grow.go b/sandboxd/pool/managed_grow.go index 600918bf..108f36cc 100644 --- a/sandboxd/pool/managed_grow.go +++ b/sandboxd/pool/managed_grow.go @@ -26,6 +26,21 @@ type volumeGrowEngine interface { ObserveVolume(context.Context, string, string, string) (engine.GuestVolumeObservation, error) } +type growthRun struct { + claim *types.Sandbox + volume managedVolume + hold volumeHold + socket string + key types.PoolKey + layer types.PolicyLayer + policySource types.PoolKey + noEgress bool + egressClass string + request types.VolumeGrow + done chan struct{} + completionSeq uint64 // guarded by the registry mutex +} + type managedGrowth struct { Request types.VolumeGrow `json:"request"` Runtime engine.DiskGrowRequest `json:"runtime"` @@ -38,6 +53,10 @@ type managedGrowth struct { GuestSizeBytes int64 `json:"guest_size_bytes"` } +func (g *managedGrowth) sameOperation(other *managedGrowth) bool { + return g == nil && other == nil || g != nil && other != nil && g.Request == other.Request && g.Runtime.Equal(other.Runtime) && g.SandboxID == other.SandboxID && g.VMName == other.VMName && g.Guest == other.Guest +} + func (g *managedGrowth) fenced() bool { return g != nil && g.State != growthCompleted && !g.settled() } @@ -55,6 +74,9 @@ func growRecovery() error { } func (r *volumeRegistry) growthOwnsHold(id string) bool { + if r.growthRuns[id] != nil { + return true + } for _, v := range r.data.Volumes { if v.Growth != nil && (v.Growth.fenced() || r.fault != nil || r.writeFault != nil) && v.Growth.SandboxID == id { return true @@ -78,6 +100,11 @@ func (m *Manager) growthOwnsVM(name string) bool { } m.managed.mu.Lock() defer m.managed.mu.Unlock() + for _, run := range m.managed.growthRuns { + if run.hold.VMName == name { + return true + } + } for _, v := range m.managed.data.Volumes { if v.Growth != nil && (v.Growth.fenced() || m.managed.fault != nil || m.managed.writeFault != nil) && v.Growth.VMName == name { return true @@ -113,6 +140,15 @@ func (m *Manager) growthTeardownTarget(ctx context.Context, name string) (string return target, true } +func (r *volumeRegistry) activeGrowth(id volumeID) *growthRun { + for _, run := range r.growthRuns { + if run.volume.ID == id { + return run + } + } + return nil +} + func (r *volumeRegistry) growVolume(name, expectedID, tenant string, req types.VolumeGrow) (managedVolume, error) { if r.closed || r.fault != nil || r.writeFault != nil { return managedVolume{}, managedStorageError() @@ -127,6 +163,15 @@ func (r *volumeRegistry) growVolume(name, expectedID, tenant string, req types.V if string(v.ID) != expectedID { return v, volumeError(412, "volume_precondition_failed", "volume identity does not match") } + if run := r.activeGrowth(v.ID); run != nil { + if req == run.request { + return v, nil + } + if req.OperationID == run.request.OperationID { + return v, volumeError(409, "volume_grow_conflict", "operation ID has different parameters") + } + return v, growBusy() + } if v.Growth != nil && req.OperationID == v.Growth.Request.OperationID { if req != v.Growth.Request { return v, volumeError(409, "volume_grow_conflict", "operation ID has different parameters") @@ -155,63 +200,41 @@ func (m *Manager) GrowVolume(ctx context.Context, name, expectedID string, req t if !types.ValidVolumeName(name) || req.SizeGiB < 1 || req.SizeGiB > 1024 || !engine.ValidGrowID(req.OperationID) { return types.ManagedVolumeInfo{}, volumeError(400, "invalid_volume_grow", "invalid volume name, size_gib (1 through 1024), or canonical operation UUID") } - r := m.managed - if r == nil { + if m.managed == nil { return types.ManagedVolumeInfo{}, managedNotFound() } - r.mu.Lock() - v, err := r.growVolume(name, expectedID, tenant, req) + run, replay, err := m.beginGrowthRun(ctx, name, expectedID, req, tenant) if err != nil { - r.mu.Unlock() return types.ManagedVolumeInfo{}, err } - if v.Growth != nil && v.Growth.Request == req && !v.Growth.fenced() { - info := r.info(v) - r.mu.Unlock() - if v.Growth.State != growthCompleted { - return info, growRecovery() - } - return info, nil - } - var owner string - if v.Growth != nil && v.Growth.Request == req { - owner = v.Growth.SandboxID - } else { - holders := r.holders(v.ID) - if len(holders) == 1 { - owner = holders[0].SandboxID - } + if replay != nil { + return *replay, nil } - r.mu.Unlock() + defer m.finishGrowthRun(run) ge, supported := m.eng.(volumeGrowEngine) if !supported { return types.ManagedVolumeInfo{}, volumeError(503, "volume_grow_unsupported", "runtime does not support managed volume growth") } - m.mu.Lock() - sb := m.claimed[owner] - m.mu.Unlock() - if sb == nil { - return types.ManagedVolumeInfo{}, growRecovery() - } - sb.Transition.Lock() - defer sb.Transition.Unlock() - r.mu.Lock() - v, err = r.growVolume(name, expectedID, tenant, req) - r.mu.Unlock() - if err != nil { - return types.ManagedVolumeInfo{}, err - } - if v.Growth == nil || v.Growth.Request != req { - if err := m.admitGrowth(ctx, sb, v, req, ge); err != nil { + if run.volume.Growth == nil || run.volume.Growth.Request != req { + if err := m.admitGrowth(ctx, run, ge); err != nil { return types.ManagedVolumeInfo{}, err } } - if err := m.driveGrowth(ctx, name, sb, ge); err != nil { + if err := m.driveGrowth(ctx, name, run.socket, ge, run); err != nil { return types.ManagedVolumeInfo{}, err } + r := m.managed + m.mu.Lock() + defer m.mu.Unlock() r.mu.Lock() defer r.mu.Unlock() - v = r.data.Volumes[name] + if err := ctx.Err(); err != nil { + return types.ManagedVolumeInfo{}, err + } + if err := m.checkGrowthRunLocked(run); err != nil { + return types.ManagedVolumeInfo{}, err + } + v := r.data.Volumes[name] info := r.info(v) if v.Growth.State == growthRecovery { return info, growRecovery() @@ -219,38 +242,183 @@ func (m *Manager) GrowVolume(ctx context.Context, name, expectedID string, req t return info, nil } +func (m *Manager) beginGrowthRun(ctx context.Context, name, expectedID string, req types.VolumeGrow, tenant string) (*growthRun, *types.ManagedVolumeInfo, error) { + r := m.managed + for { + if err := ctx.Err(); err != nil { + return nil, nil, err + } + r.mu.Lock() + v, err := r.growVolume(name, expectedID, tenant, req) + if err != nil { + r.mu.Unlock() + return nil, nil, err + } + if active := r.activeGrowth(v.ID); active != nil { + r.mu.Unlock() + select { + case <-ctx.Done(): + return nil, nil, ctx.Err() + case <-active.done: + continue + } + } + if v.Growth != nil && v.Growth.Request == req && !v.Growth.fenced() { + info := r.info(v) + r.mu.Unlock() + if v.Growth.State != growthCompleted { + return nil, nil, growRecovery() + } + return nil, &info, nil + } + var owner string + if v.Growth != nil && v.Growth.Request == req { + owner = v.Growth.SandboxID + } else if holders := r.holders(v.ID); len(holders) == 1 { + owner = holders[0].SandboxID + } + r.mu.Unlock() + m.mu.Lock() + sb := m.claimed[owner] + m.mu.Unlock() + if sb == nil { + return nil, nil, growRecovery() + } + sb.Transition.Lock() + m.mu.Lock() + r.mu.Lock() + v, err = r.growVolume(name, expectedID, tenant, req) + if err == nil && (r.activeGrowth(v.ID) != nil || v.Growth != nil && v.Growth.Request == req && !v.Growth.fenced()) { + r.mu.Unlock() + m.mu.Unlock() + sb.Transition.Unlock() + continue + } + var h volumeHold + if err == nil { + h, err = m.growthHolderLocked(sb, v) + } + if err == nil && r.growthRuns[owner] != nil { + err = growBusy() + } + if err == nil { + err = ctx.Err() + } + var run *growthRun + if err == nil { + h.Refs = slices.Clone(h.Refs) + if v.Growth != nil { + g := *v.Growth + g.Runtime.Expected.Backing.FDs = slices.Clone(g.Runtime.Expected.Backing.FDs) + v.Growth = &g + } + run = &growthRun{claim: sb, volume: v, hold: h, socket: sb.VsockSocket, key: sb.Key, layer: sb.Layer, policySource: sb.PolicySource, noEgress: sb.NoEgress, egressClass: sb.EgressClass, request: req, done: make(chan struct{})} + if r.growthRuns == nil { + r.growthRuns = make(map[string]*growthRun) + } + r.growthRuns[owner] = run + } + r.mu.Unlock() + m.mu.Unlock() + sb.Transition.Unlock() + return run, nil, err + } +} + +func (m *Manager) finishGrowthRun(run *growthRun) { + r := m.managed + r.mu.Lock() + defer r.mu.Unlock() + if r.growthRuns[run.hold.SandboxID] == run { + delete(r.growthRuns, run.hold.SandboxID) + close(run.done) + } +} + func (m *Manager) growthHolderLocked(sb *types.Sandbox, v managedVolume) (volumeHold, error) { r := m.managed h, ok := r.data.Holds[sb.ID] _, outbound := m.view.Load().out.Resolve(sb, m.claimPooledLocked) _, removing := m.pendingRemovals[h.VMName] - if !ok || m.claimed[sb.ID] != sb || h.State != holdClaimed || h.Sleep != nil || !sleepMatchesClaim(h, sb) || sb.Failed != "" || sb.HibernateSnap != "" || sb.PendingSnap != "" || m.volumeRuns[sb] != nil || outbound || removing || h.Refs[0].ID != v.ID || !h.Refs[0].Volume.RW() || h.Refs[0].Volume.Mount == "" || len(r.holders(v.ID)) != 1 || r.growthOwnsHold(sb.ID) { + if !ok || m.claimed[sb.ID] != sb || h.State != holdClaimed || h.Sleep != nil || !sleepMatchesClaim(h, sb) || sb.Failed != "" || sb.HibernateSnap != "" || sb.PendingSnap != "" || m.volumeRuns[sb] != nil || outbound || removing || h.Refs[0].ID != v.ID || !h.Refs[0].Volume.RW() || h.Refs[0].Volume.Mount == "" || len(r.holders(v.ID)) != 1 { return h, growBusy() } return h, nil } +func (m *Manager) checkGrowthRun(ctx context.Context, run *growthRun) error { + if err := ctx.Err(); err != nil { + return err + } + m.mu.Lock() + defer m.mu.Unlock() + m.managed.mu.Lock() + defer m.managed.mu.Unlock() + return m.checkGrowthRunLocked(run) +} + +func (run *growthRun) matchesClaim(sb *types.Sandbox) bool { + return sb == run.claim && sb.ID == run.hold.SandboxID && sb.VMName == run.hold.VMName && sb.Tenant == run.hold.Tenant && sb.VsockSocket == run.socket && sb.Key == run.key && sb.Layer == run.layer && sb.PolicySource == run.policySource && sb.NoEgress == run.noEgress && sb.EgressClass == run.egressClass +} + +func (m *Manager) checkGrowthRunLocked(run *growthRun) error { + r := m.managed + if r.closed || r.fault != nil || r.writeFault != nil { + return managedStorageError() + } + sb := run.claim + v := r.data.Volumes[run.volume.Name] + if r.growthRuns[run.hold.SandboxID] != run || !run.matchesClaim(m.claimed[run.hold.SandboxID]) || v.ID != run.volume.ID || v.Name != run.volume.Name || v.Tenant != run.volume.Tenant { + return growRecovery() + } + h, err := m.growthHolderLocked(sb, v) + if err != nil { + return err + } + backing, generation := run.volume.Backing, run.volume.CapacityGeneration + if v.Growth != nil && v.Growth.Request == run.request && v.Growth.State == growthCompleted { + backing.Size = run.request.SizeGiB << 30 + generation++ + } + ref := run.hold.Refs[0] + ref.Backing = backing + if h.Process != run.hold.Process || h.Refs[0] != ref || v.Size != backing.Size || v.Backing != backing || v.CapacityGeneration != generation { + return growRecovery() + } + if v.Growth != nil && v.Growth.Request == run.request { + g := v.Growth + if run.volume.Growth != nil && run.volume.Growth.Request == run.request && !g.sameOperation(run.volume.Growth) { + return growRecovery() + } + if g.SandboxID != run.hold.SandboxID || g.VMName != run.hold.VMName || g.Runtime.Expected.VMM != (engine.DiskProcess{BootID: h.Process.BootID, PID: h.Process.PID, Start: h.Process.Start}) { + return growRecovery() + } + } else if !v.Growth.sameOperation(run.volume.Growth) || v.State != volumeReady { + return growRecovery() + } + return nil +} + func (r *volumeRegistry) growthBindingMatches(v managedVolume, h volumeHold, rec types.VMRecord, inspection engine.DiskInspection) bool { b := inspection.Live.Binding p, err := r.process(rec.PID) return err == nil && p == h.Process && b != nil && b.Validate() == nil && b.Ref == inspection.Requested && b.Ref.VMID == rec.ID && b.Ref.DiskID == rec.AttachedDevices.Disks[0].ID && b.VMM == (engine.DiskProcess{BootID: p.BootID, PID: p.PID, Start: p.Start}) && b.Serial == v.Name && b.Path == r.path(v.ID) && b.Backing.Device == v.Backing.Device && b.Backing.Inode == v.Backing.Inode && b.ObservedBytes == v.Size && b.Revision != ^uint64(0) } -func (m *Manager) admitGrowth(ctx context.Context, sb *types.Sandbox, v managedVolume, req types.VolumeGrow, ge volumeGrowEngine) error { - if m.managed.verify(v) != nil { +func (m *Manager) admitGrowth(ctx context.Context, run *growthRun, ge volumeGrowEngine) error { + r := m.managed + v, h := run.volume, run.hold + if r.verify(v) != nil { return managedStorageError() } - r := m.managed - m.mu.Lock() - r.mu.Lock() - h, err := m.growthHolderLocked(sb, v) - r.mu.Unlock() - m.mu.Unlock() - if err != nil { + if err := m.checkGrowthRun(ctx, run); err != nil { return err } rec, present, err := m.eng.Inspect(ctx, h.VMName) - if err != nil || !present || rec.ID == "" || rec.Config.Name != h.VMName || rec.State != vmStateRunning || len(rec.AttachedDevices.Disks) != 1 { + if checkErr := m.checkGrowthRun(ctx, run); checkErr != nil { + return checkErr + } + if err != nil || !present || rec.ID == "" || rec.Config.Name != h.VMName || rec.State != vmStateRunning || rec.VsockSocket != run.socket || len(rec.AttachedDevices.Disks) != 1 { return growRecovery() } disk := rec.AttachedDevices.Disks[0] @@ -258,32 +426,49 @@ func (m *Manager) admitGrowth(ctx context.Context, sb *types.Sandbox, v managedV return growRecovery() } inspection, err := ge.InspectDisk(ctx, engine.DiskRef{VMID: rec.ID, DiskID: disk.ID}) + if checkErr := m.checkGrowthRun(ctx, run); checkErr != nil { + return checkErr + } if err != nil || inspection.Error != nil || inspection.GuestBlockStatus != growthUnobserved || inspection.Live.State != growthObserved || inspection.Live.Binding == nil { return volumeError(503, "volume_grow_unsupported", "runtime cannot provide a qualified disk binding") } binding := *inspection.Live.Binding + binding.Backing.FDs = slices.Clone(binding.Backing.FDs) if !r.growthBindingMatches(v, h, rec, inspection) { return growRecovery() } + if err = m.checkGrowthRun(ctx, run); err != nil { + return err + } if inspection.LatestReceipt != nil && inspection.LatestReceipt.Outcome == growthUnknown { return growRecovery() } - guest, err := ge.ObserveVolume(ctx, rec.VsockSocket, v.Name, h.Refs[0].Volume.Mount) + guest, err := ge.ObserveVolume(ctx, run.socket, v.Name, h.Refs[0].Volume.Mount) + if checkErr := m.checkGrowthRun(ctx, run); checkErr != nil { + return checkErr + } if err != nil || !guest.Valid() || guest.Serial != v.Name || guest.SizeBytes != v.Size { return growRecovery() } - next := managedGrowth{Request: req, Runtime: engine.DiskGrowRequest{Expected: binding, ID: req.OperationID, TargetBytes: req.SizeGiB << 30}, SandboxID: sb.ID, VMName: h.VMName, Guest: guest, State: growthPending, Outcome: growthUnobserved} - return m.commitGrowthIntent(sb, v, next) + next := managedGrowth{Request: run.request, Runtime: engine.DiskGrowRequest{Expected: binding, ID: run.request.OperationID, TargetBytes: run.request.SizeGiB << 30}, SandboxID: h.SandboxID, VMName: h.VMName, Guest: guest, State: growthPending, Outcome: growthUnobserved} + return m.commitGrowthIntent(ctx, run, next) } -func (m *Manager) commitGrowthIntent(sb *types.Sandbox, v managedVolume, next managedGrowth) error { +func (m *Manager) commitGrowthIntent(ctx context.Context, run *growthRun, next managedGrowth) error { if err := m.store.commitDurable(m.store.mark()); err != nil { return err } + if err := m.checkGrowthRun(ctx, run); err != nil { + return err + } r := m.managed + v := run.volume if r.verify(v) != nil { return managedStorageError() } + if err := m.checkGrowthRun(ctx, run); err != nil { + return err + } r.mu.Lock() if r.data.Version != 3 { r.data.Version = 3 @@ -295,30 +480,26 @@ func (m *Manager) commitGrowthIntent(sb *types.Sandbox, v managedVolume, next ma r.mu.Unlock() m.mu.Lock() r.mu.Lock() - current, err := r.growVolume(v.Name, string(v.ID), v.Tenant, next.Request) - if err == nil && current.Growth != nil && current.Growth.Request == next.Request { - r.mu.Unlock() - m.mu.Unlock() - return nil - } + err := ctx.Err() if err == nil { - _, err = m.growthHolderLocked(sb, current) - } - if err == nil && (current.Size != v.Size || current.Backing != v.Backing) { - err = growRecovery() + err = m.checkGrowthRunLocked(run) } if err != nil { r.mu.Unlock() m.mu.Unlock() return err } + current := r.data.Volumes[v.Name] current.State = volumeGrowing current.Growth = &next r.data.Volumes[v.Name] = current m.mu.Unlock() err = r.commitAndRelock() r.mu.Unlock() - return err + if err != nil { + return err + } + return m.checkGrowthRun(ctx, run) } func (g managedGrowth) canDispatch(i engine.DiskInspection, err error) bool { @@ -347,18 +528,50 @@ func (m *Manager) growthOwnerMatches(ctx context.Context, g managedGrowth, sb *t return false } rec, present, err := m.eng.Inspect(ctx, g.VMName) - return err == nil && present && m.growthRecordMatches(g, sb, rec) + return err == nil && present && m.growthRecordMatches(g, sb.VsockSocket, rec) } -func (m *Manager) growthRecordMatches(g managedGrowth, sb *types.Sandbox, rec types.VMRecord) bool { - if sb == nil || rec.ID != g.Runtime.Expected.Ref.VMID || rec.Config.Name != g.VMName || rec.State != vmStateRunning || rec.VsockSocket != sb.VsockSocket || rec.PID != g.Runtime.Expected.VMM.PID { +func (m *Manager) growthRecordMatches(g managedGrowth, socket string, rec types.VMRecord) bool { + if socket == "" || rec.ID != g.Runtime.Expected.Ref.VMID || rec.Config.Name != g.VMName || rec.State != vmStateRunning || rec.VsockSocket != socket || rec.PID != g.Runtime.Expected.VMM.PID { return false } p, err := m.managed.process(rec.PID) return err == nil && p == (processIdentity{BootID: g.Runtime.Expected.VMM.BootID, PID: g.Runtime.Expected.VMM.PID, Start: g.Runtime.Expected.VMM.Start}) } -func (m *Manager) driveGrowth(ctx context.Context, name string, sb *types.Sandbox, ge volumeGrowEngine) error { +func (m *Manager) checkGrowthSource(ctx context.Context, g managedGrowth, socket string, run *growthRun) error { + if err := ctx.Err(); err != nil { + return err + } + if run != nil { + if err := m.checkGrowthRun(ctx, run); err != nil { + return err + } + } + rec, present, err := m.eng.Inspect(ctx, g.VMName) + if ctx.Err() != nil { + return ctx.Err() + } + if err != nil || !present || !m.growthRecordMatches(g, socket, rec) || len(rec.AttachedDevices.Disks) != 1 { + return growRecovery() + } + disk := rec.AttachedDevices.Disks[0] + b := g.Runtime.Expected + if disk.ID != b.Ref.DiskID || disk.Name != b.Serial || disk.Path != b.Path || disk.ReadOnly { + return growRecovery() + } + actual, err := inspectBacking(b.Path) + if err != nil || actual.Device != b.Backing.Device || actual.Inode != b.Backing.Inode || actual.Size != b.ObservedBytes && actual.Size != g.Runtime.TargetBytes { + return managedStorageError() + } + if run != nil { + return m.checkGrowthRun(ctx, run) + } + return ctx.Err() +} + +// A nil run is reserved for startup recovery before claims are published. +func (m *Manager) driveGrowth(ctx context.Context, name, socket string, ge volumeGrowEngine, run *growthRun) error { r := m.managed r.mu.Lock() v := r.data.Volumes[name] @@ -367,82 +580,111 @@ func (m *Manager) driveGrowth(ctx context.Context, name string, sb *types.Sandbo if !g.fenced() { return nil } + if run != nil { + if err := m.checkGrowthRun(ctx, run); err != nil { + return err + } + } inspection, inspectErr := ge.InspectDisk(ctx, g.Runtime.Expected.Ref) if inspection.Requested != g.Runtime.Expected.Ref || inspection.GuestBlockStatus != growthUnobserved { - return nil + return ctx.Err() } receipt, live := inspection.LatestReceipt, inspection.Live if receipt == nil || !receipt.Request.Equal(g.Runtime) { - if sb == nil { - return nil + if socket == "" || ctx.Err() != nil { + return ctx.Err() } if !g.canDispatch(inspection, inspectErr) { - return m.saveGrowth(v, g, growthRecovery, g.Outcome, 0) + return m.saveGrowth(v, g, growthRecovery, g.Outcome, 0, run) + } + if err := m.checkGrowthSource(ctx, g, socket, run); err != nil { + return err } result, _ := ge.GrowDisk(ctx, g.Runtime) receipt, live = result.Receipt, result.Live if receipt == nil { - return nil + return ctx.Err() } } if !receipt.Request.Equal(g.Runtime) || receipt.Revision != g.Runtime.Expected.Revision+1 { - return m.saveGrowth(v, g, growthRecovery, g.Outcome, 0) + return m.saveGrowth(v, g, growthRecovery, g.Outcome, 0, run) } + // Retain a returned receipt even when the engine ignored cancellation. switch receipt.Outcome { case growthUnknown: - return m.saveGrowth(v, g, growthRecovery, growthUnknown, 0) + if err := m.saveGrowth(v, g, growthRecovery, growthUnknown, 0, run); err != nil { + return err + } + return ctx.Err() case growthVMMGone: + if err := ctx.Err(); err != nil { + return err + } if !m.growthProcessStopped(ctx, g) { - return m.saveGrowth(v, g, growthRecovery, g.Outcome, 0) + return m.saveGrowth(v, g, growthRecovery, g.Outcome, 0, run) } - return m.saveGrowth(v, g, growthRecovery, growthVMMGone, 0) + return m.saveGrowth(v, g, growthRecovery, growthVMMGone, 0, run) case growthAcknowledged: + if err := m.saveGrowth(v, g, growthPending, growthAcknowledged, 0, run); err != nil { + return err + } + if err := ctx.Err(); err != nil { + return err + } if m.growthProcessStopped(ctx, g) { g.VMMGone = true - return m.saveGrowth(v, g, growthRecovery, growthAcknowledged, 0) + return m.saveGrowth(v, g, growthRecovery, growthAcknowledged, 0, run) } if !g.acknowledgedBinding(live, receipt.Revision) { - return m.saveGrowth(v, g, growthRecovery, growthAcknowledged, 0) + return m.saveGrowth(v, g, growthRecovery, growthAcknowledged, 0, run) } default: - return m.saveGrowth(v, g, growthRecovery, g.Outcome, 0) + return m.saveGrowth(v, g, growthRecovery, g.Outcome, 0, run) } - return m.observeGrowth(ctx, v, g, sb, ge, live, receipt.Revision) + return m.observeGrowth(ctx, v, g, socket, ge, live, receipt.Revision, run) } -func (m *Manager) observeGrowth(ctx context.Context, v managedVolume, g managedGrowth, sb *types.Sandbox, ge volumeGrowEngine, live engine.DiskObservation, revision uint64) error { - r := m.managed - if err := m.saveGrowth(v, g, growthPending, growthAcknowledged, 0); err != nil { +func (m *Manager) observeGrowth(ctx context.Context, v managedVolume, g managedGrowth, socket string, ge volumeGrowEngine, live engine.DiskObservation, revision uint64, run *growthRun) error { + if socket == "" { + return nil + } + if err := m.checkGrowthSource(ctx, g, socket, run); err != nil { return err } - if sb == nil { - return nil + guest, err := ge.ObserveVolume(ctx, socket, v.Name, g.Guest.Mount) + if run != nil { + if checkErr := m.checkGrowthRun(ctx, run); checkErr != nil { + return checkErr + } } - guest, err := ge.ObserveVolume(ctx, sb.VsockSocket, v.Name, g.Guest.Mount) if err != nil { - return nil + return ctx.Err() } if !guest.Valid() || !guest.SameIdentity(g.Guest) { - return m.saveGrowth(v, g, growthRecovery, growthAcknowledged, 0) + return m.saveGrowth(v, g, growthRecovery, growthAcknowledged, 0, run) } if guest.SizeBytes == g.Guest.SizeBytes { return nil } if guest.SizeBytes != g.Runtime.TargetBytes { - return m.saveGrowth(v, g, growthRecovery, growthAcknowledged, guest.SizeBytes) + return m.saveGrowth(v, g, growthRecovery, growthAcknowledged, guest.SizeBytes, run) } closing, closeErr := ge.InspectDisk(ctx, g.Runtime.Expected.Ref) + if run != nil { + if err := m.checkGrowthRun(ctx, run); err != nil { + return err + } + } if closeErr != nil || closing.Error != nil || closing.Requested != g.Runtime.Expected.Ref || closing.LatestReceipt == nil || !closing.LatestReceipt.Request.Equal(g.Runtime) || closing.LatestReceipt.Revision != revision || closing.LatestReceipt.Outcome != growthAcknowledged || closing.Live.State != growthObserved || closing.Live.Binding == nil || !closing.Live.Binding.Equal(*live.Binding) { - return m.saveGrowth(v, g, growthRecovery, growthAcknowledged, guest.SizeBytes) + return m.saveGrowth(v, g, growthRecovery, growthAcknowledged, guest.SizeBytes, run) } - p, processErr := r.process(g.Runtime.Expected.VMM.PID) - if processErr != nil || p != (processIdentity{BootID: g.Runtime.Expected.VMM.BootID, PID: g.Runtime.Expected.VMM.PID, Start: g.Runtime.Expected.VMM.Start}) { - return m.saveGrowth(v, g, growthRecovery, growthAcknowledged, guest.SizeBytes) + if err := m.checkGrowthSource(ctx, g, socket, run); err != nil { + return err } - return m.saveGrowth(v, g, growthCompleted, growthAcknowledged, guest.SizeBytes) + return m.saveGrowth(v, g, growthCompleted, growthAcknowledged, guest.SizeBytes, run) } -func (m *Manager) saveGrowth(v managedVolume, g managedGrowth, state, outcome string, guestSize int64) error { +func (m *Manager) saveGrowth(v managedVolume, g managedGrowth, state, outcome string, guestSize int64, run *growthRun) error { r := m.managed var actual backingIdentity if state == growthCompleted { @@ -452,18 +694,30 @@ func (m *Manager) saveGrowth(v managedVolume, g managedGrowth, state, outcome st return managedStorageError() } } + m.mu.Lock() r.mu.Lock() - defer r.mu.Unlock() current := r.data.Volumes[v.Name] - if r.fault != nil || current.ID != v.ID || current.Growth == nil || current.Growth.Request != g.Request { - return managedStorageError() + var err error + if r.closed || r.fault != nil || r.writeFault != nil || current.ID != v.ID || !current.Growth.sameOperation(&g) { + err = managedStorageError() + } else if run != nil { + if r.growthRuns[run.hold.SandboxID] != run { + err = growRecovery() + } else if state == growthCompleted { + err = m.checkGrowthRunLocked(run) + } + } + if err == nil && state == growthCompleted && current.CapacityGeneration != g.Request.ExpectedGeneration { + err = managedStorageError() + } + if err != nil { + r.mu.Unlock() + m.mu.Unlock() + return err } g.State, g.Outcome, g.GuestSizeBytes = state, outcome, guestSize current.Growth = &g if state == growthCompleted { - if !current.Growth.fenced() && current.CapacityGeneration != g.Request.ExpectedGeneration { - return managedStorageError() - } current.Size, current.Backing, current.State = actual.Size, actual, volumeReady current.CapacityGeneration++ for id, h := range r.data.Holds { @@ -479,7 +733,13 @@ func (m *Manager) saveGrowth(v managedVolume, g managedGrowth, state, outcome st current.State = volumeRecovery } r.data.Volumes[v.Name] = current - return r.commitAndRelock() + if state == growthCompleted && run != nil { + run.completionSeq = r.seq + 1 + } + m.mu.Unlock() + err = r.commitAndRelock() + r.mu.Unlock() + return err } func (r *volumeRegistry) validateGrowth(v managedVolume) error { @@ -575,10 +835,14 @@ func (m *Manager) recoverGrowth(ctx context.Context, claims map[string]*types.Sa for _, v := range pending { sb := claims[v.Growth.SandboxID] h, held := m.managedHold(v.Growth.SandboxID) - if sb != nil && (!held || !sleepMatchesClaim(h, sb) || sb.VMName != v.Growth.VMName || sb.Tenant != v.Tenant || !m.growthRecordMatches(*v.Growth, sb, live[v.Growth.VMName]) || !m.growthOwnerMatches(ctx, *v.Growth, sb)) { + if sb != nil && (!held || !sleepMatchesClaim(h, sb) || sb.VMName != v.Growth.VMName || sb.Tenant != v.Tenant || !m.growthRecordMatches(*v.Growth, sb.VsockSocket, live[v.Growth.VMName]) || !m.growthOwnerMatches(ctx, *v.Growth, sb)) { sb = nil } - if err := m.driveGrowth(ctx, v.Name, sb, ge); err != nil { + var socket string + if sb != nil { + socket = sb.VsockSocket + } + if err := m.driveGrowth(ctx, v.Name, socket, ge, nil); err != nil { return err } if m.growthOwnsHold(v.Growth.SandboxID) && sb == nil { diff --git a/sandboxd/pool/managed_grow_concurrency_test.go b/sandboxd/pool/managed_grow_concurrency_test.go index 4510a9a7..219153ad 100644 --- a/sandboxd/pool/managed_grow_concurrency_test.go +++ b/sandboxd/pool/managed_grow_concurrency_test.go @@ -147,14 +147,14 @@ func (f *growthConcurrencyTest) waitEntered(t *testing.T, owner <-chan growthCal } } -func growthPrompt(t *testing.T, name string, done <-chan error) (error, bool) { +func growthPrompt(t *testing.T, name string, done <-chan error) (bool, error) { t.Helper() select { case err := <-done: - return err, true + return true, err case <-time.After(time.Second): t.Errorf("%s waited for the blocked growth engine", name) - return nil, false + return false, nil } } @@ -204,7 +204,7 @@ func (f *growthConcurrencyTest) guestAccess(t *testing.T) { {"token DialPort", func() error { return f.dial(t.Context(), Cred{Token: f.sb.Token}) }}, } for _, check := range checks { - if err, returned := growthPrompt(t, check.name, f.call(check.call)); returned && err != nil { + if returned, err := growthPrompt(t, check.name, f.call(check.call)); returned && err != nil { t.Errorf("%s refused the running guest during growth: %v", check.name, err) } } @@ -241,7 +241,7 @@ func (f *growthConcurrencyTest) lifecycleRefused(t *testing.T) { {"Hibernate", func() error { return f.m.Hibernate(t.Context(), f.sb.ID, Cred{Operator: true}) }}, {"Release", func() error { return f.m.Release(t.Context(), f.sb.ID, Cred{Operator: true}) }}, } { - if err, returned := growthPrompt(t, call.name, f.call(call.fn)); returned { + if returned, err := growthPrompt(t, call.name, f.call(call.fn)); returned { if call.name == "Hibernate" && f.m.managedSleepHook == nil && errors.Is(err, ErrVolumeCapture) { continue } @@ -278,9 +278,12 @@ func TestManagedGrowthGuestConnectionsDuringEngineStalls(t *testing.T) { {"admission disk inspection", "InspectDisk", 1}, {"admission guest observation", "ObserveVolume", 1}, {"dispatch disk inspection", "InspectDisk", 2}, + {"dispatch VM qualification", "Inspect", 2}, {"disk growth", "GrowDisk", 1}, + {"acknowledged VM qualification", "Inspect", 3}, {"acknowledged guest observation", "ObserveVolume", 2}, {"closing disk inspection", "InspectDisk", 3}, + {"closing VM qualification", "Inspect", 4}, } { t.Run(barrier.name, func(t *testing.T) { f := newGrowthConcurrencyTest(t, barrier.method, barrier.occurrence) @@ -338,17 +341,17 @@ func TestManagedGrowthCanceledOwnerRetainsExclusionUntilEngineReturns(t *testing waiterCtx, cancelWaiter := context.WithCancel(t.Context()) waiter := f.grow(waiterCtx, growRequest()) cancelWaiter() - if err, returned := growthPrompt(t, "canceled identical waiter", f.call(func() error { + if returned, err := growthPrompt(t, "canceled identical waiter", f.call(func() error { return (<-waiter).err })); returned && !errors.Is(err, context.Canceled) { t.Errorf("canceled waiter returned %v", err) } for _, changed := range []types.VolumeGrow{ - {SizeGiB: 3, OperationID: growRequest().OperationID}, + {SizeGiB: 3, OperationID: growRequest().OperationID, ExpectedGeneration: 99}, {SizeGiB: 2, OperationID: "c51b3d2b-4ccc-41d5-b430-3046340c30b0"}, } { done := f.grow(t.Context(), changed) - if err, returned := growthPrompt(t, "conflicting growth", f.call(func() error { return (<-done).err })); returned { + if returned, err := growthPrompt(t, "conflicting growth", f.call(func() error { return (<-done).err })); returned { requireVolumeStatus(t, err, 409) } } @@ -365,13 +368,14 @@ func TestManagedGrowthCanceledOwnerRetainsExclusionUntilEngineReturns(t *testing if err != nil { t.Fatal(err) } - if method == "Inspect" { + switch { + case method == "Inspect": if calls != 0 || current.Growth != nil { t.Errorf("canceled admission published intent or dispatched: calls=%d volume=%+v", calls, current) } - } else if calls != 1 || current.Growth == nil || current.Growth.RuntimeOutcome != "acknowledged" { + case calls != 1 || current.Growth == nil || current.Growth.RuntimeOutcome != "acknowledged": t.Errorf("canceled dispatch lost exact outcome: calls=%d volume=%+v", calls, current) - } else { + default: retry := waitGrowthResult(t, f.grow(t.Context(), growRequest())) if retry.err != nil || retry.info.Growth == nil || retry.info.Growth.State != "completed" { t.Errorf("exact retry after canceled dispatch: %+v", retry) @@ -452,6 +456,14 @@ func TestManagedGrowthCompletedReplayWaitsForDurableCommit(t *testing.T) { if persisted.Volumes["data"].Growth.State == "completed" { t.Fatal("completion barrier did not precede durable completion") } + read, readErr := f.m.GetVolume("data", "t1") + if readErr != nil || read.CapacityGeneration != 0 || read.SizeBytes != 1<<30 || read.Growth == nil || read.Growth.State != growthPending || read.Growth.GuestSizeBytes != 2<<30 { + t.Errorf("GET published completion before durable commit: volume=%+v growth=%+v err=%v", read, read.Growth, readErr) + } + listed := f.m.Volumes("t1", nil) + if len(listed) != 1 || listed[0].SizeBytes != 1<<30 || listed[0].State != string(volumeGrowing) { + t.Errorf("list published completion before durable commit: %+v", listed) + } replay := f.grow(t.Context(), growRequest()) var early *growthCallResult select { @@ -469,12 +481,17 @@ func TestManagedGrowthCompletedReplayWaitsForDurableCommit(t *testing.T) { } else { second = waitGrowthResult(t, replay) } + read, readErr = f.m.GetVolume("data", "t1") if fail { + requireVolumeStatus(t, readErr, 503) if first.err == nil || second.err == nil { t.Errorf("failed completion reported success: owner=%+v replay=%+v", first, second) } requireVolumeStatus(t, f.m.Release(t.Context(), f.sb.ID, Cred{Operator: true}), 409) } else { + if readErr != nil || read.Growth == nil || read.Growth.State != growthCompleted || read.SizeBytes != 2<<30 || read.CapacityGeneration != 1 { + t.Errorf("GET after durable completion: volume=%+v growth=%+v err=%v", read, read.Growth, readErr) + } for _, result := range []growthCallResult{first, second} { if result.err != nil || result.info.Growth == nil || result.info.Growth.State != "completed" || result.info.CapacityGeneration != 1 { t.Errorf("durable completed response: %+v", result) @@ -489,3 +506,203 @@ func TestManagedGrowthCompletedReplayWaitsForDurableCommit(t *testing.T) { }) } } + +func TestManagedGrowthSourceChangesCannotDispatchOrComplete(t *testing.T) { + for _, stage := range []struct { + method string + occurrence int + dispatched bool + }{ + {"Inspect", 1, false}, + {"ObserveVolume", 2, true}, + } { + for _, change := range []string{"claim", "socket", "policy", "hold process", "held mount", "capacity generation", "runtime socket", "runtime process", "registry fault", "registry write fault"} { + t.Run(fmt.Sprintf("%s %s", stage.method, change), func(t *testing.T) { + f := newGrowthConcurrencyTest(t, stage.method, stage.occurrence) + owner := f.grow(t.Context(), growRequest()) + f.waitEntered(t, owner) + f.m.mu.Lock() + f.m.managed.mu.Lock() + switch change { + case "claim": + f.m.claimed[f.sb.ID] = &types.Sandbox{ID: f.sb.ID, VMName: f.sb.VMName} + case "socket": + f.sb.VsockSocket = "/replacement.sock" + case "policy": + f.sb.NoEgress = !f.sb.NoEgress + case "hold process": + h := f.m.managed.data.Holds[f.sb.ID] + h.Process.Start++ + f.m.managed.data.Holds[f.sb.ID] = h + case "held mount": + h := f.m.managed.data.Holds[f.sb.ID] + h.Refs[0].Volume.Mount = "/replacement" + f.m.managed.data.Holds[f.sb.ID] = h + case "capacity generation": + v := f.m.managed.data.Volumes["data"] + v.CapacityGeneration++ + f.m.managed.data.Volumes["data"] = v + case "registry fault": + f.m.managed.fault = errors.New("injected registry fault") + case "registry write fault": + f.m.managed.writeFault = errors.New("injected write fault") + } + f.m.managed.mu.Unlock() + f.m.mu.Unlock() + f.e.mu.Lock() + switch change { + case "runtime socket": + f.e.vms[f.sb.VMName] = "/replacement.sock" + case "runtime process": + f.e.pids[f.sb.VMName]++ + } + f.e.mu.Unlock() + f.e.unblock() + result := waitGrowthResult(t, owner) + if result.err == nil { + t.Fatalf("changed source returned success: %+v", result) + } + f.e.growMu.Lock() + calls := f.e.calls + f.e.growMu.Unlock() + f.m.managed.mu.Lock() + v := f.m.managed.data.Volumes["data"] + f.m.managed.mu.Unlock() + if stage.dispatched { + if calls != 1 || v.Growth == nil || v.Growth.Outcome != growthAcknowledged || v.Growth.State == growthCompleted { + t.Fatalf("changed source lost ACK or completed: calls=%d growth=%+v", calls, v.Growth) + } + } else if calls != 0 || v.Growth != nil { + t.Fatalf("changed admission source dispatched or published intent: calls=%d growth=%+v", calls, v.Growth) + } + }) + } + } +} + +func TestManagedGrowthCanceledWaiterDoesNotFinishOwner(t *testing.T) { + f := newGrowthConcurrencyTest(t, "Inspect", 1) + owner := f.grow(t.Context(), growRequest()) + f.waitEntered(t, owner) + ctx, cancel := context.WithCancel(t.Context()) + waiter := f.grow(ctx, growRequest()) + cancel() + if got := waitGrowthResult(t, waiter); !errors.Is(got.err, context.Canceled) { + t.Fatalf("canceled waiter: %+v", got) + } + f.lifecycleRefused(t) + f.e.unblock() + if got := waitGrowthResult(t, owner); got.err != nil || got.info.Growth.State != growthCompleted { + t.Fatalf("owner after waiter cancellation: %+v", got) + } +} + +func TestManagedGrowthStaleFinishKeepsCurrentReservation(t *testing.T) { + f := newGrowthConcurrencyTest(t, "", 0) + m, v := f.m, f.volume + first, _, err := m.beginGrowthRun(t.Context(), "data", v.ID, growRequest(), "t1") + if err != nil { + t.Fatal(err) + } + m.finishGrowthRun(first) + second, _, err := m.beginGrowthRun(t.Context(), "data", v.ID, growRequest(), "t1") + if err != nil { + t.Fatal(err) + } + defer m.finishGrowthRun(second) + m.finishGrowthRun(first) + requireVolumeStatus(t, m.Release(t.Context(), second.hold.SandboxID, Cred{Operator: true}), 409) + select { + case <-second.done: + t.Fatal("stale finish released the current owner") + default: + } +} + +func TestManagedGrowthAdmissionRefusesForgetClaim(t *testing.T) { + f := newGrowthConcurrencyTest(t, "Inspect", 1) + owner := f.grow(t.Context(), growRequest()) + f.waitEntered(t, owner) + if returned, err := growthPrompt(t, "forget claim", f.call(func() error { + if f.m.forgetClaim(t.Context(), f.sb) { + return errors.New("forgot the growth owner") + } + return nil + })); returned && err != nil { + t.Fatal(err) + } + if _, present := f.m.Sandbox(f.sb.ID); !present { + t.Fatal("growth claim disappeared") + } + f.lifecycleRefused(t) + f.e.unblock() + if got := waitGrowthResult(t, owner); got.err != nil || got.info.Growth.State != growthCompleted { + t.Fatalf("owner after refused forget: %+v", got) + } +} + +func TestManagedGrowthDurableCompletionIgnoresUnrelatedPendingWrite(t *testing.T) { + f := newGrowthConcurrencyTest(t, "", 0) + r := f.m.managed + write := r.commitFile + completionEntered, completionResume := make(chan struct{}), make(chan struct{}) + otherEntered, otherResume := make(chan struct{}), make(chan struct{}) + complete := sync.OnceFunc(func() { close(completionResume) }) + resumeOther := sync.OnceFunc(func() { close(otherResume) }) + t.Cleanup(func() { + complete() + resumeOther() + }) + var completionOnce, otherOnce sync.Once + r.commitFile = func(path string, raw []byte) error { + var saved volumeRegistryData + if err := json.Unmarshal(raw, &saved); err != nil { + return err + } + switch { + case saved.Volumes["other"].State == volumeCreating: + otherOnce.Do(func() { close(otherEntered) }) + <-otherResume + case saved.Volumes["data"].Growth != nil && saved.Volumes["data"].Growth.State == growthCompleted: + completionOnce.Do(func() { close(completionEntered) }) + <-completionResume + } + return write(path, raw) + } + owner := f.grow(t.Context(), growRequest()) + select { + case <-completionEntered: + case <-time.After(5 * time.Second): + t.Fatal("completion write did not start") + } + other := f.call(func() error { + _, _, err := f.m.CreateVolume(t.Context(), "other", types.VolumeCreate{SizeGiB: 1}, "t1") + return err + }) + waitFor(t, func() bool { + r.mu.Lock() + defer r.mu.Unlock() + return r.data.Volumes["other"].State == volumeCreating + }) + f.m.mu.Lock() + unlock := sync.OnceFunc(f.m.mu.Unlock) + t.Cleanup(unlock) + complete() + select { + case <-otherEntered: + case <-time.After(5 * time.Second): + t.Fatal("unrelated write did not start after completion") + } + read, err := f.m.GetVolume("data", "t1") + if err != nil || read.Growth == nil || read.Growth.State != growthCompleted || read.SizeBytes != 2<<30 || read.CapacityGeneration != 1 { + t.Errorf("unrelated write masked durable completion: volume=%+v growth=%+v err=%v", read, read.Growth, err) + } + unlock() + if got := waitGrowthResult(t, owner); got.err != nil || got.info.Growth == nil || got.info.Growth.State != growthCompleted { + t.Errorf("owner completion during unrelated write: %+v", got) + } + resumeOther() + if returned, err := growthPrompt(t, "unrelated create", other); returned && err != nil { + t.Fatal(err) + } +} diff --git a/sandboxd/pool/managed_grow_recovery_test.go b/sandboxd/pool/managed_grow_recovery_test.go index 9119cbb2..2ef581a8 100644 --- a/sandboxd/pool/managed_grow_recovery_test.go +++ b/sandboxd/pool/managed_grow_recovery_test.go @@ -424,15 +424,19 @@ func TestGrowthMissingInventoryCannotCompleteBeforeOwnerValidation(t *testing.T) func TestGrowthMissingInventoryPreservesUndispatchedIntent(t *testing.T) { m, e, sb, cfg, v := newGrowthTest(t) - source := m.managed.data.Volumes["data"] - if err := m.admitGrowth(t.Context(), sb, source, growRequest(), e); err != nil { + run, _, err := m.beginGrowthRun(t.Context(), "data", v.ID, growRequest(), "t1") + if err != nil { + t.Fatal(err) + } + if err = m.admitGrowth(t.Context(), run, e); err != nil { t.Fatal(err) } + m.finishGrowthRun(run) m.Close() m = openGrowthTest(t, cfg, e) m.eng = &growthInventoryEngine{growthTestEngine: e, omit: true, change: func(*types.VMRecord) {}} requireManagedRecoveryFence(t, m, sb.ID) - _, err := m.GetVolume("data", "t1") + _, err = m.GetVolume("data", "t1") requireVolumeStatus(t, err, 503) pending := m.managed.data.Volumes["data"] if pending.Growth.State != "pending" || pending.Growth.Outcome != "unobserved" || e.calls != 0 { diff --git a/sandboxd/pool/managed_grow_test.go b/sandboxd/pool/managed_grow_test.go index 94b5ce62..d1205dcb 100644 --- a/sandboxd/pool/managed_grow_test.go +++ b/sandboxd/pool/managed_grow_test.go @@ -389,6 +389,8 @@ func TestManagedGrowthAdmissionSerializesOrdinaryReleaseAndReap(t *testing.T) { t.Fatal(err) } finishReap() + // A busy sweep returns promptly; the next sweep processes the expired owner. + startGrowthTestReap(t, m)() if _, ok := m.Sandbox(sb.ID); ok { t.Fatal("expiry retained claim after growth completed") } diff --git a/sandboxd/pool/managed_volume.go b/sandboxd/pool/managed_volume.go index 35c1306d..02ba38f2 100644 --- a/sandboxd/pool/managed_volume.go +++ b/sandboxd/pool/managed_volume.go @@ -286,6 +286,10 @@ func (r *volumeRegistry) info(v managedVolume) types.ManagedVolumeInfo { info := types.ManagedVolumeInfo{CapacityGeneration: v.CapacityGeneration, Name: v.Name, DefaultMount: types.DefaultVolumeMount(v.Name), SizeBytes: v.Size, Available: v.State == volumeReady, Nodes: 1, Writable: true, Kind: "managed", ID: string(v.ID), Tenant: v.Tenant, Node: v.Node, State: string(v.State), CreatedAt: v.CreatedAt, Holders: r.holders(v.ID)} if g := v.Growth; g != nil { info.Growth = &types.VolumeGrowth{OperationID: g.Request.OperationID, ExpectedGeneration: g.Request.ExpectedGeneration, TargetBytes: g.Runtime.TargetBytes, State: g.State, RuntimeOutcome: g.Outcome, GuestSizeBytes: g.GuestSizeBytes, FilesystemStatus: "not_managed"} + if run := r.activeGrowth(v.ID); run != nil && g.Request == run.request && g.State == growthCompleted && run.completionSeq > r.written { + info.SizeBytes, info.CapacityGeneration = run.volume.Size, run.volume.CapacityGeneration + info.State, info.Available, info.Growth.State = string(volumeGrowing), false, growthPending + } } return info } diff --git a/sandboxd/pool/volume_registry.go b/sandboxd/pool/volume_registry.go index 024637d9..c5e54e63 100644 --- a/sandboxd/pool/volume_registry.go +++ b/sandboxd/pool/volume_registry.go @@ -107,6 +107,7 @@ type volumeRegistry struct { closed bool uncertainVMs map[string]bool discharging map[string]bool + growthRuns map[string]*growthRun aliases map[backingIdentity]bool dir, node string lock *os.File From b604dfeeb0d646c0fb1abf12d5e090656f4ba511 Mon Sep 17 00:00:00 2001 From: minatoaquaMK2 Date: Tue, 20 Oct 2026 14:16:36 +0000 Subject: [PATCH 40/75] test(pool): cover create retries before growth completion is durable --- .../pool/managed_grow_concurrency_test.go | 115 ++++++++++++++++++ 1 file changed, 115 insertions(+) diff --git a/sandboxd/pool/managed_grow_concurrency_test.go b/sandboxd/pool/managed_grow_concurrency_test.go index 219153ad..2a507c70 100644 --- a/sandboxd/pool/managed_grow_concurrency_test.go +++ b/sandboxd/pool/managed_grow_concurrency_test.go @@ -16,6 +16,7 @@ import ( "github.com/cocoonstack/sandbox/sandboxd/config" "github.com/cocoonstack/sandbox/sandboxd/engine" + "github.com/cocoonstack/sandbox/sandboxd/tenants" "github.com/cocoonstack/sandbox/sandboxd/types" ) @@ -706,3 +707,117 @@ func TestManagedGrowthDurableCompletionIgnoresUnrelatedPendingWrite(t *testing.T t.Fatal(err) } } + +func TestManagedGrowthCreateUsesCommittedCapacity(t *testing.T) { + for _, fail := range []bool{false, true} { + t.Run(fmt.Sprintf("write failure %t", fail), func(t *testing.T) { + f := newGrowthConcurrencyTest(t, "", 0) + f.m.managedSleepHook = &config.ManagedVolumeHibernate{HookArgv: []string{"/volume-hook"}, HookTimeoutSeconds: 1} + if err := f.m.PutTenant(t.Context(), tenants.TenantSpec{Name: "t2", Token: "other-token"}); err != nil { + t.Fatal(err) + } + r := f.m.managed + write := r.commitFile + entered, resume := make(chan struct{}), make(chan struct{}) + unblock := sync.OnceFunc(func() { close(resume) }) + t.Cleanup(unblock) + var once sync.Once + r.commitFile = func(path string, raw []byte) error { + var saved volumeRegistryData + if err := json.Unmarshal(raw, &saved); err != nil { + return err + } + if v := saved.Volumes["data"]; v.Growth != nil && v.Growth.State == growthCompleted { + once.Do(func() { close(entered) }) + <-resume + if fail { + return errors.New("completion snapshot failed") + } + } + return write(path, raw) + } + owner := f.grow(t.Context(), growRequest()) + select { + case <-entered: + case result := <-owner: + t.Fatalf("growth returned before completion write: %+v", result) + case <-time.After(5 * time.Second): + t.Fatal("completion snapshot was not attempted") + } + checkCreate := func(t *testing.T, sizeGiB int64, tenant string, status int, generation uint64, state string) { + t.Helper() + got, created, err := f.m.CreateVolume(t.Context(), "data", types.VolumeCreate{SizeGiB: sizeGiB}, tenant) + if status != 200 { + var refusal *VolumeError + if !errors.As(err, &refusal) || refusal.Status != status { + t.Errorf("create size %d tenant %s: created=%t volume=%+v growth=%+v err=%v, want status %d", sizeGiB, tenant, created, got, got.Growth, err, status) + } + return + } + if err != nil || created || got.ID != f.volume.ID || got.SizeBytes != sizeGiB<<30 || got.CapacityGeneration != generation || got.Growth == nil || got.Growth.State != state { + t.Errorf("identical create size %d: created=%t volume=%+v growth=%+v err=%v, want generation %d growth %s", sizeGiB, created, got, got.Growth, err, generation, state) + return + } + wantAvailable := state == growthCompleted + wantState := string(volumeGrowing) + if wantAvailable { + wantState = string(volumeReady) + } + if got.Available != wantAvailable || got.State != wantState { + t.Errorf("identical create size %d: available=%t state=%s, want available=%t state=%s", sizeGiB, got.Available, got.State, wantAvailable, wantState) + } + } + t.Run("blocked source capacity is identical", func(t *testing.T) { + checkCreate(t, 1, "t1", 200, 0, growthPending) + }) + t.Run("blocked target capacity conflicts", func(t *testing.T) { + checkCreate(t, 2, "t1", 409, 0, "") + }) + t.Run("blocked other tenant is concealed", func(t *testing.T) { + checkCreate(t, 1, "t2", 404, 0, "") + checkCreate(t, 2, "t2", 404, 0, "") + }) + t.Run("blocked lifecycle retains owner", f.lifecycleRefused) + t.Run("blocked delete is busy", func(t *testing.T) { + done := f.call(func() error { + return f.m.DeleteVolume(t.Context(), "data", f.volume.ID, "t1") + }) + if returned, err := growthPrompt(t, "delete", done); returned { + requireVolumeStatus(t, err, 409) + } + }) + t.Run("blocked claim is busy", func(t *testing.T) { + done := f.call(func() error { + _, err := f.m.ClaimProvision(t.Context(), testKey, managedOptions("data")) + return err + }) + if returned, err := growthPrompt(t, "claim", done); returned { + requireVolumeStatus(t, err, 409) + } + }) + unblock() + result := waitGrowthResult(t, owner) + if fail { + requireVolumeStatus(t, result.err, 503) + t.Run("failed completion fences creates", func(t *testing.T) { + checkCreate(t, 1, "t1", 503, 0, "") + checkCreate(t, 2, "t1", 503, 0, "") + }) + return + } + if result.err != nil || result.info.Growth == nil || result.info.Growth.State != growthCompleted { + t.Fatalf("durable completion failed: %+v", result) + } + t.Run("durable source capacity conflicts", func(t *testing.T) { + checkCreate(t, 1, "t1", 409, 0, "") + }) + t.Run("durable target capacity is identical", func(t *testing.T) { + checkCreate(t, 2, "t1", 200, 1, growthCompleted) + }) + t.Run("durable other tenant is concealed", func(t *testing.T) { + checkCreate(t, 1, "t2", 404, 0, "") + checkCreate(t, 2, "t2", 404, 0, "") + }) + }) + } +} From 22ab64232492370d0ac9c1e3c85d54038e1f1550 Mon Sep 17 00:00:00 2001 From: minatoaquaMK2 Date: Tue, 20 Oct 2026 14:18:36 +0000 Subject: [PATCH 41/75] fix(pool): validate volume creates against committed capacity --- sandboxd/pool/managed_grow.go | 4 +--- sandboxd/pool/managed_grow_test.go | 1 - sandboxd/pool/managed_volume.go | 26 ++++++++++++++++++++++---- 3 files changed, 23 insertions(+), 8 deletions(-) diff --git a/sandboxd/pool/managed_grow.go b/sandboxd/pool/managed_grow.go index 108f36cc..1c0c66db 100644 --- a/sandboxd/pool/managed_grow.go +++ b/sandboxd/pool/managed_grow.go @@ -38,7 +38,7 @@ type growthRun struct { egressClass string request types.VolumeGrow done chan struct{} - completionSeq uint64 // guarded by the registry mutex + completionSeq uint64 } type managedGrowth struct { @@ -570,7 +570,6 @@ func (m *Manager) checkGrowthSource(ctx context.Context, g managedGrowth, socket return ctx.Err() } -// A nil run is reserved for startup recovery before claims are published. func (m *Manager) driveGrowth(ctx context.Context, name, socket string, ge volumeGrowEngine, run *growthRun) error { r := m.managed r.mu.Lock() @@ -609,7 +608,6 @@ func (m *Manager) driveGrowth(ctx context.Context, name, socket string, ge volum if !receipt.Request.Equal(g.Runtime) || receipt.Revision != g.Runtime.Expected.Revision+1 { return m.saveGrowth(v, g, growthRecovery, g.Outcome, 0, run) } - // Retain a returned receipt even when the engine ignored cancellation. switch receipt.Outcome { case growthUnknown: if err := m.saveGrowth(v, g, growthRecovery, growthUnknown, 0, run); err != nil { diff --git a/sandboxd/pool/managed_grow_test.go b/sandboxd/pool/managed_grow_test.go index d1205dcb..fb16fbf6 100644 --- a/sandboxd/pool/managed_grow_test.go +++ b/sandboxd/pool/managed_grow_test.go @@ -389,7 +389,6 @@ func TestManagedGrowthAdmissionSerializesOrdinaryReleaseAndReap(t *testing.T) { t.Fatal(err) } finishReap() - // A busy sweep returns promptly; the next sweep processes the expired owner. startGrowthTestReap(t, m)() if _, ok := m.Sandbox(sb.ID); ok { t.Fatal("expiry retained claim after growth completed") diff --git a/sandboxd/pool/managed_volume.go b/sandboxd/pool/managed_volume.go index 02ba38f2..269b9ea8 100644 --- a/sandboxd/pool/managed_volume.go +++ b/sandboxd/pool/managed_volume.go @@ -71,7 +71,7 @@ func (m *Manager) CreateVolume(ctx context.Context, name string, req types.Volum return types.ManagedVolumeInfo{}, false, managedNotFound() } size := req.SizeGiB << 30 - if exists && (v.Size != size || v.Tenant != owner) { + if exists && (r.committedCapacity(v).size != size || v.Tenant != owner) { return types.ManagedVolumeInfo{}, false, volumeError(409, "volume_create_conflict", "volume already exists with different size or owner") } if !exists { @@ -282,12 +282,30 @@ func (r *volumeRegistry) holders(id volumeID) []types.VolumeHolder { return holders } +type volumeCapacity struct { + size int64 + generation uint64 + completionPending bool +} + +func (r *volumeRegistry) committedCapacity(v managedVolume) volumeCapacity { + capacity := volumeCapacity{size: v.Size, generation: v.CapacityGeneration} + g := v.Growth + if g == nil || g.State != growthCompleted { + return capacity + } + if run := r.activeGrowth(v.ID); run != nil && g.Request == run.request && run.completionSeq > r.written { + return volumeCapacity{size: run.volume.Size, generation: run.volume.CapacityGeneration, completionPending: true} + } + return capacity +} + func (r *volumeRegistry) info(v managedVolume) types.ManagedVolumeInfo { - info := types.ManagedVolumeInfo{CapacityGeneration: v.CapacityGeneration, Name: v.Name, DefaultMount: types.DefaultVolumeMount(v.Name), SizeBytes: v.Size, Available: v.State == volumeReady, Nodes: 1, Writable: true, Kind: "managed", ID: string(v.ID), Tenant: v.Tenant, Node: v.Node, State: string(v.State), CreatedAt: v.CreatedAt, Holders: r.holders(v.ID)} + capacity := r.committedCapacity(v) + info := types.ManagedVolumeInfo{CapacityGeneration: capacity.generation, Name: v.Name, DefaultMount: types.DefaultVolumeMount(v.Name), SizeBytes: capacity.size, Available: v.State == volumeReady, Nodes: 1, Writable: true, Kind: "managed", ID: string(v.ID), Tenant: v.Tenant, Node: v.Node, State: string(v.State), CreatedAt: v.CreatedAt, Holders: r.holders(v.ID)} if g := v.Growth; g != nil { info.Growth = &types.VolumeGrowth{OperationID: g.Request.OperationID, ExpectedGeneration: g.Request.ExpectedGeneration, TargetBytes: g.Runtime.TargetBytes, State: g.State, RuntimeOutcome: g.Outcome, GuestSizeBytes: g.GuestSizeBytes, FilesystemStatus: "not_managed"} - if run := r.activeGrowth(v.ID); run != nil && g.Request == run.request && g.State == growthCompleted && run.completionSeq > r.written { - info.SizeBytes, info.CapacityGeneration = run.volume.Size, run.volume.CapacityGeneration + if capacity.completionPending { info.State, info.Available, info.Growth.State = string(volumeGrowing), false, growthPending } } From 192724a39fc60538bc054d58a1c5baaf20003496 Mon Sep 17 00:00:00 2001 From: minatoaquaMK2 Date: Tue, 20 Oct 2026 15:04:04 +0000 Subject: [PATCH 42/75] test(pool): model runtime cancellation in growth barrier engine The real engine runs cocoon through exec.CommandContext, so a canceled request kills the runtime call. The barrier fake ignored the context and completed the grow anyway, which hid that a canceled growth request aborts the dispatched runtime effect. Generated with [Devin](https://devin.ai) Co-Authored-By: Devin <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- .../pool/managed_grow_concurrency_test.go | 19 ++++++++++++++----- 1 file changed, 14 insertions(+), 5 deletions(-) diff --git a/sandboxd/pool/managed_grow_concurrency_test.go b/sandboxd/pool/managed_grow_concurrency_test.go index 2a507c70..ba675d95 100644 --- a/sandboxd/pool/managed_grow_concurrency_test.go +++ b/sandboxd/pool/managed_grow_concurrency_test.go @@ -32,7 +32,7 @@ type growthBarrierEngine struct { connections sync.WaitGroup } -func (e *growthBarrierEngine) pause(method string) { +func (e *growthBarrierEngine) pause(ctx context.Context, method string) error { e.barrierMu.Lock() block := false if method == e.method { @@ -44,25 +44,34 @@ func (e *growthBarrierEngine) pause(method string) { close(e.entered) <-e.resume } + return ctx.Err() } func (e *growthBarrierEngine) Inspect(ctx context.Context, name string) (types.VMRecord, bool, error) { - e.pause("Inspect") + if err := e.pause(ctx, "Inspect"); err != nil { + return types.VMRecord{}, false, err + } return e.growthTestEngine.Inspect(ctx, name) } func (e *growthBarrierEngine) InspectDisk(ctx context.Context, ref engine.DiskRef) (engine.DiskInspection, error) { - e.pause("InspectDisk") + if err := e.pause(ctx, "InspectDisk"); err != nil { + return engine.DiskInspection{}, err + } return e.growthTestEngine.InspectDisk(ctx, ref) } func (e *growthBarrierEngine) GrowDisk(ctx context.Context, req engine.DiskGrowRequest) (engine.DiskGrowResult, error) { - e.pause("GrowDisk") + if err := e.pause(ctx, "GrowDisk"); err != nil { + return engine.DiskGrowResult{}, err + } return e.growthTestEngine.GrowDisk(ctx, req) } func (e *growthBarrierEngine) ObserveVolume(ctx context.Context, socket, serial, mount string) (engine.GuestVolumeObservation, error) { - e.pause("ObserveVolume") + if err := e.pause(ctx, "ObserveVolume"); err != nil { + return engine.GuestVolumeObservation{}, err + } return e.growthTestEngine.ObserveVolume(ctx, socket, serial, mount) } From 364679d811249b5c06d9e9c15ed18e89bb46c929 Mon Sep 17 00:00:00 2001 From: minatoaquaMK2 Date: Tue, 20 Oct 2026 15:04:19 +0000 Subject: [PATCH 43/75] fix(pool): keep growth dispatch alive after request cancellation driveGrowth passed the request context to GrowDisk, so a client timeout or disconnect SIGKILLed `cocoon vm disk grow` mid-effect. The docs tell clients to repeat the POST after a timeout, so this was the expected path, and it left the runtime outcome unknown. The read-only checks keep the request context; only the effect call is cancellation-immune, like removeVM, and cmdTimeout still bounds it. Generated with [Devin](https://devin.ai) Co-Authored-By: Devin <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- sandboxd/pool/managed_grow.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/sandboxd/pool/managed_grow.go b/sandboxd/pool/managed_grow.go index 1c0c66db..cdafa773 100644 --- a/sandboxd/pool/managed_grow.go +++ b/sandboxd/pool/managed_grow.go @@ -599,7 +599,8 @@ func (m *Manager) driveGrowth(ctx context.Context, name, socket string, ge volum if err := m.checkGrowthSource(ctx, g, socket, run); err != nil { return err } - result, _ := ge.GrowDisk(ctx, g.Runtime) + // cancellation-immune: a canceled request would kill `cocoon vm disk grow` mid-effect. + result, _ := ge.GrowDisk(context.WithoutCancel(ctx), g.Runtime) receipt, live = result.Receipt, result.Live if receipt == nil { return ctx.Err() From 804580a4f8e0a8fbcae073eab3412814d1182135 Mon Sep 17 00:00:00 2001 From: minatoaquaMK2 Date: Tue, 20 Oct 2026 15:06:41 +0000 Subject: [PATCH 44/75] refactor(pool): drop dead managed registry branches and duplicate writer - stageManagedLiveAdd can never see a ref name it is adding: additions are target minus run.holds, run.holds starts from the hold's refs, and only the exclusive volume run mutates those refs. - openVolumeRegistry returned r on both arms of the initial commit. - writeVolumeSnapshot reimplemented utils.WriteFileSync. Generated with [Devin](https://devin.ai) Co-Authored-By: Devin <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- sandboxd/pool/managed_attachment.go | 5 ----- sandboxd/pool/volume_registry.go | 22 +++------------------- 2 files changed, 3 insertions(+), 24 deletions(-) diff --git a/sandboxd/pool/managed_attachment.go b/sandboxd/pool/managed_attachment.go index eb70b4ea..f0f1a509 100644 --- a/sandboxd/pool/managed_attachment.go +++ b/sandboxd/pool/managed_attachment.go @@ -466,11 +466,6 @@ func (m *Manager) stageManagedLiveAdd(id string, volumes []resolvedVolume) error return managedStorageError() } for _, v := range volumes { - for _, ref := range h.Refs { - if ref.Volume.Name == v.applied.Name { - return volumeError(409, "volume_attachment_uncertain", "previous attachment requires sandbox release") - } - } h.Refs = append(h.Refs, volumeRef{Volume: v.applied, Path: v.disk.Path, Backing: v.backing}) } r.data.Holds[id] = h diff --git a/sandboxd/pool/volume_registry.go b/sandboxd/pool/volume_registry.go index c5e54e63..96a09e22 100644 --- a/sandboxd/pool/volume_registry.go +++ b/sandboxd/pool/volume_registry.go @@ -19,6 +19,7 @@ import ( "github.com/projecteru2/core/log" "github.com/cocoonstack/sandbox/sandboxd/types" + "github.com/cocoonstack/sandbox/sandboxd/utils" ) type ( @@ -157,11 +158,8 @@ func openVolumeRegistry(dir, node string) (*volumeRegistry, error) { return r, nil } r.mu.Lock() - err = r.commitAndRelock() + _ = r.commitAndRelock() r.mu.Unlock() - if err != nil { - return r, nil - } return r, nil } if err != nil { @@ -515,21 +513,7 @@ func (r *volumeRegistry) unboundRetirements() []string { } func writeVolumeSnapshot(path string, raw []byte) error { - tmp, err := os.CreateTemp(filepath.Dir(path), ".registry-") - if err != nil { - return err - } - defer func() { _ = os.Remove(tmp.Name()) }() - if _, err := tmp.Write(raw); err != nil { - return errors.Join(err, tmp.Close()) - } - if err := errors.Join(tmp.Sync(), tmp.Close()); err != nil { - return err - } - if err := os.Rename(tmp.Name(), path); err != nil { - return err - } - return syncDir(filepath.Dir(path)) + return utils.WriteFileSync(path, raw, 0o600) } func (r *volumeRegistry) close() error { From 0042b4b0bb14ed1cd1d54ad65a26ae2229d340a4 Mon Sep 17 00:00:00 2001 From: minatoaquaMK2 Date: Tue, 20 Oct 2026 15:35:25 +0000 Subject: [PATCH 45/75] refactor(pool): use errors.New, vmStateStopped and restore fenceWatch godoc - Verb-less fmt.Errorf calls in the managed registry and catalog conflict check become errors.New with the same text. - The two remaining "stopped" literals use vmStateStopped. - fenceWatch lost its godoc when managed streams were added; restore it with the managed case named. Generated with [Devin](https://devin.ai) Co-Authored-By: Devin <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- sandboxd/pool/lease.go | 1 + sandboxd/pool/managed_recovery.go | 2 +- sandboxd/pool/pool.go | 2 +- sandboxd/pool/reconcile.go | 2 +- sandboxd/pool/volume_registry.go | 14 +++++++------- 5 files changed, 11 insertions(+), 10 deletions(-) diff --git a/sandboxd/pool/lease.go b/sandboxd/pool/lease.go index 05c7c90e..6c035183 100644 --- a/sandboxd/pool/lease.go +++ b/sandboxd/pool/lease.go @@ -179,6 +179,7 @@ func (m *Manager) confirmOwnership(ctx context.Context, sb *types.Sandbox, rec a return false, fmt.Errorf("confirm %s: its row kept moving under this node", sb.ID) } +// fenceWatch gates sb at resolution and returns the channel that closes when its streams must end; nil for an unreplicated, unmanaged claim. func (m *Manager) fenceWatch(sb *types.Sandbox) (<-chan struct{}, error) { if !sb.Replicated() { m.mu.Lock() diff --git a/sandboxd/pool/managed_recovery.go b/sandboxd/pool/managed_recovery.go index d3e3fddc..415cd9e6 100644 --- a/sandboxd/pool/managed_recovery.go +++ b/sandboxd/pool/managed_recovery.go @@ -49,7 +49,7 @@ func (m *Manager) recoverManagedOrphans(ctx context.Context, claims map[string]* continue } identity, err := r.process(rec.PID) - if (rec.PID > 0 && processMissing(err)) || (rec.PID == 0 && rec.State == "stopped") { + if (rec.PID > 0 && processMissing(err)) || (rec.PID == 0 && rec.State == vmStateStopped) { _ = m.eng.Remove(context.WithoutCancel(ctx), name) if !m.confirmGone(ctx, name) { return managedStorageError() diff --git a/sandboxd/pool/pool.go b/sandboxd/pool/pool.go index 2ee5ea7d..0876ede2 100644 --- a/sandboxd/pool/pool.go +++ b/sandboxd/pool/pool.go @@ -578,7 +578,7 @@ func NewManager(ctx context.Context, cfg *config.Config, eng Engine, secrets *eg for name, entry := range m.volumes { _, collision := m.managed.data.Volumes[name] if collision || m.managedCatalogAlias(entry.disk.Path) { - return nil, errors.Join(fmt.Errorf("catalog conflicts with existing managed resource"), m.managed.close()) + return nil, errors.Join(errors.New("catalog conflicts with existing managed resource"), m.managed.close()) } } } diff --git a/sandboxd/pool/reconcile.go b/sandboxd/pool/reconcile.go index 9e09ba67..b840581d 100644 --- a/sandboxd/pool/reconcile.go +++ b/sandboxd/pool/reconcile.go @@ -327,7 +327,7 @@ func (m *Manager) stopUncertainEgress(ctx context.Context, name string) error { if err != nil { return fmt.Errorf("confirm egress VM %s stopped: %w", name, err) } - if found && rec.State != "stopped" { + if found && rec.State != vmStateStopped { return fmt.Errorf("egress VM %s is not confirmed stopped", name) } return nil diff --git a/sandboxd/pool/volume_registry.go b/sandboxd/pool/volume_registry.go index 96a09e22..e649a70d 100644 --- a/sandboxd/pool/volume_registry.go +++ b/sandboxd/pool/volume_registry.go @@ -125,7 +125,7 @@ var managedIDPattern = regexp.MustCompile(`^vol_[0-9a-f]{32}$`) func openVolumeRegistry(dir, node string) (*volumeRegistry, error) { if !filepath.IsAbs(dir) || filepath.Clean(dir) != dir || dir == "/" { - return nil, fmt.Errorf("invalid managed volume directory") + return nil, errors.New("invalid managed volume directory") } if err := privateVolumeDir(dir); err != nil { return nil, err @@ -195,13 +195,13 @@ func privateVolumeDir(path string) error { } owner := st.Sys().(*syscall.Stat_t).Uid if owner != 0 && int64(owner) != int64(os.Geteuid()) { - return fmt.Errorf("managed directory ancestors must belong to root or the daemon user") + return errors.New("managed directory ancestors must belong to root or the daemon user") } if st.Mode().Perm()&0o022 != 0 && st.Mode()&os.ModeSticky == 0 { - return fmt.Errorf("managed directory ancestors must not be writable by others") + return errors.New("managed directory ancestors must not be writable by others") } if !st.IsDir() || st.Mode()&os.ModeSymlink != 0 { - return fmt.Errorf("managed directory component is not a directory") + return errors.New("managed directory component is not a directory") } } if err := os.MkdirAll(path, 0o700); err != nil { @@ -212,10 +212,10 @@ func privateVolumeDir(path string) error { return err } if !st.IsDir() || st.Mode().Perm()&0o077 != 0 { - return fmt.Errorf("managed directory must be private (0700)") + return errors.New("managed directory must be private (0700)") } if int64(st.Sys().(*syscall.Stat_t).Uid) != int64(os.Geteuid()) { - return fmt.Errorf("managed directory must belong to the daemon user") + return errors.New("managed directory must belong to the daemon user") } return nil } @@ -229,7 +229,7 @@ func openPrivateFile(path string, flags int) (*os.File, error) { st, err := f.Stat() if err != nil || !st.Mode().IsRegular() || st.Mode().Perm()&0o077 != 0 || st.Sys().(*syscall.Stat_t).Nlink != 1 || int64(st.Sys().(*syscall.Stat_t).Uid) != int64(os.Geteuid()) { _ = f.Close() - return nil, fmt.Errorf("managed file must be private, owned, regular, and unlinked elsewhere") + return nil, errors.New("managed file must be private, owned, regular, and unlinked elsewhere") } return f, nil } From ebb4199efab803494356d7bb1e580d6954dab45f Mon Sep 17 00:00:00 2001 From: minatoaquaMK2 Date: Wed, 21 Oct 2026 00:33:35 +0000 Subject: [PATCH 46/75] test(server): cover managed hibernate and wake mount and runtime refusals Managed Hibernate and Wake return the engine's VolumeMountConflict for a busy, bind, foreign, or alias guest mount, and LiveVolumeUnsupported for a cocoon below the live-volume floor. Only the live volume route maps those, so the sleep verbs answer a generic 500 that hides the cause. Generated with [Devin](https://devin.ai) Co-Authored-By: Devin <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- sandboxd/server/server_test.go | 32 ++++++++++++++++++++++++++++++++ 1 file changed, 32 insertions(+) diff --git a/sandboxd/server/server_test.go b/sandboxd/server/server_test.go index 6debba66..86226de5 100644 --- a/sandboxd/server/server_test.go +++ b/sandboxd/server/server_test.go @@ -143,6 +143,38 @@ func TestHibernateVolumeCaptureMapsConflict(t *testing.T) { } } +func TestManagedSleepReportsGuestMountAndRuntimeRefusals(t *testing.T) { + const busy = `cannot unmount "/volumes/data"; close guest references and retry` + for _, tt := range []struct { + name string + err error + code int + cause string + }{ + {"busy unmount", errors.Join(&pool.VolumeMountConflict{Message: busy}, nil), http.StatusConflict, busy}, + {"below cocoon floor", &pool.LiveVolumeUnsupported{Version: "v0.7.1"}, http.StatusServiceUnavailable, "Cocoon >= v0.7.2"}, + } { + for _, verb := range []string{"hibernate", "wake"} { + t.Run(tt.name+"/"+verb, func(t *testing.T) { + refuse := func(string, string) error { return tt.err } + ts := newTestServer(t, "", &fakeManager{hibernate: refuse, wake: refuse}, nil) + resp := doReq(t, http.MethodPost, ts.URL+"/v1/sandboxes/sb_1/"+verb, "tok", "") + defer resp.Body.Close() + var got map[string]string + if err := json.UnmarshalRead(resp.Body, &got); err != nil { + t.Fatal(err) + } + if resp.StatusCode != tt.code || !strings.Contains(got["error"], tt.cause) { + t.Fatalf("status=%d body=%v, want %d naming %q", resp.StatusCode, got, tt.code, tt.cause) + } + if resp.Header.Get("Retry-After") != "" { + t.Fatalf("Retry-After=%q on a refusal that retrying alone cannot clear", resp.Header.Get("Retry-After")) + } + }) + } + } +} + func TestRenewGrantsAndReportsTheDeadline(t *testing.T) { want := time.Now().Add(time.Hour).UTC().Truncate(time.Second) var gotTTL time.Duration From 16a577d631cc15e6a8240292c4ca13618c73c827 Mon Sep 17 00:00:00 2001 From: minatoaquaMK2 Date: Wed, 21 Oct 2026 00:34:19 +0000 Subject: [PATCH 47/75] fix(server): map guest mount and runtime-floor refusals for every verb VolumeMountConflict and LiveVolumeUnsupported were mapped only by the live volume route, so managed Hibernate and Wake answered 500 for a busy guest mount or an old cocoon. The shared pool error mapping now answers 409 and 503 with their messages, matching joined errors too, and the volume route drops its copy. The runtime floor keeps omitting Retry-After because only an upgrade and restart clear it. Generated with [Devin](https://devin.ai) Co-Authored-By: Devin <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- sandboxd/server/server_http.go | 9 +++++++++ sandboxd/server/volume_mutation.go | 6 ------ 2 files changed, 9 insertions(+), 6 deletions(-) diff --git a/sandboxd/server/server_http.go b/sandboxd/server/server_http.go index e209eda8..cbaba9cb 100644 --- a/sandboxd/server/server_http.go +++ b/sandboxd/server/server_http.go @@ -186,6 +186,9 @@ func writePoolErr(w http.ResponseWriter, err error) bool { func mapPoolErr(w http.ResponseWriter, err error) (code int, msg string, ok bool) { code, msg, ok = lookupPoolErr(err) if ok && code == http.StatusServiceUnavailable { + if _, unsupported := errors.AsType[*pool.LiveVolumeUnsupported](err); unsupported { + return code, msg, ok + } if volumeErr, managed := errors.AsType[*pool.VolumeError](err); managed && (volumeErr.Code == "volume_create_incomplete" || volumeErr.Code == "volume_backing_mismatch" || volumeErr.Code == "volume_grow_unsupported") { return code, msg, ok @@ -199,6 +202,12 @@ func lookupPoolErr(err error) (code int, msg string, ok bool) { if volumeErr, ok := errors.AsType[*pool.VolumeError](err); ok { return volumeErr.Status, volumeErr.Message, true } + if conflict, ok := errors.AsType[*pool.VolumeMountConflict](err); ok { + return http.StatusConflict, conflict.Error(), true + } + if unsupported, ok := errors.AsType[*pool.LiveVolumeUnsupported](err); ok { + return http.StatusServiceUnavailable, unsupported.Error(), true + } for _, m := range poolErrHTTP { if errors.Is(err, m.err) { return m.code, cmp.Or(m.msg, err.Error()), true diff --git a/sandboxd/server/volume_mutation.go b/sandboxd/server/volume_mutation.go index 1e4f86f9..d3c52618 100644 --- a/sandboxd/server/volume_mutation.go +++ b/sandboxd/server/volume_mutation.go @@ -44,13 +44,7 @@ func (s *Server) handleSandboxVolumes(w http.ResponseWriter, r *http.Request) { code := http.StatusOK if err != nil { code, view.Error = http.StatusInternalServerError, "volume convergence failed; inspect the volume states before retrying" - conflict, isConflict := errors.AsType[*pool.VolumeMountConflict](err) - unsupported, isUnsupported := errors.AsType[*pool.LiveVolumeUnsupported](err) switch { - case isConflict: - code, view.Error = http.StatusConflict, conflict.Error() - case isUnsupported: - code, view.Error = http.StatusServiceUnavailable, unsupported.Error() case errors.Is(err, context.DeadlineExceeded): code, view.Error = http.StatusGatewayTimeout, "volume change timed out; inspect the volume states before retrying" case errors.Is(err, context.Canceled): From 8e8e16a20fa9ef316cb1b7b6a4cfc8193be5338d Mon Sep 17 00:00:00 2001 From: minatoaquaMK2 Date: Wed, 21 Oct 2026 00:35:05 +0000 Subject: [PATCH 48/75] test(engine): cover managed mount verification behind a symlinked parent mount(8) and MountLiveVolume follow a symlinked parent, so mountinfo lists the resolved path. VerifyMountedVolume compares the requested path literally, refusing a correct mount as a bind and naming no path in its conflicts, so hibernate preflight blocks a valid managed claim. Generated with [Devin](https://devin.ai) Co-Authored-By: Devin <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- sandboxd/engine/managed_sleep_test.go | 32 +++++++++++++++++++++++++++ 1 file changed, 32 insertions(+) diff --git a/sandboxd/engine/managed_sleep_test.go b/sandboxd/engine/managed_sleep_test.go index d9c5cf14..5f4dde76 100644 --- a/sandboxd/engine/managed_sleep_test.go +++ b/sandboxd/engine/managed_sleep_test.go @@ -3,6 +3,7 @@ package engine import ( "errors" "slices" + "strings" "testing" "time" ) @@ -67,6 +68,37 @@ func TestManagedMountRequiresLiteralWritableRoot(t *testing.T) { } } +func TestManagedMountResolvesASymlinkedParent(t *testing.T) { + for _, tc := range []struct { + name, mountinfo string + valid bool + }{ + {"mounted", "2 1 8:32 / /home/u/ws/data rw - ext4 /dev/vdc rw\n", true}, + {"read only", "2 1 8:32 / /home/u/ws/data ro - ext4 /dev/vdc ro\n", false}, + {"absent", "", false}, + } { + t.Run(tc.name, func(t *testing.T) { + sock := sockPath(t) + fake := serveFakeSilkd(t, sock) + configureVolumeDevices(fake) + fake.mu.Lock() + fake.serial["/sys/block/vdc/dev"] = []byte("8:32\n") + fake.serial["/proc/self/mountinfo"] = []byte(rootMountInfo + tc.mountinfo) + fake.realpath = map[string]string{"/workspace/data": "/home/u/ws/data"} + fake.mu.Unlock() + err := New("cocoon", nil, nil, false, false, "").VerifyMountedVolume(t.Context(), sock, "imagenet", "/workspace/data") + if tc.valid && err != nil { + t.Fatal(err) + } + if !tc.valid { + if _, ok := errors.AsType[*VolumeMountConflict](err); !ok || !strings.Contains(err.Error(), `"/home/u/ws/data"`) { + t.Fatalf("mount accepted or conflict hid the resolved guest path: %v", err) + } + } + }) + } +} + func TestHibernateSourceRequiresExactInspectMetadata(t *testing.T) { for _, tc := range []struct { name, listed, metadata string From 4638bea92dee748b667741b3fa2e9cbf81f20867 Mon Sep 17 00:00:00 2001 From: minatoaquaMK2 Date: Wed, 21 Oct 2026 00:35:50 +0000 Subject: [PATCH 49/75] fix(engine): resolve the managed mount path before verifying it VerifyMountedVolume compared the requested path with mountinfo literally, so a mount under a symlinked parent read as a bind and hibernate refused a valid claim. It now resolves the target like ObserveVolume and the live mount paths do, and its conflicts name the resolved guest path. The literal-root test now counts guest changes, not the read-only realpath lookup. Generated with [Devin](https://devin.ai) Co-Authored-By: Devin <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- sandboxd/engine/managed_sleep.go | 9 +++++---- sandboxd/engine/managed_sleep_test.go | 2 +- 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/sandboxd/engine/managed_sleep.go b/sandboxd/engine/managed_sleep.go index 5c6b95cd..14805725 100644 --- a/sandboxd/engine/managed_sleep.go +++ b/sandboxd/engine/managed_sleep.go @@ -68,18 +68,19 @@ func (e *Engine) VerifyMountedVolume(ctx context.Context, socket, name, mount st if err != nil { return err } + target := e.resolveMount(ctx, socket, mount, mounts) matched := false for _, entry := range mounts { - if entry.path != mount && entry.devNum != devNum { + if entry.path != target && entry.devNum != devNum { continue } - if matched || entry.path != mount || entry.devNum != devNum || entry.root != "/" || !slices.Contains(strings.Split(entry.options, ","), "rw") { - return &VolumeMountConflict{Message: "managed volume must have one exact writable root mount"} + if matched || entry.path != target || entry.devNum != devNum || entry.root != "/" || !slices.Contains(strings.Split(entry.options, ","), "rw") { + return &VolumeMountConflict{Message: fmt.Sprintf("managed volume must have one exact writable root mount at %q", target)} } matched = true } if !matched { - return &VolumeMountConflict{Message: "managed volume mount is absent"} + return &VolumeMountConflict{Message: fmt.Sprintf("managed volume mount %q is absent", target)} } return nil } diff --git a/sandboxd/engine/managed_sleep_test.go b/sandboxd/engine/managed_sleep_test.go index 5f4dde76..f36d4f42 100644 --- a/sandboxd/engine/managed_sleep_test.go +++ b/sandboxd/engine/managed_sleep_test.go @@ -61,7 +61,7 @@ func TestManagedMountRequiresLiteralWritableRoot(t *testing.T) { } fake.mu.Lock() defer fake.mu.Unlock() - if len(fake.execCalls) != 0 { + if len(guestChanges(fake.execCalls)) != 0 { t.Fatal("mount preflight changed guest") } }) From 63fbe9c3d1fedc3df4d5e2b38249c5677517728d Mon Sep 17 00:00:00 2001 From: minatoaquaMK2 Date: Wed, 21 Oct 2026 00:38:16 +0000 Subject: [PATCH 50/75] test(pool): cover managed expiry and sleep-capable release during a volume run Managed expiry and release of a sleep-capable claim go through releaseManagedSleepLocked, which drops the claim without cancelling its live volume run. The run keeps issuing device calls against a removed sandbox, a catalog ref the run already dropped from the hold keeps its admission charge, and the claim's per-volume diagnostics outlive it. Generated with [Devin](https://devin.ai) Co-Authored-By: Devin <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- sandboxd/pool/managed_integration_test.go | 147 ++++++++++++++++++++++ 1 file changed, 147 insertions(+) diff --git a/sandboxd/pool/managed_integration_test.go b/sandboxd/pool/managed_integration_test.go index fc84a893..1f5b48f6 100644 --- a/sandboxd/pool/managed_integration_test.go +++ b/sandboxd/pool/managed_integration_test.go @@ -6,6 +6,7 @@ import ( "os" "path/filepath" "strings" + "sync" "testing" "time" @@ -258,6 +259,152 @@ func TestManagedReleaseCancelsExistingVolumeRunAndRetainsFullUnion(t *testing.T) } } +func TestManagedExpiryCancelsExistingVolumeRun(t *testing.T) { + cfg := managedConfig(t) + cfg.Volumes = []config.VolumeSpec{{Name: "catalog", Path: writeVolumeImage(t, "catalog.img", "catalog"), Writable: true}} + e := &managedCancelEngine{managedTestEngine: newManagedEngine(), entered: make(chan struct{})} + m, err := NewManager(t.Context(), cfg, e, testSecrets(t), testBackends(t, cfg)) + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { m.Close() }) + m.managed.process = e.identity + if err := m.PutTenant(t.Context(), tenants.TenantSpec{Name: "t1", Token: "t1-token"}); err != nil { + t.Fatal(err) + } + createManagedTest(t, m, "data") + sb, err := m.ClaimProvision(t.Context(), testKey, managedOptions("data")) + if err != nil { + t.Fatal(err) + } + done := make(chan error, 1) + go func() { + _, err := m.PatchVolumes(t.Context(), sb.ID, "t1", types.VolumePatch{Add: []types.Volume{{Name: "catalog"}}}) + done <- err + }() + <-e.entered + m.mu.Lock() + sb.Deadline = time.Now().Add(-time.Second) + m.mu.Unlock() + m.reapOnce(t.Context()) + waitFor(t, func() bool { _, held := m.managedHold(sb.ID); return !held }) + select { + case err := <-done: + if !errors.Is(err, context.Canceled) { + t.Fatalf("mutation cancellation: %v", err) + } + case <-time.After(time.Second): + t.Fatal("expiry did not cancel the live volume run") + } + if got := volumeHoldersOf(m, "catalog"); got != (volumeHolders{}) { + t.Fatalf("expired claim kept the canceled run's charge: %+v", got) + } +} + +func TestManagedExpiryDropsVolumeDiagnostics(t *testing.T) { + cfg := managedConfig(t) + cfg.Volumes = []config.VolumeSpec{{Name: "catalog", Path: writeVolumeImage(t, "catalog.img", "catalog")}} + e := newManagedEngine() + m := openManagedTest(t, cfg, e) + createManagedTest(t, m, "data") + sb := foundationClaim(t, m, "data") + if _, err := m.PatchVolumes(t.Context(), sb.ID, "t1", types.VolumePatch{Add: []types.Volume{{Name: "catalog"}}}); err != nil { + t.Fatal(err) + } + e.detachVolumeErr = &engine.VolumeMountConflict{Message: `cannot unmount "/volumes/catalog"; close guest references and retry`} + if _, err := m.PatchVolumes(t.Context(), sb.ID, "t1", types.VolumePatch{Remove: []string{"catalog"}}); err == nil { + t.Fatal("busy detach succeeded") + } + e.detachVolumeErr = nil + m.mu.Lock() + recorded := len(m.volumeErrors[sb]) + sb.Deadline = time.Now().Add(-time.Second) + m.mu.Unlock() + if recorded == 0 { + t.Fatal("busy detach recorded no diagnostic") + } + m.reapOnce(t.Context()) + waitFor(t, func() bool { _, live := m.Sandbox(sb.ID); return !live }) + m.mu.Lock() + defer m.mu.Unlock() + if leaked := m.volumeErrors[sb]; leaked != nil { + t.Fatalf("expired claim kept its diagnostics: %v", leaked) + } +} + +func TestManagedTeardownDuringLiveRemovalReleasesCatalogCharge(t *testing.T) { + for _, path := range []string{"release", "reap", "sleep-capable release"} { + t.Run(path, func(t *testing.T) { + cfg := managedConfig(t) + cfg.Volumes = []config.VolumeSpec{{Name: "catalog", Path: writeVolumeImage(t, "catalog.img", "catalog"), Writable: true}} + opts := managedOptions("data") + base := newManagedEngine() + var eng Engine = base + process := base.identity + if path == "sleep-capable release" { + cfg.ManagedVolumeHibernate = &config.ManagedVolumeHibernate{HookArgv: []string{"/usr/local/bin/volume-hook"}, HookTimeoutSeconds: 1} + se := &sleepTestEngine{managedTestEngine: base, generation: 2, reason: "boot", vmID: "immutable-vm", mounted: true} + eng, process = se, se.identity + opts.Volumes[0].AttachOnly = false + } + m, err := NewManager(t.Context(), cfg, eng, testSecrets(t), testBackends(t, cfg)) + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { _ = m.Close() }) + m.managed.process = process + if err := m.PutTenant(t.Context(), tenants.TenantSpec{Name: "t1", Token: "t1-token"}); err != nil { + t.Fatal(err) + } + createManagedTest(t, m, "data") + sb, err := m.ClaimProvision(t.Context(), testKey, opts) + if err != nil { + t.Fatal(err) + } + if _, err := m.PatchVolumes(t.Context(), sb.ID, "t1", types.VolumePatch{Add: []types.Volume{{Name: "catalog"}}}); err != nil { + t.Fatal(err) + } + entered, resume := make(chan struct{}), make(chan struct{}) + var once sync.Once + m.managed.mu.Lock() + m.managed.commitFile = func(p string, raw []byte) error { + once.Do(func() { close(entered); <-resume }) + return writeVolumeSnapshot(p, raw) + } + m.managed.mu.Unlock() + patched := make(chan error, 1) + go func() { + _, err := m.PatchVolumes(t.Context(), sb.ID, "t1", types.VolumePatch{Remove: []string{"catalog"}}) + patched <- err + }() + <-entered + torn := make(chan struct{}) + go func() { + defer close(torn) + if path == "reap" { + m.mu.Lock() + sb.Deadline = time.Now().Add(-time.Second) + m.mu.Unlock() + m.reapOnce(t.Context()) + return + } + _ = m.Release(t.Context(), sb.ID, Cred{Operator: true}) + }() + waitFor(t, func() bool { _, live := m.Sandbox(sb.ID); return !live }) + close(resume) + <-torn + <-patched + waitFor(t, func() bool { _, held := m.managedHold(sb.ID); return !held }) + if got := volumeHoldersOf(m, "catalog"); got != (volumeHolders{}) { + t.Fatalf("%s during a live catalog removal kept its charge: %+v", path, got) + } + if _, err := m.ClaimProvision(t.Context(), testKey, managedOptions("catalog")); err != nil { + t.Fatalf("catalog writer refused after teardown: %v", err) + } + }) + } +} + func TestManagedQuarantineStillRemovesVMWhenRetirementCannotCommit(t *testing.T) { cfg := managedConfig(t) e := newManagedEngine() From b5be2a66c563de9b7cfc275859dd05c9a972760a Mon Sep 17 00:00:00 2001 From: minatoaquaMK2 Date: Wed, 21 Oct 2026 00:38:51 +0000 Subject: [PATCH 51/75] fix(pool): cancel the volume run when a managed lifecycle claim is dropped releaseManagedSleepLocked, used by managed expiry and by release of a sleep-capable claim, dropped the claim without cancelVolumeRun. The run kept calling the device, its diagnostics stayed in volumeErrors, and a catalog ref it had already dropped from the hold never left admission. Cancel it under m.mu like releaseResolved and charge its holds to the teardown. Generated with [Devin](https://devin.ai) Co-Authored-By: Devin <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- sandboxd/pool/managed_sleep_recovery.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/sandboxd/pool/managed_sleep_recovery.go b/sandboxd/pool/managed_sleep_recovery.go index 8f2e923b..a9337203 100644 --- a/sandboxd/pool/managed_sleep_recovery.go +++ b/sandboxd/pool/managed_sleep_recovery.go @@ -37,6 +37,7 @@ func (m *Manager) releaseManagedSleepLocked(ctx context.Context, sb *types.Sandb return growBusy() } previousHold, _ := m.managedHold(sb.ID) + volumeHolds := m.cancelVolumeRun(sb) m.dropClaimLocked(sb) js := m.store.del(sb.ID) m.mu.Unlock() @@ -58,6 +59,7 @@ func (m *Manager) releaseManagedSleepLocked(ctx context.Context, sb *types.Sandb if h.Sleep == nil { td = m.quiesceVolumes(ctx, sb) } + td.holds = volumeHolds removed := m.removeOrRetry(ctx, sb.VMName, sb.ID, td) m.out.Disarm(sb.ID, removed) if !removed { From 8c112239cd9214dca1e8f540f998561ffd54c324 Mon Sep 17 00:00:00 2001 From: minatoaquaMK2 Date: Wed, 21 Oct 2026 00:41:35 +0000 Subject: [PATCH 52/75] test(pool): cover catalog admission after a failed managed live removal write When a live catalog removal drops the ref from the managed hold in memory but the registry write fails, commitVolumeRun and the dirty catalog path in stageVolumeTarget return before releasing admission, and finishVolumeRun skips managed claims. The background retrier persists the drop, yet the catalog stays busy until restart. Generated with [Devin](https://devin.ai) Co-Authored-By: Devin <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- sandboxd/pool/managed_integration_test.go | 64 +++++++++++++++++++++++ 1 file changed, 64 insertions(+) diff --git a/sandboxd/pool/managed_integration_test.go b/sandboxd/pool/managed_integration_test.go index 1f5b48f6..3ccd143c 100644 --- a/sandboxd/pool/managed_integration_test.go +++ b/sandboxd/pool/managed_integration_test.go @@ -7,6 +7,7 @@ import ( "path/filepath" "strings" "sync" + "sync/atomic" "testing" "time" @@ -405,6 +406,69 @@ func TestManagedTeardownDuringLiveRemovalReleasesCatalogCharge(t *testing.T) { } } +func TestManagedLiveRemovalReleasesCatalogChargeWhenRegistryWriteFails(t *testing.T) { + for _, tc := range []struct { + name string + dirty bool + goodWrites int32 + }{ + {"detached", false, 0}, + {"dirty catalog refused", true, 1}, + } { + t.Run(tc.name, func(t *testing.T) { + cfg := managedConfig(t) + catalog := writeVolumeImage(t, "catalog.img", "catalog") + cfg.Volumes = []config.VolumeSpec{{Name: "catalog", Path: catalog, Writable: true}} + m := openManagedTest(t, cfg, newManagedEngine()) + createManagedTest(t, m, "data") + sb := foundationClaim(t, m, "data") + patch := types.VolumePatch{Add: []types.Volume{{Name: "catalog"}}} + if tc.dirty { + if err := markVolumeDirty(catalog); err != nil { + t.Fatal(err) + } + } else { + if _, err := m.PatchVolumes(t.Context(), sb.ID, "t1", patch); err != nil { + t.Fatal(err) + } + patch = types.VolumePatch{Remove: []string{"catalog"}} + } + var writes atomic.Int32 + writes.Store(tc.goodWrites) + m.managed.mu.Lock() + m.managed.commitFile = func(path string, raw []byte) error { + if writes.Add(-1) < 0 { + return errors.New("injected registry write failure") + } + return writeVolumeSnapshot(path, raw) + } + m.managed.mu.Unlock() + if _, err := m.PatchVolumes(t.Context(), sb.ID, "t1", patch); err == nil { + t.Fatal("registry write failure hidden") + } + writes.Store(1 << 20) + m.retryManagedSnapshot() + waitFor(t, func() bool { return m.ManagedVolumeStatus() == nil && !m.managed.retrying.Load() }) + if err := clearVolumeDirty(catalog); err != nil { + t.Fatal(err) + } + if view, _ := m.GetVolumes(sb.ID, "t1"); len(view.States) != 1 { + t.Fatalf("claim still lists the catalog: %+v", view) + } + writer, err := m.ClaimProvision(t.Context(), testKey, managedOptions("catalog")) + if err != nil { + t.Fatalf("catalog the claim no longer holds is still busy: %v", err) + } + for _, id := range []string{writer.ID, sb.ID} { + if err := m.Release(t.Context(), id, Cred{Operator: true}); err != nil { + t.Fatal(err) + } + } + waitFor(t, func() bool { return volumeHoldersOf(m, "catalog") == (volumeHolders{}) }) + }) + } +} + func TestManagedQuarantineStillRemovesVMWhenRetirementCannotCommit(t *testing.T) { cfg := managedConfig(t) e := newManagedEngine() From 6118f5a117fc20178a5631401f569c34c7f84d8b Mon Sep 17 00:00:00 2001 From: minatoaquaMK2 Date: Wed, 21 Oct 2026 00:42:21 +0000 Subject: [PATCH 53/75] fix(pool): release catalog admission once the managed hold drops the ref managedLiveRemove drops refs from the hold in memory before writing the registry, and the retrier persists that drop after a write failure. The callers returned on that error before releasing admission, and finishVolumeRun skips managed claims, so the catalog stayed busy until restart. Both callers now release whatever the hold no longer references, which is nothing when managedLiveRemove refused first. Generated with [Devin](https://devin.ai) Co-Authored-By: Devin <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- sandboxd/pool/volume_mutation.go | 18 ++++++++++++++---- 1 file changed, 14 insertions(+), 4 deletions(-) diff --git a/sandboxd/pool/volume_mutation.go b/sandboxd/pool/volume_mutation.go index c301007d..f4ba0f62 100644 --- a/sandboxd/pool/volume_mutation.go +++ b/sandboxd/pool/volume_mutation.go @@ -339,14 +339,13 @@ func (m *Manager) stageVolumeTarget(ctx context.Context, sb *types.Sandbox, run removeErr := m.managedLiveRemove(sb.ID, additions) sb.Transition.Lock() m.mu.Lock() + m.releaseUnheldLocked(sb, run, additions) if err := removeErr; err != nil { return claimSnapshot{}, false, err } if err := m.checkVolumeRunLocked(ctx, sb, run); err != nil { return claimSnapshot{}, false, err } - m.releaseVolumes(additions) - run.holds = volumeDifference(run.holds, additions) return claimSnapshot{}, false, cleanErr } replaced := func(v types.Volume) bool { @@ -402,17 +401,28 @@ func (m *Manager) commitVolumeRun(ctx context.Context, sb *types.Sandbox, run *v m.mu.Unlock() removeErr := m.managedLiveRemove(sb.ID, dropped) m.mu.Lock() + m.releaseUnheldLocked(sb, run, dropped) if err := removeErr; err != nil { return err } if !m.ownsRunLocked(sb, run) { return ErrUnknownSandbox } - m.releaseVolumes(dropped) - run.holds = volumeDifference(run.holds, dropped) return nil } +// releaseUnheldLocked frees run holds the managed hold no longer references; the registry retrier persists a failed drop. +func (m *Manager) releaseUnheldLocked(sb *types.Sandbox, run *volumeRun, volumes []types.Volume) { + if !m.ownsRunLocked(sb, run) { + return + } + if held, managed := m.managedVolumesHeld(sb.ID); managed { + volumes = volumeDifference(volumes, held) + } + m.releaseVolumes(volumes) + run.holds = volumeDifference(run.holds, volumes) +} + func (m *Manager) ownsRunLocked(sb *types.Sandbox, run *volumeRun) bool { return m.claimed[sb.ID] == sb && m.volumeRuns[sb] == run } From 6968b57d3cca9df95214c08fafa59619e1a5a814 Mon Sep 17 00:00:00 2001 From: minatoaquaMK2 Date: Wed, 21 Oct 2026 00:43:51 +0000 Subject: [PATCH 54/75] test(pool): cover a live volume change while growth owns the claim A live catalog change on a claim whose growth is unresolved returns ErrFailed, telling the caller its VMM is down and to release it. Release and Hibernate report the same state as 409 volume_grow_busy, and release is exactly what the growth fence refuses. Generated with [Devin](https://devin.ai) Co-Authored-By: Devin <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- sandboxd/pool/managed_grow_test.go | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/sandboxd/pool/managed_grow_test.go b/sandboxd/pool/managed_grow_test.go index fb16fbf6..cf6855c7 100644 --- a/sandboxd/pool/managed_grow_test.go +++ b/sandboxd/pool/managed_grow_test.go @@ -270,6 +270,17 @@ func TestManagedGrowthACKGuestLagAndLostResponseRecoverWithoutDispatch(t *testin } } +func TestManagedGrowthOwnerReportsGrowBusyToLiveVolumeChanges(t *testing.T) { + m, e, sb, _, v := newGrowthTest(t) + e.unknown = true + _, err := m.GrowVolume(t.Context(), "data", v.ID, growRequest(), "t1") + requireVolumeStatus(t, err, 409) + _, err = m.PatchVolumes(t.Context(), sb.ID, "t1", types.VolumePatch{}) + if got := requireVolumeStatus(t, err, 409); got.Code != "volume_grow_busy" { + t.Fatalf("live change during growth = %+v, want volume_grow_busy", got) + } +} + func TestManagedGrowthUnknownFencesLifecycleUntilExplicitExactStop(t *testing.T) { m, e, sb, _, v := newGrowthTest(t) e.unknown = true From ce3430fefb7aa5c3da0e241930939d0415f94e17 Mon Sep 17 00:00:00 2001 From: minatoaquaMK2 Date: Wed, 21 Oct 2026 00:45:41 +0000 Subject: [PATCH 55/75] fix(pool): report growth ownership to live volume changes as grow busy beginVolumeRun folded an unresolved growth into the not-claimed hold check and returned ErrFailed, whose message tells the caller to release a VMM that is running and that the growth fence will not let go. It now returns volume_grow_busy like Release and Hibernate. checkVolumeRunLocked keeps ErrFailed: growth admission refuses a claim with an active run, so growth ownership mid-run only comes from a registry fault. Generated with [Devin](https://devin.ai) Co-Authored-By: Devin <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- sandboxd/pool/volume_mutation.go | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/sandboxd/pool/volume_mutation.go b/sandboxd/pool/volume_mutation.go index f4ba0f62..6c8f6993 100644 --- a/sandboxd/pool/volume_mutation.go +++ b/sandboxd/pool/volume_mutation.go @@ -190,7 +190,11 @@ func (m *Manager) beginVolumeRun(ctx context.Context, sb *types.Sandbox) (contex m.mu.Unlock() return ctx, nil, managedStorageError() } - if h.State != holdClaimed || m.growthOwnsHold(sb.ID) { + if m.growthOwnsHold(sb.ID) { + m.mu.Unlock() + return ctx, nil, growBusy() + } + if h.State != holdClaimed { m.mu.Unlock() return ctx, nil, ErrFailed } From 32899533143bc42dc03f60a61e36ea21263ce9df Mon Sep 17 00:00:00 2001 From: minatoaquaMK2 Date: Wed, 21 Oct 2026 00:45:59 +0000 Subject: [PATCH 56/75] test(pool): cover claiming a managed volume while its owner grows it A second claim of a growing managed volume gets 503 volume_storage_unavailable, an outage to retry, instead of the 409 volume_busy with holders it gets while the owner is running or asleep. lookupManaged rejects every non-ready state before admission can name the holder. Generated with [Devin](https://devin.ai) Co-Authored-By: Devin <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- sandboxd/pool/managed_grow_test.go | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/sandboxd/pool/managed_grow_test.go b/sandboxd/pool/managed_grow_test.go index cf6855c7..90a50a60 100644 --- a/sandboxd/pool/managed_grow_test.go +++ b/sandboxd/pool/managed_grow_test.go @@ -281,6 +281,18 @@ func TestManagedGrowthOwnerReportsGrowBusyToLiveVolumeChanges(t *testing.T) { } } +func TestManagedGrowingVolumeReportsItsHolderToANewClaim(t *testing.T) { + m, e, sb, _, v := newGrowthTest(t) + e.unknown = true + _, err := m.GrowVolume(t.Context(), "data", v.ID, growRequest(), "t1") + requireVolumeStatus(t, err, 409) + _, err = m.ClaimProvision(t.Context(), testKey, managedOptions("data")) + got := requireVolumeStatus(t, err, 409) + if got.Code != volumeBusyCode || got.Volume != "data" || len(got.Holders) != 1 || got.Holders[0].SandboxID != sb.ID { + t.Fatalf("second writer during growth = %+v, want %s naming the owner", got, volumeBusyCode) + } +} + func TestManagedGrowthUnknownFencesLifecycleUntilExplicitExactStop(t *testing.T) { m, e, sb, _, v := newGrowthTest(t) e.unknown = true From 1351c0b786d54f300cbcf4d73b2b794b6d8407ca Mon Sep 17 00:00:00 2001 From: minatoaquaMK2 Date: Wed, 21 Oct 2026 00:46:31 +0000 Subject: [PATCH 57/75] fix(pool): report a growing managed volume as busy to new claims lookupManaged rejected every non-ready state as a storage outage, so a claim of a volume its owner was growing got 503 with Retry-After. The volume is healthy and held, so answer 409 volume_busy with its holders, the same refusal admission gives while the owner runs or sleeps. Generated with [Devin](https://devin.ai) Co-Authored-By: Devin <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- sandboxd/pool/managed_volume.go | 3 +++ 1 file changed, 3 insertions(+) diff --git a/sandboxd/pool/managed_volume.go b/sandboxd/pool/managed_volume.go index 269b9ea8..057c7447 100644 --- a/sandboxd/pool/managed_volume.go +++ b/sandboxd/pool/managed_volume.go @@ -371,6 +371,9 @@ func (m *Manager) lookupManaged(name string) (catalogVolume, bool, error) { return catalogVolume{}, false, nil } entry := catalogVolume{disk: engine.VolumeSpec{Name: name, Path: r.path(v.ID), DirectIO: types.DirectIOOff}, tenants: []string{v.Tenant}, writable: true, managedID: v.ID} + if v.State == volumeGrowing { + return entry, true, &VolumeError{Status: 409, Code: volumeBusyCode, Message: volumeBusyMessage, Volume: name, Holders: r.holders(v.ID)} + } if v.State != volumeReady { return entry, true, managedStorageError() } From 2c8b4b0b2441f8b380fe43ec078ea1f3f9007849 Mon Sep 17 00:00:00 2001 From: minatoaquaMK2 Date: Wed, 21 Oct 2026 00:47:22 +0000 Subject: [PATCH 58/75] test(server): cover Retry-After on managed volume storage fallbacks Create, delete, and grow answer an unmapped manager error with 503 volume_storage_unavailable and a "retry" message but no Retry-After, while managedStorageError with the same code sends one, so clients that key retries on the header treat the same outage two ways. Generated with [Devin](https://devin.ai) Co-Authored-By: Devin <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- sandboxd/server/volume_grow_test.go | 46 +++++++++++++++++++++++++++-- 1 file changed, 44 insertions(+), 2 deletions(-) diff --git a/sandboxd/server/volume_grow_test.go b/sandboxd/server/volume_grow_test.go index ebbd5145..c9b6a6e2 100644 --- a/sandboxd/server/volume_grow_test.go +++ b/sandboxd/server/volume_grow_test.go @@ -50,20 +50,62 @@ func TestManagedVolumeRecoveryRetryHints(t *testing.T) { } } +func TestManagedVolumeStorageFallbackKeepsRetryAfter(t *testing.T) { + outage := errors.New("write /private/claims.json: no space left on device") + for _, tc := range []struct { + name, method, path, match, body string + mgr Manager + }{ + {"create", http.MethodPut, "/v1/volumes/data", "", `{"size_gib":1}`, &volumeManagerFake{fakeManager: &fakeManager{}, err: outage}}, + {"delete", http.MethodDelete, "/v1/volumes/data", `"vol_123"`, "", &volumeManagerFake{fakeManager: &fakeManager{}, err: outage}}, + {"grow", http.MethodPost, "/v1/volumes/data/grow", `"vol_123"`, growBody, &volumeGrowFake{volumeManagerFake: &volumeManagerFake{fakeManager: &fakeManager{}}, grow: func(context.Context, string, string, types.VolumeGrow, string) (types.ManagedVolumeInfo, error) { + return types.ManagedVolumeInfo{}, outage + }}}, + {"grow unsupported lookup", http.MethodPost, "/v1/volumes/data/grow", `"vol_123"`, growBody, &volumeManagerFake{fakeManager: &fakeManager{}, get: func(string, string) (types.ManagedVolumeInfo, error) { + return types.ManagedVolumeInfo{}, outage + }}}, + } { + t.Run(tc.name, func(t *testing.T) { + ts := newTestServer(t, "root-token", tc.mgr, nil) + req, _ := http.NewRequest(tc.method, ts.URL+tc.path, strings.NewReader(tc.body)) + req.Header.Set("Authorization", "Bearer root-token") + if tc.match != "" { + req.Header.Set("If-Match", tc.match) + } + resp, err := ts.Client().Do(req) + if err != nil { + t.Fatal(err) + } + defer resp.Body.Close() + var body pool.VolumeError + if err := json.UnmarshalRead(resp.Body, &body); err != nil { + t.Fatal(err) + } + if resp.StatusCode != http.StatusServiceUnavailable || body.Code != "volume_storage_unavailable" || strings.Contains(body.Message, "/private") { + t.Fatalf("status=%d body=%+v", resp.StatusCode, body) + } + if resp.Header.Get("Retry-After") == "" { + t.Fatalf("storage outage %q without Retry-After", body.Message) + } + }) + } +} + type volumeManagerFake struct { *fakeManager get func(string, string) (types.ManagedVolumeInfo, error) + err error } func (f *volumeManagerFake) CreateVolume(context.Context, string, types.VolumeCreate, string) (types.ManagedVolumeInfo, bool, error) { - return types.ManagedVolumeInfo{}, false, nil + return types.ManagedVolumeInfo{}, false, f.err } func (f *volumeManagerFake) GetVolume(name, tenant string) (types.ManagedVolumeInfo, error) { return f.get(name, tenant) } -func (f *volumeManagerFake) DeleteVolume(context.Context, string, string, string) error { return nil } +func (f *volumeManagerFake) DeleteVolume(context.Context, string, string, string) error { return f.err } func (f *volumeManagerFake) ManagedVolumeStatus() error { return nil } From 422a60c34bd3ffc5c5c4717499076cf690ec465d Mon Sep 17 00:00:00 2001 From: minatoaquaMK2 Date: Wed, 21 Oct 2026 00:47:48 +0000 Subject: [PATCH 59/75] fix(server): send Retry-After on managed volume storage fallbacks The create, delete, and grow fallbacks for an unmapped manager error wrote 503 volume_storage_unavailable with "retry" text but no Retry-After, unlike managedStorageError with the same code. One helper now writes that fallback with the header. Generated with [Devin](https://devin.ai) Co-Authored-By: Devin <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- sandboxd/server/managed_volumes.go | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/sandboxd/server/managed_volumes.go b/sandboxd/server/managed_volumes.go index a9c39f3f..dc5c9bbc 100644 --- a/sandboxd/server/managed_volumes.go +++ b/sandboxd/server/managed_volumes.go @@ -31,6 +31,11 @@ func volumeHTTPError(w http.ResponseWriter, status int, code, message string) { writeJSON(w, status, &pool.VolumeError{Status: status, Code: code, Message: message}) } +func volumeStorageUnavailable(w http.ResponseWriter) { + w.Header().Set("Retry-After", retryAfterSeconds) + volumeHTTPError(w, 503, "volume_storage_unavailable", "volume storage unavailable; retry") +} + func (s *Server) requireVolumeListToken(next http.HandlerFunc) http.HandlerFunc { if mgr, ok := s.mgr.(managedVolumeManager); ok && mgr.ManagedVolumesEnabled() { return s.requireVolumeManager(next) @@ -86,7 +91,7 @@ func (s *Server) handleCreateVolume(w http.ResponseWriter, r *http.Request) { v, created, err := mgr.CreateVolume(r.Context(), r.PathValue("name"), req, tenantFrom(r.Context())) if err != nil { if !writePoolErr(w, err) { - volumeHTTPError(w, 503, "volume_storage_unavailable", "volume storage unavailable; retry") + volumeStorageUnavailable(w) } return } @@ -128,7 +133,7 @@ func (s *Server) handleDeleteVolume(w http.ResponseWriter, r *http.Request) { } if err := mgr.DeleteVolume(r.Context(), r.PathValue("name"), expected[1:len(expected)-1], tenantFrom(r.Context())); err != nil { if !writePoolErr(w, err) { - volumeHTTPError(w, 503, "volume_storage_unavailable", "volume storage unavailable; retry") + volumeStorageUnavailable(w) } return } @@ -167,7 +172,7 @@ func (s *Server) handleGrowVolume(w http.ResponseWriter, r *http.Request) { if !ok { if _, getErr := mgr.GetVolume(r.PathValue("name"), tenantFrom(r.Context())); getErr != nil { if !writePoolErr(w, getErr) { - volumeHTTPError(w, 503, "volume_storage_unavailable", "volume storage unavailable; retry") + volumeStorageUnavailable(w) } return } @@ -179,7 +184,7 @@ func (s *Server) handleGrowVolume(w http.ResponseWriter, r *http.Request) { }, tenantFrom(r.Context())) if err != nil { if !writePoolErr(w, err) { - volumeHTTPError(w, 503, "volume_storage_unavailable", "volume storage unavailable; retry") + volumeStorageUnavailable(w) } return } From 3e70f8913d6b40c094dc071125455c488c71247e Mon Sep 17 00:00:00 2001 From: minatoaquaMK2 Date: Wed, 21 Oct 2026 00:49:21 +0000 Subject: [PATCH 60/75] test(e2e): cover recovery_required volumes refusing without a retry hint A managed volume whose backing was replaced settles as recovery_required, which only an operator clears. PUT and claim still answer 503 volume_storage_unavailable with Retry-After, inviting a retry loop that cannot succeed. Generated with [Devin](https://devin.ai) Co-Authored-By: Devin <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- e2e/managed_volume_http_test.go | 57 ++++++++++++++++++++++++++++++--- 1 file changed, 53 insertions(+), 4 deletions(-) diff --git a/e2e/managed_volume_http_test.go b/e2e/managed_volume_http_test.go index 68857812..2de41887 100644 --- a/e2e/managed_volume_http_test.go +++ b/e2e/managed_volume_http_test.go @@ -5,6 +5,7 @@ import ( "fmt" "io" "net/http" + "os" "path/filepath" "strings" "sync" @@ -212,10 +213,58 @@ func TestManagedVolumeHTTPRequiresExplicitCredential(t *testing.T) { } } +func TestManagedVolumeHTTPRecoveryRequiredOmitsRetryAfter(t *testing.T) { + dir := filepath.Join(t.TempDir(), "managed") + var cfg config.Config + if err := json.Unmarshal(fmt.Appendf(nil, `{"managed_volume_dir":%q}`, dir), &cfg); err != nil { + t.Fatal(err) + } + st := startConfigStack(t, "root-token", &cfg) + if err := st.mgr.SetTenants(t.Context(), []tenants.TenantSpec{{Name: "t1", Token: "t1-token"}}); err != nil { + t.Fatal(err) + } + created := requestManagedVolume(t, st, http.MethodPut, "/v1/volumes/work", "t1-token", `{"size_gib":1}`, "") + if created.status != http.StatusCreated { + t.Fatalf("create status=%d body=%s", created.status, created.body) + } + backing := filepath.Join(dir, "disks", decodeManagedVolume(t, created.body).ID+".raw") + if err := os.Remove(backing); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(backing, nil, 0o600); err != nil { + t.Fatal(err) + } + if err := os.Truncate(backing, 1<<30); err != nil { + t.Fatal(err) + } + claim := `{"template":"rt:24.04","net":"none","size":"small","volumes":[{"name":"work","mode":"rw"}]}` + for _, step := range []struct{ method, route, body string }{ + {http.MethodPut, "/v1/volumes/work", `{"size_gib":1}`}, + {http.MethodPut, "/v1/volumes/work", `{"size_gib":1}`}, + {http.MethodPost, "/v1/claim", claim}, + } { + reply := requestManagedVolume(t, st, step.method, step.route, "t1-token", step.body, "") + var failure struct { + Code string `json:"code"` + } + if err := json.Unmarshal(reply.body, &failure); err != nil { + t.Fatal(err) + } + if reply.status != http.StatusServiceUnavailable || failure.Code != "volume_recovery_required" || reply.retryAfter != "" { + t.Fatalf("%s %s after backing replacement: status=%d Retry-After=%q body=%s, want 503 volume_recovery_required without a retry hint", step.method, step.route, reply.status, reply.retryAfter, reply.body) + } + } + got := requestManagedVolume(t, st, http.MethodGet, "/v1/volumes/work", "t1-token", "", "") + if state := decodeManagedVolume(t, got.body).State; state != "recovery_required" { + t.Fatalf("GET state %q, want recovery_required", state) + } +} + type managedHTTPReply struct { - status int - body []byte - etag string + status int + body []byte + etag string + retryAfter string } type managedHTTPVolume struct { @@ -269,7 +318,7 @@ func requestManagedVolume(t *testing.T, st *stack, method, route, token, body, e t.Fatal(err) } assertManagedReplyPrivate(t, data) - return managedHTTPReply{status: resp.StatusCode, body: data, etag: resp.Header.Get("ETag")} + return managedHTTPReply{status: resp.StatusCode, body: data, etag: resp.Header.Get("ETag"), retryAfter: resp.Header.Get("Retry-After")} } func decodeManagedVolume(t *testing.T, body []byte) managedHTTPVolume { From fd912ba73daecb40c516c7e0ccee31e5df6e02f3 Mon Sep 17 00:00:00 2001 From: minatoaquaMK2 Date: Wed, 21 Oct 2026 00:51:16 +0000 Subject: [PATCH 61/75] fix(pool): give recovery_required volumes a permanent error code markRecovery and the recovery_required state answered PUT and claim with volume_storage_unavailable, so the server added Retry-After to a condition only an operator clears. They now return 503 volume_recovery_required, which joins the codes that omit Retry-After, and the API docs list it. Generated with [Devin](https://devin.ai) Co-Authored-By: Devin <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- docs/managed-volumes.md | 4 +++- docs/sandboxd-api.md | 13 +++++++++---- sandboxd/pool/managed_volume.go | 17 +++++++++++++---- sandboxd/server/server_http.go | 2 +- 4 files changed, 26 insertions(+), 10 deletions(-) diff --git a/docs/managed-volumes.md b/docs/managed-volumes.md index e5f82cbe..2402fbcd 100644 --- a/docs/managed-volumes.md +++ b/docs/managed-volumes.md @@ -39,7 +39,9 @@ concurrent pathname swap. Keep the registry and backing files together. Do not resize, rename, replace or hardlink a backing file by hand. Each managed resource records its immutable ID, owner, node, capacity and backing device/inode/size. An identity mismatch -requires operator recovery and fails closed. +requires operator recovery and fails closed. The volume then reports state +`recovery_required`, and PUT and claims return `503` with code +`volume_recovery_required` and no `Retry-After`. The existing static catalog remains operator-owned. Its files and every path component must be controlled by trusted operator or daemon principals. Static diff --git a/docs/sandboxd-api.md b/docs/sandboxd-api.md index 03ac0e60..64b5f5cc 100644 --- a/docs/sandboxd-api.md +++ b/docs/sandboxd-api.md @@ -300,8 +300,11 @@ for an invalid body, name, size, or owner; `401` for a missing or invalid token; volume; `409` for a catalog name, different size or owner, or pending deletion; and `503` for unavailable storage or tenant lookup. A create interrupted after allocation may return `503` with code `volume_create_incomplete` and require -operator recovery. `volume_create_incomplete` and `volume_backing_mismatch` -omit `Retry-After`. Transient storage failures include a retry hint. +operator recovery. A volume whose backing identity changed settles in state +`recovery_required`, and PUT and claims of it return `503` with code +`volume_recovery_required`. `volume_create_incomplete`, `volume_backing_mismatch`, +and `volume_recovery_required` omit `Retry-After`. Transient storage failures +include a retry hint. ## GET /v1/volumes/{name} @@ -334,8 +337,10 @@ invalid token; `404` for an unknown or foreign name or tombstone; `409` for a held volume or pending growth; `412` for a malformed or mismatched `If-Match`; `428` for a missing `If-Match`; and `503` for unavailable storage or backing identity that requires operator recovery. A busy response has code -`volume_busy` and includes the volume name and public holder list. A backing -mismatch omits `Retry-After`; transient storage failures include a retry hint. +`volume_busy` and includes the volume name and public holder list. A missing +backing returns `volume_recovery_required`. A backing mismatch and +`volume_recovery_required` omit `Retry-After`; transient storage failures +include a retry hint. ## POST /v1/volumes/{name}/grow diff --git a/sandboxd/pool/managed_volume.go b/sandboxd/pool/managed_volume.go index 057c7447..efd22b37 100644 --- a/sandboxd/pool/managed_volume.go +++ b/sandboxd/pool/managed_volume.go @@ -41,6 +41,10 @@ func managedStorageError() *VolumeError { return volumeError(503, "volume_storage_unavailable", "volume storage requires recovery; retry after recovery") } +func managedRecoveryError() *VolumeError { + return volumeError(503, "volume_recovery_required", "volume requires operator recovery; inspect the volume") +} + func managedNotFound() *VolumeError { return volumeError(404, "volume_not_found", "unknown volume") } func (m *Manager) CreateVolume(ctx context.Context, name string, req types.VolumeCreate, tenant string) (types.ManagedVolumeInfo, bool, error) { @@ -104,6 +108,8 @@ func (m *Manager) CreateVolume(ctx context.Context, name string, req types.Volum } case volumeDeleting: return types.ManagedVolumeInfo{}, false, volumeError(409, "volume_deleting", "volume deletion is pending") + case volumeRecovery: + return types.ManagedVolumeInfo{}, false, managedRecoveryError() default: return types.ManagedVolumeInfo{}, false, managedStorageError() } @@ -156,7 +162,7 @@ func (r *volumeRegistry) markRecovery(v managedVolume) error { if err := r.commitAndRelock(); err != nil { return err } - return managedStorageError() + return managedRecoveryError() } func (m *Manager) GetVolume(name, tenant string) (types.ManagedVolumeInfo, error) { @@ -371,10 +377,13 @@ func (m *Manager) lookupManaged(name string) (catalogVolume, bool, error) { return catalogVolume{}, false, nil } entry := catalogVolume{disk: engine.VolumeSpec{Name: name, Path: r.path(v.ID), DirectIO: types.DirectIOOff}, tenants: []string{v.Tenant}, writable: true, managedID: v.ID} - if v.State == volumeGrowing { + switch v.State { + case volumeReady: + case volumeGrowing: return entry, true, &VolumeError{Status: 409, Code: volumeBusyCode, Message: volumeBusyMessage, Volume: name, Holders: r.holders(v.ID)} - } - if v.State != volumeReady { + case volumeRecovery: + return entry, true, managedRecoveryError() + default: return entry, true, managedStorageError() } r.mu.Unlock() diff --git a/sandboxd/server/server_http.go b/sandboxd/server/server_http.go index cbaba9cb..77734fd9 100644 --- a/sandboxd/server/server_http.go +++ b/sandboxd/server/server_http.go @@ -190,7 +190,7 @@ func mapPoolErr(w http.ResponseWriter, err error) (code int, msg string, ok bool return code, msg, ok } if volumeErr, managed := errors.AsType[*pool.VolumeError](err); managed && - (volumeErr.Code == "volume_create_incomplete" || volumeErr.Code == "volume_backing_mismatch" || volumeErr.Code == "volume_grow_unsupported") { + (volumeErr.Code == "volume_create_incomplete" || volumeErr.Code == "volume_backing_mismatch" || volumeErr.Code == "volume_grow_unsupported" || volumeErr.Code == "volume_recovery_required") { return code, msg, ok } w.Header().Set("Retry-After", retryAfterSeconds) From e27f3e6d565faf8d6b92f97a7d97808c6bb4159b Mon Sep 17 00:00:00 2001 From: minatoaquaMK2 Date: Wed, 21 Oct 2026 00:51:37 +0000 Subject: [PATCH 62/75] test(e2e): cover claiming a replaced managed backing before any PUT GET reports a ready volume whose backing identity changed as recovery_required without persisting it, but a claim that arrives before PUT or DELETE marks the volume still gets volume_storage_unavailable with Retry-After for the same operator-only condition. Generated with [Devin](https://devin.ai) Co-Authored-By: Devin <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- e2e/managed_volume_http_test.go | 1 + 1 file changed, 1 insertion(+) diff --git a/e2e/managed_volume_http_test.go b/e2e/managed_volume_http_test.go index 2de41887..95a804af 100644 --- a/e2e/managed_volume_http_test.go +++ b/e2e/managed_volume_http_test.go @@ -239,6 +239,7 @@ func TestManagedVolumeHTTPRecoveryRequiredOmitsRetryAfter(t *testing.T) { } claim := `{"template":"rt:24.04","net":"none","size":"small","volumes":[{"name":"work","mode":"rw"}]}` for _, step := range []struct{ method, route, body string }{ + {http.MethodPost, "/v1/claim", claim}, {http.MethodPut, "/v1/volumes/work", `{"size_gib":1}`}, {http.MethodPut, "/v1/volumes/work", `{"size_gib":1}`}, {http.MethodPost, "/v1/claim", claim}, From 02dc2d0743f52a797552795a80a8e2f2d8175ab2 Mon Sep 17 00:00:00 2001 From: minatoaquaMK2 Date: Wed, 21 Oct 2026 00:52:17 +0000 Subject: [PATCH 63/75] fix(pool): report a replaced managed backing as recovery_required on claim lookupManaged answered a backing mismatch or missing backing with volume_storage_unavailable, while GET already reports the same observation as recovery_required. Claims now return volume_recovery_required for it too, without persisting the state, like GET. Generated with [Devin](https://devin.ai) Co-Authored-By: Devin <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- sandboxd/pool/managed_volume.go | 3 +++ 1 file changed, 3 insertions(+) diff --git a/sandboxd/pool/managed_volume.go b/sandboxd/pool/managed_volume.go index efd22b37..070b3035 100644 --- a/sandboxd/pool/managed_volume.go +++ b/sandboxd/pool/managed_volume.go @@ -389,6 +389,9 @@ func (m *Manager) lookupManaged(name string) (catalogVolume, bool, error) { r.mu.Unlock() err := r.verify(v) r.mu.Lock() + if errors.Is(err, errBackingMismatch) || errors.Is(err, os.ErrNotExist) { + return entry, true, managedRecoveryError() + } if err != nil { return entry, true, managedStorageError() } From d45566abd83355fcb3eb75939fa3869c29f3a204 Mon Sep 17 00:00:00 2001 From: minatoaquaMK2 Date: Wed, 21 Oct 2026 00:53:05 +0000 Subject: [PATCH 64/75] test(sdk/python): cover claims rejecting a bare string volume list Client.new and Template.new iterate volumes="work" into four one-letter volumes and send them, while set_volumes and patch_volumes already reject a bare string where a list is expected. Generated with [Devin](https://devin.ai) Co-Authored-By: Devin <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- sdk/python/tests/test_client.py | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/sdk/python/tests/test_client.py b/sdk/python/tests/test_client.py index 90b799dc..cfeaf463 100644 --- a/sdk/python/tests/test_client.py +++ b/sdk/python/tests/test_client.py @@ -84,6 +84,15 @@ def test_claim_rejects_invalid_volumes(node, volumes, match): Client(node).new("rt:24.04", volumes=volumes) +def test_claim_rejects_a_bare_string_volume_list(node): + seen = recording_claim({"id": "sb_1", "token": "tok"}) + with pytest.raises(TypeError): + Client(node).new("rt:24.04", volumes="work") + with pytest.raises(TypeError): + Template(Client(node), node, "task:v1", "none", "small").new(volumes="work") + assert seen == [] + + def test_claim_sends_volume_mode_rw(node): seen = recording_claim( {"id": "sb_1", "token": "tok", "volumes": [{"name": "scratch", "mount": "/data", "mode": "rw"}]} From fdc77859daad4250fe56d441467f6e9fe2712645 Mon Sep 17 00:00:00 2001 From: minatoaquaMK2 Date: Wed, 21 Oct 2026 00:54:12 +0000 Subject: [PATCH 65/75] fix(sdk/python): reject a bare string volume list in claims _claim_body iterated volumes="work" into four one-letter volumes. It now raises TypeError before building the claim, matching how set_volumes and patch_volumes reject a bare string. Generated with [Devin](https://devin.ai) Co-Authored-By: Devin <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- sdk/python/cocoonsandbox/template.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/sdk/python/cocoonsandbox/template.py b/sdk/python/cocoonsandbox/template.py index 5e468aa0..386ca584 100644 --- a/sdk/python/cocoonsandbox/template.py +++ b/sdk/python/cocoonsandbox/template.py @@ -59,6 +59,8 @@ def _claim_body( mount: bool = True, claim_ref: str = "", ) -> dict[str, Any]: + if isinstance(volumes, str): + raise TypeError("volumes takes a list of names or mappings") claim: dict[str, Any] = {"template": template} if net: claim["net"] = net From 56b316f64ed9ab7b5a36627696680f609bebcb78 Mon Sep 17 00:00:00 2001 From: minatoaquaMK2 Date: Wed, 21 Oct 2026 00:55:56 +0000 Subject: [PATCH 66/75] test(pool): stop shadowing err in managed teardown tests govet's shadow check flags the PutTenant calls in the expiry and teardown tests added earlier on this branch. Generated with [Devin](https://devin.ai) Co-Authored-By: Devin <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- sandboxd/pool/managed_integration_test.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/sandboxd/pool/managed_integration_test.go b/sandboxd/pool/managed_integration_test.go index 3ccd143c..bc7f1036 100644 --- a/sandboxd/pool/managed_integration_test.go +++ b/sandboxd/pool/managed_integration_test.go @@ -270,7 +270,7 @@ func TestManagedExpiryCancelsExistingVolumeRun(t *testing.T) { } t.Cleanup(func() { m.Close() }) m.managed.process = e.identity - if err := m.PutTenant(t.Context(), tenants.TenantSpec{Name: "t1", Token: "t1-token"}); err != nil { + if err = m.PutTenant(t.Context(), tenants.TenantSpec{Name: "t1", Token: "t1-token"}); err != nil { t.Fatal(err) } createManagedTest(t, m, "data") @@ -354,7 +354,7 @@ func TestManagedTeardownDuringLiveRemovalReleasesCatalogCharge(t *testing.T) { } t.Cleanup(func() { _ = m.Close() }) m.managed.process = process - if err := m.PutTenant(t.Context(), tenants.TenantSpec{Name: "t1", Token: "t1-token"}); err != nil { + if err = m.PutTenant(t.Context(), tenants.TenantSpec{Name: "t1", Token: "t1-token"}); err != nil { t.Fatal(err) } createManagedTest(t, m, "data") From 4c6fbb1d57f24b3a230605fd21b63f782353a236 Mon Sep 17 00:00:00 2001 From: minatoaquaMK2 Date: Wed, 21 Oct 2026 01:14:50 +0000 Subject: [PATCH 67/75] test(pool): cover claims over a missing managed directory or sleep-owned backing A claim treats any ENOENT from verify as a lost backing, so a moved managed root or disks directory answers the permanent volume_recovery_required. A claim of a sleeping volume's lost backing does the same, though PUT keeps it transient because wake still owns the backing. Both should answer volume_storage_unavailable with a retry hint. Generated with [Devin](https://devin.ai) Co-Authored-By: Devin <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- e2e/managed_volume_http_test.go | 53 +++++++++++++++++++++++++++++ sandboxd/pool/managed_sleep_test.go | 10 +++--- 2 files changed, 59 insertions(+), 4 deletions(-) diff --git a/e2e/managed_volume_http_test.go b/e2e/managed_volume_http_test.go index 95a804af..673e8ec4 100644 --- a/e2e/managed_volume_http_test.go +++ b/e2e/managed_volume_http_test.go @@ -261,6 +261,59 @@ func TestManagedVolumeHTTPRecoveryRequiredOmitsRetryAfter(t *testing.T) { } } +func TestManagedVolumeHTTPMissingDirectoryIsTransient(t *testing.T) { + dir := filepath.Join(t.TempDir(), "managed") + var cfg config.Config + if err := json.Unmarshal(fmt.Appendf(nil, `{"managed_volume_dir":%q}`, dir), &cfg); err != nil { + t.Fatal(err) + } + st := startConfigStack(t, "root-token", &cfg) + if err := st.mgr.SetTenants(t.Context(), []tenants.TenantSpec{{Name: "t1", Token: "t1-token"}}); err != nil { + t.Fatal(err) + } + created := requestManagedVolume(t, st, http.MethodPut, "/v1/volumes/work", "t1-token", `{"size_gib":1}`, "") + if created.status != http.StatusCreated { + t.Fatalf("create status=%d body=%s", created.status, created.body) + } + claim := `{"template":"rt:24.04","net":"none","size":"small","volumes":[{"name":"work","mode":"ro"}]}` + for _, moved := range []string{filepath.Join(dir, "disks"), dir} { + if err := os.Rename(moved, moved+".aside"); err != nil { + t.Fatal(err) + } + for _, step := range []struct{ method, route, body string }{ + {http.MethodPost, "/v1/claim", claim}, + {http.MethodPut, "/v1/volumes/work", `{"size_gib":1}`}, + {http.MethodPost, "/v1/claim", claim}, + } { + reply := requestManagedVolume(t, st, step.method, step.route, "t1-token", step.body, "") + var failure struct { + Code string `json:"code"` + } + if err := json.Unmarshal(reply.body, &failure); err != nil { + t.Fatal(err) + } + if reply.status != http.StatusServiceUnavailable || failure.Code != "volume_storage_unavailable" || reply.retryAfter == "" { + t.Fatalf("%s %s with %s moved aside: status=%d Retry-After=%q body=%s, want 503 volume_storage_unavailable with a retry hint", step.method, step.route, filepath.Base(moved), reply.status, reply.retryAfter, reply.body) + } + } + if err := os.Rename(moved+".aside", moved); err != nil { + t.Fatal(err) + } + reply := requestManagedVolume(t, st, http.MethodPost, "/v1/claim", "t1-token", claim, "") + var claimed rawClaimResponse + if reply.status != http.StatusOK { + t.Fatalf("claim after restoring %s: status=%d body=%s, want 200", filepath.Base(moved), reply.status, reply.body) + } + if err := json.Unmarshal(reply.body, &claimed); err != nil { + t.Fatal(err) + } + released := requestManagedVolume(t, st, http.MethodPost, "/v1/sandboxes/"+claimed.ID+"/release", claimed.Token, "", "") + if released.status != http.StatusNoContent { + t.Fatalf("release after restoring %s: status=%d body=%s", filepath.Base(moved), released.status, released.body) + } + } +} + type managedHTTPReply struct { status int body []byte diff --git a/sandboxd/pool/managed_sleep_test.go b/sandboxd/pool/managed_sleep_test.go index 130faf28..340bdd5e 100644 --- a/sandboxd/pool/managed_sleep_test.go +++ b/sandboxd/pool/managed_sleep_test.go @@ -387,11 +387,13 @@ func TestManagedSleepBackingRefusesBeforeRestore(t *testing.T) { if len(infos) != 1 || infos[0].Available { t.Fatalf("missing backing list: %+v", infos) } - if _, _, err = m.CreateVolume(t.Context(), "data", types.VolumeCreate{SizeGiB: 1}, "t1"); err == nil { - t.Fatal("missing backing create replay succeeded") + _, _, err = m.CreateVolume(t.Context(), "data", types.VolumeCreate{SizeGiB: 1}, "t1") + if v := requireVolumeStatus(t, err, 503); v.Code != "volume_storage_unavailable" { + t.Fatalf("create replay over a sleeping volume's lost backing = %s, want volume_storage_unavailable while wake owns it", v.Code) } - if _, err = m.ClaimProvision(t.Context(), testKey, managedOptions("data")); err == nil { - t.Fatal("missing backing admitted second writer") + _, err = m.ClaimProvision(t.Context(), testKey, managedOptions("data")) + if v := requireVolumeStatus(t, err, 503); v.Code != "volume_storage_unavailable" { + t.Fatalf("claim of a sleeping volume's lost backing = %s, want volume_storage_unavailable while wake owns it", v.Code) } if replace { if err := os.Remove(path); err != nil { From f4479c1ac44eb5dd58c7007ea62d1253a05b0e1f Mon Sep 17 00:00:00 2001 From: minatoaquaMK2 Date: Wed, 21 Oct 2026 01:16:31 +0000 Subject: [PATCH 68/75] test(e2e): check a restored managed directory through PUT, not a claim The e2e fake engine has no process identity, so a managed claim can never bind there. A PUT that answers 200 ready after the directory comes back shows the outage left neither a registry fault nor a recovery_required volume behind. Generated with [Devin](https://devin.ai) Co-Authored-By: Devin <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- e2e/managed_volume_http_test.go | 14 +++----------- 1 file changed, 3 insertions(+), 11 deletions(-) diff --git a/e2e/managed_volume_http_test.go b/e2e/managed_volume_http_test.go index 673e8ec4..6f0d0a10 100644 --- a/e2e/managed_volume_http_test.go +++ b/e2e/managed_volume_http_test.go @@ -299,17 +299,9 @@ func TestManagedVolumeHTTPMissingDirectoryIsTransient(t *testing.T) { if err := os.Rename(moved+".aside", moved); err != nil { t.Fatal(err) } - reply := requestManagedVolume(t, st, http.MethodPost, "/v1/claim", "t1-token", claim, "") - var claimed rawClaimResponse - if reply.status != http.StatusOK { - t.Fatalf("claim after restoring %s: status=%d body=%s, want 200", filepath.Base(moved), reply.status, reply.body) - } - if err := json.Unmarshal(reply.body, &claimed); err != nil { - t.Fatal(err) - } - released := requestManagedVolume(t, st, http.MethodPost, "/v1/sandboxes/"+claimed.ID+"/release", claimed.Token, "", "") - if released.status != http.StatusNoContent { - t.Fatalf("release after restoring %s: status=%d body=%s", filepath.Base(moved), released.status, released.body) + replay := requestManagedVolume(t, st, http.MethodPut, "/v1/volumes/work", "t1-token", `{"size_gib":1}`, "") + if replay.status != http.StatusOK || decodeManagedVolume(t, replay.body).State != "ready" { + t.Fatalf("PUT after restoring %s: status=%d body=%s, want 200 ready", filepath.Base(moved), replay.status, replay.body) } } } From 7d843b4a48f774eef8056d456614381eb359a44d Mon Sep 17 00:00:00 2001 From: minatoaquaMK2 Date: Wed, 21 Oct 2026 01:16:46 +0000 Subject: [PATCH 69/75] fix(pool): keep directory loss and owned backings transient on claim verify returned the raw ENOENT from a moved managed root or disks directory, which every caller read as a lost backing file. It now returns errBackingMismatch only for a missing or mismatched backing file, so a directory outage stays volume_storage_unavailable and PUT no longer faults the registry over it. lookupManaged also released r.mu around verify and then answered recovery_required without rechecking. It now shares markRecovery's check: the record must be unchanged and not owned by a sleep or a fenced growth, or the answer stays transient. Generated with [Devin](https://devin.ai) Co-Authored-By: Devin <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- sandboxd/pool/managed_volume.go | 28 ++++++++++++++++------------ sandboxd/pool/volume_registry.go | 9 +++------ 2 files changed, 19 insertions(+), 18 deletions(-) diff --git a/sandboxd/pool/managed_volume.go b/sandboxd/pool/managed_volume.go index 070b3035..71ef613c 100644 --- a/sandboxd/pool/managed_volume.go +++ b/sandboxd/pool/managed_volume.go @@ -101,7 +101,7 @@ func (m *Manager) CreateVolume(ctx context.Context, name string, req types.Volum err := r.verify(v) r.mu.Lock() if err != nil { - if errors.Is(err, errBackingMismatch) || errors.Is(err, os.ErrNotExist) { + if errors.Is(err, errBackingMismatch) { return types.ManagedVolumeInfo{}, false, r.markRecovery(v) } return types.ManagedVolumeInfo{}, false, managedStorageError() @@ -148,13 +148,16 @@ func (r *volumeRegistry) completeCreate(v managedVolume) error { return r.commitAndRelock() } -func (r *volumeRegistry) markRecovery(v managedVolume) error { +// A sleep or a fenced growth reconciles its own backing, so a loss under either stays transient. +func (r *volumeRegistry) recoverable(v managedVolume) bool { current, ok := r.data.Volumes[v.Name] - if !ok || current.ID != v.ID || current.State != v.State || current.Size != v.Size || - current.Backing != v.Backing || current.CapacityGeneration != v.CapacityGeneration || current.Growth != v.Growth { - return managedStorageError() - } - if r.sleepOwnsVolume(v.ID) || v.Growth.fenced() { + return ok && current.ID == v.ID && current.State == v.State && current.Size == v.Size && + current.Backing == v.Backing && current.CapacityGeneration == v.CapacityGeneration && current.Growth == v.Growth && + !r.sleepOwnsVolume(v.ID) && !v.Growth.fenced() +} + +func (r *volumeRegistry) markRecovery(v managedVolume) error { + if !r.recoverable(v) { return managedStorageError() } v.State = volumeRecovery @@ -190,7 +193,7 @@ func (m *Manager) GetVolume(name, tenant string) (types.ManagedVolumeInfo, error } if v.State == volumeReady { if err := r.verify(v); err != nil { - if !errors.Is(err, errBackingMismatch) && !errors.Is(err, os.ErrNotExist) { + if !errors.Is(err, errBackingMismatch) { return types.ManagedVolumeInfo{}, managedStorageError() } info.State = string(volumeRecovery) @@ -389,13 +392,14 @@ func (m *Manager) lookupManaged(name string) (catalogVolume, bool, error) { r.mu.Unlock() err := r.verify(v) r.mu.Lock() - if errors.Is(err, errBackingMismatch) || errors.Is(err, os.ErrNotExist) { + switch { + case err == nil: + return entry, true, nil + case errors.Is(err, errBackingMismatch) && r.recoverable(v): return entry, true, managedRecoveryError() - } - if err != nil { + default: return entry, true, managedStorageError() } - return entry, true, nil } func (m *Manager) managedHold(id string) (volumeHold, bool) { diff --git a/sandboxd/pool/volume_registry.go b/sandboxd/pool/volume_registry.go index e649a70d..f298c176 100644 --- a/sandboxd/pool/volume_registry.go +++ b/sandboxd/pool/volume_registry.go @@ -119,7 +119,7 @@ type volumeRegistry struct { process func(int) (processIdentity, error) } -var errBackingMismatch = errors.New("managed backing identity changed") +var errBackingMismatch = errors.New("managed backing is missing or changed") var managedIDPattern = regexp.MustCompile(`^vol_[0-9a-f]{32}$`) @@ -265,13 +265,10 @@ func (r *volumeRegistry) verify(v managedVolume) error { return err } actual, err := inspectBacking(r.path(v.ID)) - if err != nil { - return err - } - if v.Backing.Inode == 0 || actual != v.Backing || actual.Size != v.Size { + if errors.Is(err, os.ErrNotExist) || err == nil && (v.Backing.Inode == 0 || actual != v.Backing || actual.Size != v.Size) { return errBackingMismatch } - return nil + return err } func (r *volumeRegistry) validate() error { From 0ae1b84dca083334f9c20abff17ee1152d77e388 Mon Sep 17 00:00:00 2001 From: minatoaquaMK2 Date: Wed, 21 Oct 2026 01:17:46 +0000 Subject: [PATCH 70/75] test(pool): cover a failed resume after a hibernate mount conflict A guest mount conflict aborts hibernate preparation and resumes the guest. When that resume fails too, the caller only sees the conflict, which the server maps to 409 "close guest references and retry", while the sandbox is actually stuck aborting and needs sleep recovery. Generated with [Devin](https://devin.ai) Co-Authored-By: Devin <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- sandboxd/pool/managed_sleep_test.go | 25 +++++++++++++++++++++++++ 1 file changed, 25 insertions(+) diff --git a/sandboxd/pool/managed_sleep_test.go b/sandboxd/pool/managed_sleep_test.go index 340bdd5e..398b7771 100644 --- a/sandboxd/pool/managed_sleep_test.go +++ b/sandboxd/pool/managed_sleep_test.go @@ -326,6 +326,31 @@ func TestManagedSleepAbortKeepsOriginalVM(t *testing.T) { } } +func TestManagedSleepMountConflictReportsFailedResume(t *testing.T) { + for _, resumeFails := range []bool{false, true} { + t.Run(map[bool]string{false: "resumed", true: "resume failed"}[resumeFails], func(t *testing.T) { + m, e, sb, _ := newSleepTest(t) + e.detachVolumeErr = &engine.VolumeMountConflict{Message: `cannot unmount "/volumes/data"; close guest references and retry`} + if resumeFails { + e.hookErr = "resume" + } + err := m.Hibernate(t.Context(), sb.ID, Cred{Token: sb.Token}) + if _, conflict := errors.AsType[*engine.VolumeMountConflict](err); !conflict { + t.Fatalf("hibernate error %v lost the guest mount conflict", err) + } + if !resumeFails { + if v, ok := errors.AsType[*VolumeError](err); ok { + t.Fatalf("resumed abort reported %s, want only the guest mount conflict", v.Code) + } + return + } + if v := requireVolumeStatus(t, err, 409); v.Code != "managed_sleep_recovery_required" { + t.Fatalf("failed resume after a mount conflict = %s, want managed_sleep_recovery_required", v.Code) + } + }) + } +} + func TestManagedSleepWakeFailuresKeepRestoredProcess(t *testing.T) { for _, failure := range []string{"restore reply", "attach", "mount", "resume"} { t.Run(failure, func(t *testing.T) { From 3e77e34b8e6c8ddbeb547831453a8578b43a44ab Mon Sep 17 00:00:00 2001 From: minatoaquaMK2 Date: Wed, 21 Oct 2026 01:18:03 +0000 Subject: [PATCH 71/75] fix(pool): report sleep recovery when a hibernate abort cannot resume prepareManagedSleep joined the preparation error with the raw resume error. With a guest mount conflict first, the server ranks the conflict and answers "close guest references and retry", hiding that the guest never resumed. A failed resume now joins sleepRecoveryError(), as captureRunningManagedSleep already does, so the caller gets 409 managed_sleep_recovery_required. Generated with [Devin](https://devin.ai) Co-Authored-By: Devin <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- sandboxd/pool/managed_sleep.go | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/sandboxd/pool/managed_sleep.go b/sandboxd/pool/managed_sleep.go index f4f0c7a4..03fd64d6 100644 --- a/sandboxd/pool/managed_sleep.go +++ b/sandboxd/pool/managed_sleep.go @@ -201,7 +201,10 @@ func (m *Manager) prepareManagedSleep(ctx context.Context, sb *types.Sandbox, h return errors.Join(err, persistErr) } h, _ = m.managedHold(sb.ID) - return errors.Join(err, m.resumeManagedSleep(ctx, sb, h, false)) + if resumeErr := m.resumeManagedSleep(ctx, sb, h, false); resumeErr != nil { + return errors.Join(err, sleepRecoveryError(), resumeErr) + } + return err } return m.setSleepPhase(h, sleepCapturing) } From 1c920ef9a3e978648e13b75550c37cd6be0a6b4d Mon Sep 17 00:00:00 2001 From: minatoaquaMK2 Date: Wed, 21 Oct 2026 01:20:30 +0000 Subject: [PATCH 72/75] test(pool): keep a bystander catalog reader through managed teardown races The teardown tests asserted an empty admission entry, but releaseVolumes deletes an entry once its counts drop below one, so a double release looked identical to a correct one. A second read-only holder of the catalog now has to survive each race with readers:1, and a writer stays refused until that reader leaves. Test-only; these pass on the previous head. Generated with [Devin](https://devin.ai) Co-Authored-By: Devin <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- sandboxd/pool/managed_integration_test.go | 39 ++++++++++++++++++++--- 1 file changed, 34 insertions(+), 5 deletions(-) diff --git a/sandboxd/pool/managed_integration_test.go b/sandboxd/pool/managed_integration_test.go index bc7f1036..3c77f39a 100644 --- a/sandboxd/pool/managed_integration_test.go +++ b/sandboxd/pool/managed_integration_test.go @@ -196,10 +196,11 @@ func TestManagedSettledRetryDrainsPreviouslyUncertainCatalogRef(t *testing.T) { type managedCancelEngine struct { *managedTestEngine entered chan struct{} + vm string } func (e *managedCancelEngine) DiskAttach(ctx context.Context, name string, spec engine.VolumeSpec) error { - if spec.Name == "catalog" { + if spec.Name == "catalog" && (e.vm == "" || e.vm == name) { close(e.entered) <-ctx.Done() return ctx.Err() @@ -278,6 +279,8 @@ func TestManagedExpiryCancelsExistingVolumeRun(t *testing.T) { if err != nil { t.Fatal(err) } + e.vm = sb.VMName + catalogReader(t, m) done := make(chan error, 1) go func() { _, err := m.PatchVolumes(t.Context(), sb.ID, "t1", types.VolumePatch{Add: []types.Volume{{Name: "catalog"}}}) @@ -297,11 +300,20 @@ func TestManagedExpiryCancelsExistingVolumeRun(t *testing.T) { case <-time.After(time.Second): t.Fatal("expiry did not cancel the live volume run") } - if got := volumeHoldersOf(m, "catalog"); got != (volumeHolders{}) { - t.Fatalf("expired claim kept the canceled run's charge: %+v", got) + if got := volumeHoldersOf(m, "catalog"); got != (volumeHolders{readers: 1}) { + t.Fatalf("expired claim left catalog admission %+v, want only the bystander reader", got) } } +func catalogReader(t *testing.T, m *Manager) *types.Sandbox { + t.Helper() + sb, err := m.ClaimProvision(t.Context(), testKey, ClaimOptions{Tenant: "t1", Volumes: []types.Volume{{Name: "catalog", Mode: types.VolumeModeRO, AttachOnly: true}}}) + if err != nil { + t.Fatalf("bystander catalog reader: %v", err) + } + return sb +} + func TestManagedExpiryDropsVolumeDiagnostics(t *testing.T) { cfg := managedConfig(t) cfg.Volumes = []config.VolumeSpec{{Name: "catalog", Path: writeVolumeImage(t, "catalog.img", "catalog")}} @@ -365,6 +377,7 @@ func TestManagedTeardownDuringLiveRemovalReleasesCatalogCharge(t *testing.T) { if _, err := m.PatchVolumes(t.Context(), sb.ID, "t1", types.VolumePatch{Add: []types.Volume{{Name: "catalog"}}}); err != nil { t.Fatal(err) } + bystander := catalogReader(t, m) entered, resume := make(chan struct{}), make(chan struct{}) var once sync.Once m.managed.mu.Lock() @@ -396,8 +409,14 @@ func TestManagedTeardownDuringLiveRemovalReleasesCatalogCharge(t *testing.T) { <-torn <-patched waitFor(t, func() bool { _, held := m.managedHold(sb.ID); return !held }) - if got := volumeHoldersOf(m, "catalog"); got != (volumeHolders{}) { - t.Fatalf("%s during a live catalog removal kept its charge: %+v", path, got) + if got := volumeHoldersOf(m, "catalog"); got != (volumeHolders{readers: 1}) { + t.Fatalf("%s during a live catalog removal left catalog admission %+v, want only the bystander reader", path, got) + } + if _, err := m.ClaimProvision(t.Context(), testKey, managedOptions("catalog")); !errors.Is(err, ErrVolumeBusy) { + t.Fatalf("catalog writer beside the bystander reader = %v, want busy", err) + } + if err := m.Release(t.Context(), bystander.ID, Cred{Operator: true}); err != nil { + t.Fatal(err) } if _, err := m.ClaimProvision(t.Context(), testKey, managedOptions("catalog")); err != nil { t.Fatalf("catalog writer refused after teardown: %v", err) @@ -422,6 +441,7 @@ func TestManagedLiveRemovalReleasesCatalogChargeWhenRegistryWriteFails(t *testin m := openManagedTest(t, cfg, newManagedEngine()) createManagedTest(t, m, "data") sb := foundationClaim(t, m, "data") + bystander := catalogReader(t, m) patch := types.VolumePatch{Add: []types.Volume{{Name: "catalog"}}} if tc.dirty { if err := markVolumeDirty(catalog); err != nil { @@ -455,6 +475,15 @@ func TestManagedLiveRemovalReleasesCatalogChargeWhenRegistryWriteFails(t *testin if view, _ := m.GetVolumes(sb.ID, "t1"); len(view.States) != 1 { t.Fatalf("claim still lists the catalog: %+v", view) } + if got := volumeHoldersOf(m, "catalog"); got != (volumeHolders{readers: 1}) { + t.Fatalf("failed removal write left catalog admission %+v, want only the bystander reader", got) + } + if _, err := m.ClaimProvision(t.Context(), testKey, managedOptions("catalog")); !errors.Is(err, ErrVolumeBusy) { + t.Fatalf("catalog writer beside the bystander reader = %v, want busy", err) + } + if err := m.Release(t.Context(), bystander.ID, Cred{Operator: true}); err != nil { + t.Fatal(err) + } writer, err := m.ClaimProvision(t.Context(), testKey, managedOptions("catalog")) if err != nil { t.Fatalf("catalog the claim no longer holds is still busy: %v", err) From 6132addb3c423c320c636b82001d2a4b2f5c4b91 Mon Sep 17 00:00:00 2001 From: minatoaquaMK2 Date: Wed, 21 Oct 2026 01:21:50 +0000 Subject: [PATCH 73/75] docs(api): list managed volume refusals for claim, hibernate, and wake The claim section lacked the growing volume's 409 volume_busy and the two managed 503 codes. Hibernate and wake lacked the guest mount 409, the sleep recovery 409, and the 503 below Cocoon v0.7.2, which carries no Retry-After. Generated with [Devin](https://devin.ai) Co-Authored-By: Devin <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- docs/sandboxd-api.md | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/docs/sandboxd-api.md b/docs/sandboxd-api.md index 64b5f5cc..1a19ca7a 100644 --- a/docs/sandboxd-api.md +++ b/docs/sandboxd-api.md @@ -235,7 +235,11 @@ unclean `rw` release (needs recovery — one `rw` claim must replay and cleanly release before `ro` claims resume; see [deploy](deploy.md#writable-dataset-volumes)); 429 node at `max_claims`, the calling tenant at its own `max_claims`, or the node draining; 500 -provisioning failed. +provisioning failed. Managed volumes use the managed volume error body: a +volume that is growing returns 409 `volume_busy` with its public holder list; +a missing or replaced backing returns 503 `volume_recovery_required` without +`Retry-After`; unavailable managed storage returns 503 +`volume_storage_unavailable` with `Retry-After`. ## GET /v1/volumes @@ -529,7 +533,10 @@ token, 409 on the egress lane or unsupported volume claims, 429 while the disk i [`disk_reserve`](deploy.md). An opt-in node hook supports one mounted writable managed volume on the `none` lane. See [managed volume hibernation](managed-volumes.md#hibernate-one-mounted-managed-volume) for hook installation, exclusive ownership, and failure recovery. Other volume -capture operations remain blocked. +capture operations remain blocked. A managed hibernate also returns 409 when a +guest mount blocks the unmount (close guest references and retry), 409 +`managed_sleep_recovery_required` when the guest could not be resumed after a +failed preparation, and 503 without `Retry-After` below Cocoon v0.7.2. ## POST /v1/sandboxes/{id}/wake @@ -544,6 +551,8 @@ while stored, and waking it starts a fresh lease of the length the claim was gra its own disk (memory is lost) and its restart budget resets. 204 on success, 404 unknown id or wrong token, 409 a failed egress-lane or volume sandbox, or one whose VM record is gone, 500 when the cold boot itself fails. +A managed wake also returns 409 when a guest mount conflicts with the remount +and 503 without `Retry-After` below Cocoon v0.7.2. ## POST /v1/sandboxes/{id}/renew From 430f9981276fc035d6b8df22fe8a9279d25cd54b Mon Sep 17 00:00:00 2001 From: minatoaquaMK2 Date: Wed, 21 Oct 2026 01:22:23 +0000 Subject: [PATCH 74/75] test(sdk/python): cover claims rejecting a bare mapping volume list volumes={"name": "work"} iterates the mapping's keys and claims a volume named "name". A bare mapping should raise TypeError like a bare string, before any request. Generated with [Devin](https://devin.ai) Co-Authored-By: Devin <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- sdk/python/tests/test_client.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/sdk/python/tests/test_client.py b/sdk/python/tests/test_client.py index cfeaf463..1e3c8f7f 100644 --- a/sdk/python/tests/test_client.py +++ b/sdk/python/tests/test_client.py @@ -84,12 +84,13 @@ def test_claim_rejects_invalid_volumes(node, volumes, match): Client(node).new("rt:24.04", volumes=volumes) -def test_claim_rejects_a_bare_string_volume_list(node): +@pytest.mark.parametrize("volumes", ["work", {"name": "work"}]) +def test_claim_rejects_a_bare_string_or_mapping_volume_list(node, volumes): seen = recording_claim({"id": "sb_1", "token": "tok"}) with pytest.raises(TypeError): - Client(node).new("rt:24.04", volumes="work") + Client(node).new("rt:24.04", volumes=volumes) with pytest.raises(TypeError): - Template(Client(node), node, "task:v1", "none", "small").new(volumes="work") + Template(Client(node), node, "task:v1", "none", "small").new(volumes=volumes) assert seen == [] From 9adcfe2a0241b90c0aac446b2c62761bc61b1791 Mon Sep 17 00:00:00 2001 From: minatoaquaMK2 Date: Wed, 21 Oct 2026 01:22:39 +0000 Subject: [PATCH 75/75] fix(sdk/python): reject a bare mapping volume list in claims _claim_body rejected a bare string but iterated a bare mapping's keys, so volumes={"name": "work"} claimed a volume named "name". It now raises TypeError for either before sending the claim. Generated with [Devin](https://devin.ai) Co-Authored-By: Devin <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- sdk/python/cocoonsandbox/template.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sdk/python/cocoonsandbox/template.py b/sdk/python/cocoonsandbox/template.py index 386ca584..7d123b3f 100644 --- a/sdk/python/cocoonsandbox/template.py +++ b/sdk/python/cocoonsandbox/template.py @@ -59,7 +59,7 @@ def _claim_body( mount: bool = True, claim_ref: str = "", ) -> dict[str, Any]: - if isinstance(volumes, str): + if isinstance(volumes, (str, Mapping)): raise TypeError("volumes takes a list of names or mappings") claim: dict[str, Any] = {"template": template} if net: