Skip to content

Secure, Fast, and Extensible Sandbox runtime for AI agents.

fix(egress): protect lazy vault materialization after null bindings

lihaopenga3c5364738d9
diff --git a/components/egress/fastsandbox_actions.go b/components/egress/fastsandbox_actions.go
index c411c55a8..8c1255374 100644
--- a/components/egress/fastsandbox_actions.go
+++ b/components/egress/fastsandbox_actions.go
@@ -245,7 +245,7 @@ func (s *fastSandboxPolicyServer) applySetBinding(ctx context.Context, env *acti
return http.StatusInternalServerError, fmt.Errorf("active subject %s has no registered network", subj)
}
}
- willPurge := token != nil && (!token.expectedNew.Matches(fence) || !token.purged) || token == nil && registered && !old.Matches(fence)
+ willPurge := (token != nil && (!token.expectedNew.Matches(fence) || !token.purged)) || (token == nil && registered && !old.Matches(fence))
if willPurge {
status, purgeErr := s.vaultPurgeStatusLocked(subj)
if purgeErr != nil {
diff --git a/components/egress/fastsandbox_server.go b/components/egress/fastsandbox_server.go
index 0682fbaed..6256a84c7 100644
--- a/components/egress/fastsandbox_server.go
+++ b/components/egress/fastsandbox_server.go
@@ -377,7 +377,14 @@ func (s *fastSandboxPolicyServer) handleCredentialVaultActive(w http.ResponseWri
if !registered || !s.fenceMatchesLocked(subj, fence) || s.subjAtt[subj].IP != ip {
return func(w http.ResponseWriter) { http.Error(w, "no subject for clientIp", http.StatusNotFound) }
}
- return captureActiveVaultSnapshot(r, s.vaultForLocked(subj, fence))
+ store := s.vaultForLocked(subj, fence)
+ if s.vaultSealedLocked(subj) {
+ return func(w http.ResponseWriter) { http.Error(w, "subject vault is sealed", http.StatusConflict) }
+ }
+ if entry := s.vaults[subj]; entry.vaultStateUnknown {
+ return func(w http.ResponseWriter) { http.Error(w, "vault state is unknown", http.StatusServiceUnavailable) }
+ }
+ return captureActiveVaultSnapshot(r, store)
}()
respond(w)
}
@@ -425,9 +432,12 @@ type vaultEntry struct {
fileMeta vaultFileMetadata
vaultStateUnknown bool
// vaultStateProtected records that this live fence previously observed a
- // non-v1 source (v2 or an unsupported schema). A later missing/legacy file
- // cannot erase that provenance and reopen the v1 writer path.
+ // non-v1 or unreadable source. A later missing/legacy file cannot erase that
+ // provenance and reopen the v1 writer path.
vaultStateProtected bool
+ // lazyVaultProbe limits retry-on-read to entries first created without a
+ // binding reload; ordinary reload failures still require SET_BINDING replay.
+ lazyVaultProbe bool
}
// vaultFileMetadata keeps the complete decoded ordering/initialization
@@ -471,12 +481,24 @@ func (s *fastSandboxPolicyServer) vaultForLocked(subj subject.Subject, fence sub
}
if e, ok := s.vaults[subj]; ok {
if e.fence == fence {
+ if e.lazyVaultProbe && e.vaultStateUnknown && !e.vaultStateProtected {
+ s.inspectLazyVaultStateLocked(subj, &e)
+ }
+ if s.vaultFiles != nil && s.vaultFiles.isSealed(subj) {
+ e.sealed = true
+ }
+ s.vaults[subj] = e
return e.store
}
delete(s.vaults, subj)
}
v := credentialvault.NewStore(nil, func() bool { return true })
- s.vaults[subj] = vaultEntry{store: v, fence: fence}
+ e := vaultEntry{store: v, fence: fence, lazyVaultProbe: true}
+ s.inspectLazyVaultStateLocked(subj, &e)
+ if s.vaultFiles != nil && s.vaultFiles.isSealed(subj) {
+ e.sealed = true
+ }
+ s.vaults[subj] = e
return v
}
diff --git a/components/egress/fastsandbox_vault.go b/components/egress/fastsandbox_vault.go
index 60340a056..6a3c4dbf4 100644
--- a/components/egress/fastsandbox_vault.go
+++ b/components/egress/fastsandbox_vault.go
@@ -162,6 +162,11 @@ func (s *fastSandboxPolicyServer) handleVaultRelayWrite(w http.ResponseWriter, r
if store == nil {
return func() { http.Error(w, "unknown subject", http.StatusNotFound) }
}
+ if s.vaultSealedLocked(subj) {
+ return func() {
+ http.Error(w, "subject vault is sealed (paused); the bundle copy is already taken", http.StatusConflict)
+ }
+ }
if entry, ok := s.vaults[subj]; ok {
if entry.vaultStateUnknown {
return func() { http.Error(w, "vault state is unknown; refusing mutation", http.StatusServiceUnavailable) }
@@ -302,13 +307,16 @@ func (s *fastSandboxPolicyServer) readRelayVault(r *http.Request, subj subject.S
if s.vaultSealedLocked(subj) {
return credentialvault.FullRevision{}, http.StatusConflict, "subject vault is sealed (paused); the bundle copy is already taken", nil
}
- if entry, ok := s.vaults[subj]; ok && entry.vaultStateUnknown {
- return credentialvault.FullRevision{}, http.StatusServiceUnavailable, "vault state is unknown; refusing to treat it as absent", nil
- }
store := s.vaultForLocked(subj, fence)
if store == nil {
return credentialvault.FullRevision{}, http.StatusNotFound, "unknown subject", nil
}
+ if s.vaultSealedLocked(subj) {
+ return credentialvault.FullRevision{}, http.StatusConflict, "subject vault is sealed (paused); the bundle copy is already taken", nil
+ }
+ if entry := s.vaults[subj]; entry.vaultStateUnknown {
+ return credentialvault.FullRevision{}, http.StatusServiceUnavailable, "vault state is unknown; refusing to treat it as absent", nil
+ }
full, err := store.Full()
return full, 0, "", err
}
diff --git a/components/egress/fastsandbox_vault_persist.go b/components/egress/fastsandbox_vault_persist.go
index 1ee477798..0913a8068 100644
--- a/components/egress/fastsandbox_vault_persist.go
+++ b/components/egress/fastsandbox_vault_persist.go
@@ -308,6 +308,44 @@ func (p *vaultPersistence) isSealed(subj subject.Subject) bool {
return err == nil
}
+// inspectLazyVaultStateLocked discovers only whether a lazy vault must stay
+// unavailable to the v1 relay path. Removal bindings have no policy against
+// which to reload legacy contents, so this deliberately does not import or
+// quarantine v1 files. Callers hold s.mu.
+func (s *fastSandboxPolicyServer) inspectLazyVaultStateLocked(subj subject.Subject, entry *vaultEntry) {
+ if s.vaultFiles == nil || entry.vaultStateProtected {
+ return
+ }
+ fileStore, err := s.vaultFiles.storeFor(subj)
+ if err != nil {
+ entry.degraded = fmt.Sprintf("vault file store unavailable: %v", err)
+ entry.vaultStateUnknown = true
+ return
+ }
+ snapshot, err := fileStore.Snapshot()
+ if err != nil {
+ entry.degraded = fmt.Sprintf("vault file unreadable: %v", err)
+ entry.vaultStateUnknown = true
+ return
+ }
+ entry.degraded = ""
+ entry.vaultStateUnknown = false
+ if !snapshot.Exists() {
+ return
+ }
+ vf, _, decodeErr := credentialvault.DecodeVaultFile(snapshot.Data())
+ if decodeErr == nil && vf.Version == credentialvault.VaultFileVersion {
+ return
+ }
+ entry.vaultStateProtected = true
+ entry.vaultStateUnknown = true
+ entry.degraded = "unreadable or non-v1 vault state requires a lineage-aware reload"
+ entry.fileMeta = vaultFileMetadata{
+ version: vf.Version, revision: vf.Revision, state: vf.State, lineage: vf.StateLineage,
+ stateRevision: vf.StateRevision, initializationID: vf.InitializationID,
+ }
+}
+
// SetVaultFiles wires persistence under the relay handlers; nil (the default)
// keeps the vault memory-only.
func (s *fastSandboxPolicyServer) SetVaultFiles(vp *vaultPersistence) {
diff --git a/components/egress/fastsandbox_vault_snapshot.go b/components/egress/fastsandbox_vault_snapshot.go
index 75e91830a..6dc618f24 100644
--- a/components/egress/fastsandbox_vault_snapshot.go
+++ b/components/egress/fastsandbox_vault_snapshot.go
@@ -235,9 +235,7 @@ func (s *fastSandboxPolicyServer) exportSourceImageLocked(subj subject.Subject,
// treated as absent, preserving the legacy behavior.
if decErr != nil || vf.Incarnation != regFence.RuntimeInstanceID || vf.AttachmentID != regFence.AttachmentID {
what := "unattributable fence"
- if errors.Is(decErr, credentialvault.ErrUnsupportedFileVersion) {
- what = "newer schema (preserved for the newer writer)"
- } else if decErr != nil {
+ if decErr != nil {
what = "unreadable"
}
log.Infof("subject %s: vault file not carried (%s)", subj, what)
diff --git a/components/egress/fastsandbox_vault_v2_compat_test.go b/components/egress/fastsandbox_vault_v2_compat_test.go
index 6214ba8e7..3f17cbba2 100644
--- a/components/egress/fastsandbox_vault_v2_compat_test.go
+++ b/components/egress/fastsandbox_vault_v2_compat_test.go
@@ -221,6 +221,151 @@ func TestVaultV2EmptyAndUnknownStateDoNotBecomeAbsent(t *testing.T) {
}
}
+func TestNullBindingLazyVaultAccessPreservesProtectedDiskState(t *testing.T) {
+ for _, file := range []struct {
+ name string
+ raw []byte
+ }{
+ {name: "active v2", raw: encodeVaultV2Fixture(t, credentialvault.FileStateActive, 7, 12, "", bundleVaultBody(t))},
+ {name: "deleted v2", raw: encodeVaultV2Fixture(t, credentialvault.FileStateDeleted, 0, 13, "", emptyVaultFileBody())},
+ {name: "empty v2", raw: encodeVaultV2Fixture(t, credentialvault.FileStateEmpty, 0, 14, "init-1", emptyVaultFileBody())},
+ {name: "foreign fence v2", raw: []byte(strings.Replace(string(encodeVaultV2Fixture(t, credentialvault.FileStateActive, 7, 12, "", bundleVaultBody(t))), testFenceR, "runtime-foreign", 1))},
+ {name: "truncated v2", raw: []byte(`{"v":2,"incarnation":"runtime-1"`)},
+ {name: "unknown schema", raw: []byte(`{"v":9,"incarnation":"runtime-1","attachmentId":"attachment-1","revision":7,"vault":{}}` + "\n")},
+ {name: "unreadable bytes", raw: []byte("not-json\n")},
+ } {
+ for _, method := range []string{http.MethodGet, http.MethodPost, http.MethodPut, http.MethodDelete} {
+ t.Run(file.name+"/"+method, func(t *testing.T) {
+ h := newVaultPersistHarness(t)
+ f := seedVaultV2Fixture(h, vaultTestUID, file.raw)
+ set := doAction(t, h.srv, bindBody(t, vaultTestUID, vaultTestIP, testFenceR, testFenceA, 1, nil))
+ require.Equal(t, http.StatusOK, set.Code)
+ before := append([]byte(nil), f.data...)
+ body := ""
+ if method == http.MethodPost {
+ body = `{"credentials":[],"bindings":[]}`
+ } else if method == http.MethodPut {
+ body = `{"credentials":[],"bindings":[]}`
+ }
+ rec := doVault(t, h.srv, method, vaultTestUID, body, "")
+ assert.Equal(t, http.StatusServiceUnavailable, rec.Code, "lazy materialization must not expose or overwrite protected state")
+ assert.True(t, f.exists)
+ assert.Equal(t, before, f.data)
+ assert.Zero(t, f.saveCalls)
+ assert.Zero(t, f.deleteCalls, "lazy inspection must never quarantine")
+ })
+ }
+ }
+}
+
+func TestNullBindingLazyVaultProtectionIsStickyAndRetriesTransientIO(t *testing.T) {
+ for _, replacement := range []string{"missing", "legacy"} {
+ t.Run("protected-state-"+replacement, func(t *testing.T) {
+ h := newVaultPersistHarness(t)
+ f := seedVaultV2Fixture(h, vaultTestUID, encodeVaultV2Fixture(t, credentialvault.FileStateEmpty, 0, 14, "init-1", emptyVaultFileBody()))
+ require.Equal(t, http.StatusOK, doAction(t, h.srv, bindBody(t, vaultTestUID, vaultTestIP, testFenceR, testFenceA, 1, nil)).Code)
+ assert.Equal(t, http.StatusServiceUnavailable, doVault(t, h.srv, http.MethodGet, vaultTestUID, "", "").Code)
+ if replacement == "missing" {
+ f.data, f.exists = nil, false
+ } else {
+ h.seedVaultFile(vaultTestUID, testFenceR, testFenceA, 7, `{"credentials":[],"bindings":[]}`)
+ }
+ before := append([]byte(nil), f.data...)
+ for _, method := range []string{http.MethodGet, http.MethodPost} {
+ body := ""
+ if method == http.MethodPost {
+ body = `{"credentials":[],"bindings":[]}`
+ }
+ assert.Equal(t, http.StatusServiceUnavailable, doVault(t, h.srv, method, vaultTestUID, body, "").Code)
+ }
+ assert.Equal(t, before, f.data)
+ assert.Zero(t, f.saveCalls)
+ assert.Zero(t, f.deleteCalls)
+ })
+ }
+
+ for _, source := range []string{"snapshot", "storeFor"} {
+ t.Run("transient-"+source, func(t *testing.T) {
+ h := newVaultPersistHarness(t)
+ f := h.fileFor(vaultTestUID)
+ h.seedVaultFile(vaultTestUID, testFenceR, testFenceA, 7, `{"credentials":[],"bindings":[]}`)
+ if source == "snapshot" {
+ f.snapErr = errors.New("temporary snapshot error")
+ } else {
+ h.newFileErr = errors.New("temporary store error")
+ }
+ require.Equal(t, http.StatusOK, doAction(t, h.srv, bindBody(t, vaultTestUID, vaultTestIP, testFenceR, testFenceA, 1, nil)).Code)
+ assert.Equal(t, http.StatusServiceUnavailable, doVault(t, h.srv, http.MethodGet, vaultTestUID, "", "").Code)
+ f.snapErr = nil
+ assert.Equal(t, http.StatusNotFound, doVault(t, h.srv, http.MethodGet, vaultTestUID, "", "").Code, "a successful v1/absence probe restores legacy empty-vault behavior")
+ assert.True(t, f.exists)
+ assert.Zero(t, f.saveCalls)
+ assert.Zero(t, f.deleteCalls)
+ })
+ }
+}
+
+func TestNullBindingLazyVaultMaterializationPreservesLegacyAndSealSemantics(t *testing.T) {
+ for _, withFile := range []bool{false, true} {
+ t.Run(map[bool]string{false: "absent", true: "legacy-v1"}[withFile], func(t *testing.T) {
+ h := newVaultPersistHarness(t)
+ f := h.fileFor(vaultTestUID)
+ if withFile {
+ h.seedVaultFile(vaultTestUID, testFenceR, testFenceA, 7, `{"credentials":[],"bindings":[]}`)
+ }
+ before := append([]byte(nil), f.data...)
+ require.Equal(t, http.StatusOK, doAction(t, h.srv, bindBody(t, vaultTestUID, vaultTestIP, testFenceR, testFenceA, 1, nil)).Code)
+ assert.Equal(t, http.StatusNotFound, doVault(t, h.srv, http.MethodGet, vaultTestUID, "", "").Code)
+ assert.Equal(t, before, f.data)
+ assert.Zero(t, f.deleteCalls)
+ post := doVault(t, h.srv, http.MethodPost, vaultTestUID, `{"credentials":[],"bindings":[]}`, "")
+ assert.Equal(t, http.StatusCreated, post.Code, "legacy v1/no-file subjects retain the v1 writer behavior")
+ })
+ }
+
+ h := newVaultPersistHarness(t)
+ subj := subject.FromSandboxUID(vaultTestUID)
+ require.NoError(t, h.vp.seal(subj))
+ require.Equal(t, http.StatusOK, doAction(t, h.srv, bindBody(t, vaultTestUID, vaultTestIP, testFenceR, testFenceA, 1, nil)).Code)
+ assert.Equal(t, http.StatusConflict, doVault(t, h.srv, http.MethodGet, vaultTestUID, "", "").Code)
+ assert.Equal(t, http.StatusConflict, doVault(t, h.srv, http.MethodPost, vaultTestUID, `{"credentials":[],"bindings":[]}`, "").Code)
+ active := httptest.NewRecorder()
+ h.srv.handleCredentialVaultActive(active, httptest.NewRequest(http.MethodGet, "/credential-vault/_active?clientIp="+vaultTestIP, nil))
+ assert.Equal(t, http.StatusConflict, active.Code, "the restored seal also covers the addon active-snapshot path")
+ f := h.fileFor(vaultTestUID)
+ assert.False(t, f.exists)
+ assert.Zero(t, f.saveCalls)
+}
+
+func TestNullBindingLazyVaultMaterializationProtectsActiveAddonAndMemoryOnlyLegacy(t *testing.T) {
+ h := newVaultPersistHarness(t)
+ seedVaultV2Fixture(h, vaultTestUID, encodeVaultV2Fixture(t, credentialvault.FileStateActive, 7, 12, "", bundleVaultBody(t)))
+ require.Equal(t, http.StatusOK, doAction(t, h.srv, bindBody(t, vaultTestUID, vaultTestIP, testFenceR, testFenceA, 1, nil)).Code)
+ active := httptest.NewRecorder()
+ h.srv.handleCredentialVaultActive(active, httptest.NewRequest(http.MethodGet, "/credential-vault/_active?clientIp="+vaultTestIP, nil))
+ assert.Equal(t, http.StatusServiceUnavailable, active.Code, "the addon path must not expose a lazy v2 file as an empty snapshot")
+
+ srv, _, _ := fastSandboxTestServer(t)
+ require.Equal(t, http.StatusOK, doAction(t, srv, bindBody(t, "memory-only", vaultTestIP, testFenceR, testFenceA, 1, nil)).Code)
+ assert.Equal(t, http.StatusNotFound, doVault(t, srv, http.MethodGet, "memory-only", "", "").Code)
+ assert.Equal(t, http.StatusCreated, doVault(t, srv, http.MethodPost, "memory-only", `{"credentials":[],"bindings":[]}`, "").Code)
+}
+
+func TestNullBindingVaultExportRejectsProtectedStateBeforeSeal(t *testing.T) {
+ h := newVaultPersistHarness(t)
+ raw := encodeVaultV2Fixture(t, credentialvault.FileStateDeleted, 0, 13, "", emptyVaultFileBody())
+ f := seedVaultV2Fixture(h, snapshotTestUID, raw)
+ require.Equal(t, http.StatusOK, doAction(t, h.srv, bindBody(t, snapshotTestUID, snapshotTestIP, testFenceR, testFenceA, 1, nil)).Code)
+ target := filepath.Join(t.TempDir(), "bundle", "v2.json")
+ rec := doAction(t, h.srv, snapshotBody(t, snapshotTestUID, testFenceR, testFenceA, "VAULT_EXPORT", target))
+ assert.Equal(t, http.StatusConflict, rec.Code)
+ assert.Equal(t, raw, f.data)
+ assert.False(t, h.fakeForPath(target).exists, "rejected export must not create a bundle copy")
+ assert.False(t, h.vp.isSealed(subject.FromSandboxUID(snapshotTestUID)), "rejected export must not seal")
+ assert.Zero(t, f.saveCalls)
+ assert.Zero(t, f.deleteCalls)
+}
+
func TestVaultV2ExportAndImportFailBeforeSideEffects(t *testing.T) {
h := newVaultPersistHarness(t)
active := encodeVaultV2Fixture(t, credentialvault.FileStateActive, 7, 12, "", bundleVaultBody(t))