Skip to content

Secure, Fast, and Extensible Sandbox runtime for AI agents.

64.8 KB
Raw Blame
// Copyright 2026 The OpenSandbox Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package main

import (
	"bytes"
	"context"
	"encoding/json"
	"errors"
	"fmt"
	"net/http"
	"net/http/httptest"
	"net/netip"
	"sync"
	"testing"
	"time"

	"github.com/stretchr/testify/assert"
	"github.com/stretchr/testify/require"

	"github.com/alibaba/opensandbox/egress/pkg/actionhandler"
	"github.com/alibaba/opensandbox/egress/pkg/constants"
	"github.com/alibaba/opensandbox/egress/pkg/credentialvault"
	"github.com/alibaba/opensandbox/egress/pkg/iptables"
	"github.com/alibaba/opensandbox/egress/pkg/mitmproxy"
	"github.com/alibaba/opensandbox/egress/pkg/policy"
	"github.com/alibaba/opensandbox/egress/pkg/subject"
)

// fakeNft implements fastSandboxNftApplier with recording.
type fakeNft struct {
	mu            sync.Mutex
	denyFirst     []subject.Subject
	policyApplied []subject.Subject
	lastPolicy    *policy.NetworkPolicy
	removed       []subject.Subject
	denyFirstErr  error
	policyErr     error
	removeErr     error
}

// fakeGatewayDNSRedirectApplier models the entire owned nft table, not only
// callback counts. A failure can occur either before the replacement or
// after it took effect (a lost/ambiguous result).
type fakeGatewayDNSRedirectApplier struct {
	gateways      map[netip.Addr]struct{}
	calls         int
	failAt        int
	failErr       error
	applyThenFail bool
}

func newFakeGatewayDNSRedirectApplier() *fakeGatewayDNSRedirectApplier {
	return &fakeGatewayDNSRedirectApplier{gateways: make(map[netip.Addr]struct{})}
}

func (f *fakeGatewayDNSRedirectApplier) apply(_ context.Context, gateways []netip.Addr, _ int) error {
	f.calls++
	fail := f.calls == f.failAt
	if fail && !f.applyThenFail {
		return f.failErr
	}
	f.gateways = make(map[netip.Addr]struct{}, len(gateways))
	for _, gateway := range gateways {
		f.gateways[gateway.Unmap()] = struct{}{}
	}
	if fail {
		return f.failErr
	}
	return nil
}

func (f *fakeGatewayDNSRedirectApplier) failOnCall(call int, err error, afterApply bool) {
	f.failAt, f.failErr, f.applyThenFail = call, err, afterApply
}

func (f *fakeGatewayDNSRedirectApplier) clearFailures() {
	f.failAt, f.failErr, f.applyThenFail = 0, nil, false
}

func (f *fakeGatewayDNSRedirectApplier) snapshot() map[netip.Addr]struct{} {
	return f.gateways
}

func (f *fakeGatewayDNSRedirectApplier) callCount() int {
	return f.calls
}

func (f *fakeNft) ApplyDenyFirst(_ context.Context, s subject.Subject, _ actionhandler.NetworkAttachment) error {
	f.mu.Lock()
	defer f.mu.Unlock()
	if f.denyFirstErr != nil {
		return f.denyFirstErr
	}
	f.denyFirst = append(f.denyFirst, s)
	return nil
}

func (f *fakeNft) ApplyPolicy(_ context.Context, s subject.Subject, pol *policy.NetworkPolicy) error {
	f.mu.Lock()
	defer f.mu.Unlock()
	if f.policyErr != nil {
		return f.policyErr
	}
	f.policyApplied = append(f.policyApplied, s)
	f.lastPolicy = pol
	return nil
}

func (f *fakeNft) Remove(_ context.Context, s subject.Subject) error {
	f.mu.Lock()
	defer f.mu.Unlock()
	if f.removeErr != nil {
		return f.removeErr
	}
	f.removed = append(f.removed, s)
	return nil
}

func (f *fakeNft) appliedCount() int {
	f.mu.Lock()
	defer f.mu.Unlock()
	return len(f.policyApplied)
}

func fastSandboxTestServer(t *testing.T) (*fastSandboxPolicyServer, *subject.MemoryRegistry, *fakeNft) {
	t.Helper()
	reg := subject.NewRegistry(nil, nil)
	nft := &fakeNft{}
	srv := newFastSandboxPolicyServer(context.Background(), reg, nft)
	// no-op the gateway DNS redirect (no iptables/nft in unit tests)
	srv.dnsRedirectApply = func(context.Context, []netip.Addr, int) error { return nil }
	// Policy persistence is REQUIRED in the fast-sandbox profile — every
	// test server carries a store, mirroring the production posture (no
	// memory-only mode). The file seam defaults to a cross-platform fake
	// keyed by path (the atomic machinery itself is Linux-only); tests that
	// need real files or outcome injection build their own harness.
	pp, err := newPolicyPersistence(t.TempDir())
	require.NoError(t, err)
	files := make(map[string]*fakeVaultFile)
	pp.newFile = func(path string) (vaultFileStore, error) {
		f, ok := files[path]
		if !ok {
			f = newFakeVaultFile(path)
			files[path] = f
		}
		return f, nil
	}
	srv.SetPolicyFiles(pp)
	return srv, reg, nft
}

// wireTestPolicyPersistence attaches the required temp-dir-backed policy
// persistence to a server constructed directly (bypassing
// fastSandboxTestServer).
func wireTestPolicyPersistence(t *testing.T, srv *fastSandboxPolicyServer) {
	t.Helper()
	pp, err := newPolicyPersistence(t.TempDir())
	require.NoError(t, err)
	files := make(map[string]*fakeVaultFile)
	pp.newFile = func(path string) (vaultFileStore, error) {
		f, ok := files[path]
		if !ok {
			f = newFakeVaultFile(path)
			files[path] = f
		}
		return f, nil
	}
	srv.SetPolicyFiles(pp)
}

func doRequest(t *testing.T, srv *fastSandboxPolicyServer, method, path, body string) *httptest.ResponseRecorder {
	t.Helper()
	var rd *bytes.Reader
	if body == "" {
		rd = bytes.NewReader(nil)
	} else {
		rd = bytes.NewReader([]byte(body))
	}
	req := httptest.NewRequest(method, path, rd)
	rec := httptest.NewRecorder()
	srv.Handler().ServeHTTP(rec, req)
	return rec
}

// doAction posts an action envelope to the Fastlet dispatch endpoint.
func doAction(t *testing.T, srv *fastSandboxPolicyServer, body string) *httptest.ResponseRecorder {
	t.Helper()
	return doRequest(t, srv, http.MethodPost, constants.ActionsDispatchPath, body)
}

// ---------------------------------------------------------------------------
// Envelope builders (wire format via the actionhandler types)
// ---------------------------------------------------------------------------

const (
	testUID    = "u-1"
	testIP     = "10.0.0.5"
	testGW     = "10.0.0.1"
	testFenceR = "runtime-1"
	testFenceA = "attachment-1"
)

func testAttachment() *actionhandler.Attachment {
	return &actionhandler.Attachment{Network: actionhandler.NetworkAttachment{
		IP:          netip.MustParseAddr(testIP),
		Gateway:     netip.MustParseAddr(testGW),
		PrivateCIDR: netip.MustParsePrefix("10.0.0.0/24"),
		HostVeth:    "veth0",
	}}
}

// bindBody builds a SET_BINDING envelope; input nil = JSON-null removal.
func bindBody(t *testing.T, uid, ip, runtimeID, attID string, specGen uint64, input *string) string {
	t.Helper()
	env := actionhandler.Envelope{
		APIVersion:   constants.ActionsAPIVersion,
		Operation:    actionhandler.OperationSetBinding,
		InvocationID: fmt.Sprintf("inv-%s-%d", uid, specGen),
		Sandbox:      actionhandler.SandboxRef{UID: uid, Name: "sb", Namespace: "ns"},
		Revision: actionhandler.Revision{
			SpecGeneration:    specGen,
			RuntimeInstanceID: runtimeID,
			AttachmentID:      attID,
			RouteGeneration:   1,
		},
		Attachment: testAttachment(),
		Binding:    &actionhandler.Binding{},
	}
	if ip != testIP {
		env.Attachment.Network.IP = netip.MustParseAddr(ip)
	}
	if input != nil {
		quoted, err := json.Marshal(*input)
		require.NoError(t, err)
		env.Binding.Input = json.RawMessage(quoted)
	} else {
		env.Binding.Input = json.RawMessage("null")
	}
	raw, err := json.Marshal(env)
	require.NoError(t, err)
	return string(raw)
}

func hookBody(t *testing.T, uid, runtimeID, attID, name string) string {
	t.Helper()
	env := actionhandler.Envelope{
		APIVersion:   constants.ActionsAPIVersion,
		Operation:    actionhandler.OperationLifecycleHook,
		InvocationID: fmt.Sprintf("inv-%s-hook", uid),
		Sandbox:      actionhandler.SandboxRef{UID: uid, Name: "sb", Namespace: "ns"},
		Revision: actionhandler.Revision{
			SpecGeneration:    1,
			RuntimeInstanceID: runtimeID,
			AttachmentID:      attID,
			RouteGeneration:   1,
		},
		Hook: &actionhandler.Hook{Name: name, Sequence: 1},
	}
	raw, err := json.Marshal(env)
	require.NoError(t, err)
	return string(raw)
}

func removeBody(t *testing.T, uid, runtimeID, attID string) string {
	t.Helper()
	env := actionhandler.Envelope{
		APIVersion:   constants.ActionsAPIVersion,
		Operation:    actionhandler.OperationRemoveBinding,
		InvocationID: fmt.Sprintf("inv-%s-remove", uid),
		Sandbox:      actionhandler.SandboxRef{UID: uid, Name: "sb", Namespace: "ns"},
		Revision: actionhandler.Revision{
			SpecGeneration:    1,
			RuntimeInstanceID: runtimeID,
			AttachmentID:      attID,
			RouteGeneration:   1,
		},
	}
	raw, err := json.Marshal(env)
	require.NoError(t, err)
	return string(raw)
}

// setBindingAndReady drives a sandbox to active through the action protocol:
// SET_BINDING -> runtime-ready -> data-plane-ready.
func setBindingAndReady(t *testing.T, srv *fastSandboxPolicyServer, reg *subject.MemoryRegistry, uid, ip string, input *string) subject.Subject {
	t.Helper()
	s := subject.FromSandboxUID(uid)
	rec := doAction(t, srv, bindBody(t, uid, ip, testFenceR, testFenceA, 1, input))
	require.Equal(t, http.StatusOK, rec.Code)
	rec = doAction(t, srv, hookBody(t, uid, testFenceR, testFenceA, constants.HookRuntimeReady))
	require.Equal(t, http.StatusOK, rec.Code)
	rec = doAction(t, srv, hookBody(t, uid, testFenceR, testFenceA, constants.HookDataPlaneReady))
	require.Equal(t, http.StatusOK, rec.Code)
	require.Equal(t, subject.StateActive, mustState(reg.Get(s)))
	return s
}

// ---------------------------------------------------------------------------
// Status endpoint
// ---------------------------------------------------------------------------

func TestActionsStatus(t *testing.T) {
	srv, _, _ := fastSandboxTestServer(t)
	rec := doRequest(t, srv, http.MethodGet, constants.ActionsStatusPath, "")
	require.Equal(t, http.StatusOK, rec.Code)
	var status actionhandler.StatusResponse
	require.NoError(t, json.Unmarshal(rec.Body.Bytes(), &status))
	assert.Equal(t, constants.ActionsAPIVersion, status.APIVersion)
	assert.True(t, status.Ready)
	assert.NotEmpty(t, status.InstanceID)
	// No vault persistence configured: the capability stays false.
	assert.False(t, status.VaultTransfer)
	assert.NotContains(t, rec.Body.String(), "vaultTransfer", "the capability field stays omitted when false")

	// instanceId is stable for the process incarnation
	rec = doRequest(t, srv, http.MethodGet, constants.ActionsStatusPath, "")
	require.NoError(t, json.Unmarshal(rec.Body.Bytes(), &status))
	assert.Equal(t, srv.instanceID, status.InstanceID)
}

func TestActionsStatusMirrorsMitmGate(t *testing.T) {
	t.Setenv(constants.EnvMitmproxyTransparent, "true")
	srv, _, _ := fastSandboxTestServer(t)
	gate := mitmproxy.NewHealthGate()
	srv.mitmGate = gate
	gate.SetReady(false)

	rec := doRequest(t, srv, http.MethodGet, constants.ActionsStatusPath, "")
	require.Equal(t, http.StatusOK, rec.Code)
	var status actionhandler.StatusResponse
	require.NoError(t, json.Unmarshal(rec.Body.Bytes(), &status))
	assert.False(t, status.Ready, "handler not ready while the MITM stack is pending")

	gate.SetReady(true)
	rec = doRequest(t, srv, http.MethodGet, constants.ActionsStatusPath, "")
	require.NoError(t, json.Unmarshal(rec.Body.Bytes(), &status))
	assert.True(t, status.Ready)
}

// ---------------------------------------------------------------------------
// SET_BINDING + LIFECYCLE_HOOK lifecycle
// ---------------------------------------------------------------------------

func TestActionsSetBindingDenyFirstThenHookActivates(t *testing.T) {
	srv, reg, nft := fastSandboxTestServer(t)
	s := subject.FromSandboxUID(testUID)
	input := `{"defaultAction":"deny","egress":[{"action":"allow","target":"example.com"}]}`

	// SET_BINDING: deny-first registered, policy stored pending, DNS denies
	rec := doAction(t, srv, bindBody(t, testUID, testIP, testFenceR, testFenceA, 1, &input))
	require.Equal(t, http.StatusOK, rec.Code)
	require.Equal(t, subject.StateDenying, mustState(reg.Get(s)))
	require.Len(t, nft.denyFirst, 1)
	assert.Nil(t, reg.EffectivePolicy(s), "DNS must keep denying while the policy is pending")

	// runtime-ready confirms; still denying
	rec = doAction(t, srv, hookBody(t, testUID, testFenceR, testFenceA, constants.HookRuntimeReady))
	require.Equal(t, http.StatusOK, rec.Code)
	require.Equal(t, subject.StateDenying, mustState(reg.Get(s)))

	// data-plane-ready applies the pending policy -> active
	rec = doAction(t, srv, hookBody(t, testUID, testFenceR, testFenceA, constants.HookDataPlaneReady))
	require.Equal(t, http.StatusOK, rec.Code)
	require.Equal(t, subject.StateActive, mustState(reg.Get(s)))
	require.Equal(t, 1, nft.appliedCount())
	eff := reg.EffectivePolicy(s)
	require.NotNil(t, eff)
	assert.Equal(t, "allow", eff.Evaluate("example.com"))
}

// TestActionsSetBindingReplayNeverMutatesRecordedPolicy: the binding input
// only seeds a fresh subject's policy. A same-fence SET_BINDING carrying a
// different input (the superseded CRD-driven update path, or an at-least-once
// replay) must never mutate the recorded policy — runtime changes go through
// the relay surface.
func TestActionsSetBindingReplayNeverMutatesRecordedPolicy(t *testing.T) {
	srv, reg, nft := fastSandboxTestServer(t)
	input := `{"defaultAction":"deny","egress":[{"action":"allow","target":"example.com"}]}`
	s := setBindingAndReady(t, srv, reg, testUID, testIP, &input)
	nft.mu.Lock()
	denyFirstInstalls := len(nft.denyFirst)
	appliedAfterSeed := len(nft.policyApplied)
	nft.mu.Unlock()

	// same-fence binding with a different input: the recorded policy wins
	newInput := `{"defaultAction":"deny","egress":[{"action":"allow","target":"updated.com"}]}`
	rec := doAction(t, srv, bindBody(t, testUID, testIP, testFenceR, testFenceA, 2, &newInput))
	require.Equal(t, http.StatusOK, rec.Code)
	require.Equal(t, subject.StateActive, mustState(reg.Get(s)))
	nft.mu.Lock()
	assert.Equal(t, appliedAfterSeed, len(nft.policyApplied), "a replayed binding must not re-apply a policy")
	assert.Equal(t, denyFirstInstalls, len(nft.denyFirst), "a replayed binding must not re-install deny-first")
	nft.mu.Unlock()
	assert.Equal(t, "allow", reg.EffectivePolicy(s).Evaluate("example.com"), "the seeded policy survives the replay")
	assert.Equal(t, "deny", reg.EffectivePolicy(s).Evaluate("updated.com"))
}

func TestActionsSetBindingNullRemovesPolicy(t *testing.T) {
	srv, reg, nft := fastSandboxTestServer(t)
	input := `{"defaultAction":"deny","egress":[{"action":"allow","target":"example.com"}]}`
	s := setBindingAndReady(t, srv, reg, testUID, testIP, &input)
	denyFirstInstalls := len(nft.denyFirst)

	// binding removed from a still-live sandbox: back to deny-first
	rec := doAction(t, srv, bindBody(t, testUID, testIP, testFenceR, testFenceA, 3, nil))
	require.Equal(t, http.StatusOK, rec.Code)
	require.Equal(t, subject.StateDenying, mustState(reg.Get(s)))
	assert.Nil(t, reg.EffectivePolicy(s))
	assert.Nil(t, reg.UserPolicy(s))
	require.Equal(t, denyFirstInstalls+1, len(nft.denyFirst), "revert must re-install deny-first")

	// the sandbox is still registered and can be re-activated by a new binding
	reInput := `{"defaultAction":"deny"}`
	rec = doAction(t, srv, bindBody(t, testUID, testIP, testFenceR, testFenceA, 4, &reInput))
	require.Equal(t, http.StatusOK, rec.Code)
	rec = doAction(t, srv, hookBody(t, testUID, testFenceR, testFenceA, constants.HookDataPlaneReady))
	require.Equal(t, http.StatusOK, rec.Code)
	require.Equal(t, subject.StateActive, mustState(reg.Get(s)))
}

func TestActionsDataPlaneReadyWithoutPendingPolicyFailsClosed(t *testing.T) {
	srv, reg, _ := fastSandboxTestServer(t)
	s := subject.FromSandboxUID(testUID)

	// binding removed immediately (null input): no pending policy exists
	rec := doAction(t, srv, bindBody(t, testUID, testIP, testFenceR, testFenceA, 1, nil))
	require.Equal(t, http.StatusOK, rec.Code)
	require.Equal(t, subject.StateDenying, mustState(reg.Get(s)))

	// a (protocol-violating) data-plane-ready without a pending policy must
	// fail closed: the subject never activates
	rec = doAction(t, srv, hookBody(t, testUID, testFenceR, testFenceA, constants.HookDataPlaneReady))
	require.Equal(t, http.StatusConflict, rec.Code)
	require.Equal(t, subject.StateDenying, mustState(reg.Get(s)))
}

func TestActionsRebindDiscardsPolicy(t *testing.T) {
	srv, reg, nft := fastSandboxTestServer(t)
	input := `{"defaultAction":"deny","egress":[{"action":"allow","target":"old.com"}]}`
	s := setBindingAndReady(t, srv, reg, testUID, testIP, &input)

	// same UID, new runtime instance: full reset to deny-first, old policy gone
	rec := doAction(t, srv, bindBody(t, testUID, testIP, "runtime-2", "attachment-2", 5, &input))
	require.Equal(t, http.StatusOK, rec.Code)
	require.Equal(t, subject.StateDenying, mustState(reg.Get(s)))
	assert.Nil(t, reg.EffectivePolicy(s), "old policy must not survive a rebind")

	// the new instance's data-plane-ready activates with its own (re-seeded)
	// policy
	rec = doAction(t, srv, hookBody(t, testUID, "runtime-2", "attachment-2", constants.HookDataPlaneReady))
	require.Equal(t, http.StatusOK, rec.Code)
	require.Equal(t, subject.StateActive, mustState(reg.Get(s)))
	assert.Equal(t, "allow", reg.EffectivePolicy(s).Evaluate("old.com"))

	// a runtime policy change goes through the relay: the relayed policy is
	// what the successor enforces
	rec = doPolicy(t, srv, http.MethodPut, testUID, policyPutDoc(
		`{"defaultAction":"deny","egress":[{"action":"allow","target":"new.com"}]}`, 1, false, true), "")
	require.Equal(t, http.StatusOK, rec.Code)
	assert.Equal(t, "allow", reg.EffectivePolicy(s).Evaluate("new.com"))
	assert.Equal(t, "deny", reg.EffectivePolicy(s).Evaluate("old.com"))
	require.Greater(t, len(nft.denyFirst), 0)
}

func TestActionsSetBindingIdempotentRetry(t *testing.T) {
	srv, reg, nft := fastSandboxTestServer(t)
	s := subject.FromSandboxUID(testUID)
	input := `{"defaultAction":"deny"}`

	// deny-first install fails: 500, subject stays absent-of-effective-policy
	nft.mu.Lock()
	nft.denyFirstErr = errors.New("nft busy")
	nft.mu.Unlock()
	rec := doAction(t, srv, bindBody(t, testUID, testIP, testFenceR, testFenceA, 1, &input))
	require.Equal(t, http.StatusInternalServerError, rec.Code)
	require.Equal(t, subject.StateDenying, mustState(reg.Get(s)))

	// fastlet retries the same logical operation; the retry succeeds
	nft.mu.Lock()
	nft.denyFirstErr = nil
	nft.mu.Unlock()
	rec = doAction(t, srv, bindBody(t, testUID, testIP, testFenceR, testFenceA, 1, &input))
	require.Equal(t, http.StatusOK, rec.Code)
	require.Equal(t, subject.StateDenying, mustState(reg.Get(s)))
	require.Len(t, nft.denyFirst, 1, "only the successful attempt installs deny-first")
}

func TestActionsInvalidPolicyInputRejected(t *testing.T) {
	srv, reg, _ := fastSandboxTestServer(t)
	bad := "not-json"
	rec := doAction(t, srv, bindBody(t, testUID, testIP, testFenceR, testFenceA, 1, &bad))
	require.Equal(t, http.StatusBadRequest, rec.Code)
	_, ok := reg.Get(subject.FromSandboxUID(testUID))
	assert.False(t, ok, "invalid input must not leave the subject half-registered")

	// empty input and the literal string "null" are ordinary values -> deny
	empty := ""
	rec = doAction(t, srv, bindBody(t, testUID, testIP, testFenceR, testFenceA, 2, &empty))
	require.Equal(t, http.StatusOK, rec.Code)
	require.Equal(t, subject.StateDenying, mustState(reg.Get(subject.FromSandboxUID(testUID))))
}

// ---------------------------------------------------------------------------
// REMOVE_BINDING
// ---------------------------------------------------------------------------

func TestActionsRemoveBindingCleansUp(t *testing.T) {
	srv, reg, nft := fastSandboxTestServer(t)
	input := `{"defaultAction":"deny"}`
	s := setBindingAndReady(t, srv, reg, testUID, testIP, &input)

	rec := doAction(t, srv, removeBody(t, testUID, testFenceR, testFenceA))
	require.Equal(t, http.StatusOK, rec.Code)
	_, ok := reg.Get(s)
	assert.False(t, ok)
	require.Len(t, nft.removed, 1)
}

func TestActionsRemoveBindingMissingStateSuccess(t *testing.T) {
	srv, _, nft := fastSandboxTestServer(t)
	rec := doAction(t, srv, removeBody(t, "ghost", testFenceR, testFenceA))
	require.Equal(t, http.StatusOK, rec.Code, "missing Handler state is success")
	require.Len(t, nft.removed, 0)
}

func TestActionsRemoveBindingStaleFenceIgnored(t *testing.T) {
	srv, reg, nft := fastSandboxTestServer(t)
	input := `{"defaultAction":"deny"}`
	s := setBindingAndReady(t, srv, reg, testUID, testIP, &input)

	// a removal from a PREVIOUS instance of the same UID must not unload the
	// current subject
	rec := doAction(t, srv, removeBody(t, testUID, "runtime-old", "attachment-old"))
	require.Equal(t, http.StatusOK, rec.Code)
	require.Equal(t, subject.StateActive, mustState(reg.Get(s)))
	require.Len(t, nft.removed, 0)
}

func TestActionsHookForUnregisteredSubjectConflict(t *testing.T) {
	srv, _, _ := fastSandboxTestServer(t)
	rec := doAction(t, srv, hookBody(t, "ghost", testFenceR, testFenceA, constants.HookRuntimeReady))
	require.Equal(t, http.StatusConflict, rec.Code)
	rec = doAction(t, srv, hookBody(t, "ghost", testFenceR, testFenceA, constants.HookDataPlaneReady))
	require.Equal(t, http.StatusConflict, rec.Code)
}

// ---------------------------------------------------------------------------
// TestHookRemoveConcurrentNoRace (review: subjFence map race): a
// data-plane-ready hook (holding policyCommitMu, reading subjFence under
// s.mu) racing terminal cleanup (which deletes subjFence under s.mu without
// policyCommitMu) must be data-race-free and converge to a consistent
// outcome. Run under -race in CI.
func TestHookRemoveConcurrentNoRace(t *testing.T) {
	srv, reg, _ := fastSandboxTestServer(t)
	s := subject.FromSandboxUID(policyTestUID)

	rec := doAction(t, srv, bindBody(t, policyTestUID, vaultTestIP, testFenceR, testFenceA, 1, strPtr(policyAllow())))
	require.Equal(t, http.StatusOK, rec.Code)
	rec = doAction(t, srv, hookBody(t, policyTestUID, testFenceR, testFenceA, constants.HookRuntimeReady))
	require.Equal(t, http.StatusOK, rec.Code)

	done := make(chan struct{})
	go func() {
		defer close(done)
		r := doAction(t, srv, removeBody(t, policyTestUID, testFenceR, testFenceA))
		assert.Equal(t, http.StatusOK, r.Code)
	}()
	rec = doAction(t, srv, hookBody(t, policyTestUID, testFenceR, testFenceA, constants.HookDataPlaneReady))
	hookCode := rec.Code
	<-done

	// The removal always completes (we waited), so the converged terminal
	// state is absent regardless of the hook's outcome (200 activation
	// first → removal reverts; removal first → the hook is a 404/409).
	assert.Contains(t, []int{http.StatusOK, http.StatusNotFound, http.StatusConflict}, hookCode)
	_, present := reg.Get(s)
	assert.False(t, present, "terminal cleanup wins: the subject is gone")
}

// Stale-subject takeover on slot reuse (OSEP-0022)
// ---------------------------------------------------------------------------

// TestActionsSetBindingTakeoverOnSlotReuse: a SET_BINDING claiming a live
// subject's IP tears the stale subject down and registers the successor
// deny-first; a late REMOVE_BINDING for the stale subject stays a no-op.
func TestActionsSetBindingTakeoverOnSlotReuse(t *testing.T) {
	srv, reg, nft := fastSandboxTestServer(t)
	inst := &fakeMitmInstaller{}
	srv.mitmInstall = inst.install
	srv.mitmRemove = inst.remove

	staleUID, takeUID := "u-stale", "u-take"
	stale, succ := subject.FromSandboxUID(staleUID), subject.FromSandboxUID(takeUID)
	input := `{"defaultAction":"deny","egress":[{"action":"allow","target":"stale.com"}]}`
	setBindingAndReady(t, srv, reg, staleUID, testIP, &input)

	// the stale subject owns a vault slice
	vault := srv.vaultFor(stale)
	_, err := vault.Create(credentialvault.CreateRequest{}, reg.EffectivePolicy(stale))
	require.NoError(t, err)

	// a new sandbox's SET_BINDING claims the reused slot IP: takeover
	succInput := `{"defaultAction":"deny","egress":[{"action":"allow","target":"succ.com"}]}`
	rec := doAction(t, srv, bindBody(t, takeUID, testIP, "runtime-2", "attachment-2", 1, &succInput))
	require.Equal(t, http.StatusOK, rec.Code)

	// successor deny-first and owning the dispatch key
	require.Equal(t, subject.StateDenying, mustState(reg.Get(succ)))
	assert.Nil(t, reg.EffectivePolicy(succ), "successor must be deny-first until its own data-plane-ready")
	got, ok := reg.Resolve(subject.SubjectKey{SourceIP: netip.MustParseAddr(testIP)})
	require.True(t, ok)
	assert.Equal(t, succ, got)
	nft.mu.Lock()
	require.Contains(t, nft.denyFirst, succ)
	nft.mu.Unlock()

	// stale subject fully torn down
	_, ok = reg.Get(stale)
	assert.False(t, ok)
	nft.mu.Lock()
	require.Equal(t, []subject.Subject{stale}, nft.removed)
	nft.mu.Unlock()
	srv.mu.Lock()
	_, hasStaleVault := srv.vaults[stale]
	srv.mu.Unlock()
	assert.False(t, hasStaleVault, "the stale vault memory slice must be dropped")

	// gateway refcount and MITM maps hold only the successor
	srv.gwMu.Lock()
	gws := make(map[subject.Subject]netip.Addr, len(srv.gatewayDNSRefs))
	for gs, g := range srv.gatewayDNSRefs {
		gws[gs] = g
	}
	srv.gwMu.Unlock()
	assert.Equal(t, map[subject.Subject]netip.Addr{succ: netip.MustParseAddr(testGW)}, gws)
	srv.mitmMu.Lock()
	_, hasStaleMitm := srv.mitmEntries[stale]
	_, hasSuccMitm := srv.mitmEntries[succ]
	srv.mitmMu.Unlock()
	assert.False(t, hasStaleMitm)
	assert.True(t, hasSuccMitm)

	// the successor's lifecycle completes normally
	rec = doAction(t, srv, hookBody(t, takeUID, "runtime-2", "attachment-2", constants.HookDataPlaneReady))
	require.Equal(t, http.StatusOK, rec.Code)
	require.Equal(t, subject.StateActive, mustState(reg.Get(succ)))
	assert.Equal(t, "allow", reg.EffectivePolicy(succ).Evaluate("succ.com"))

	// a late REMOVE_BINDING for the stale subject: no-op, successor untouched
	nft.mu.Lock()
	removedBefore := len(nft.removed)
	nft.mu.Unlock()
	rec = doAction(t, srv, removeBody(t, staleUID, testFenceR, testFenceA))
	require.Equal(t, http.StatusOK, rec.Code)
	_, ok = reg.Get(succ)
	require.True(t, ok)
	require.Equal(t, subject.StateActive, mustState(reg.Get(succ)))
	nft.mu.Lock()
	assert.Equal(t, removedBefore, len(nft.removed), "the stale removal must not touch enforcement")
	nft.mu.Unlock()
	got, ok = reg.Resolve(subject.SubjectKey{SourceIP: netip.MustParseAddr(testIP)})
	require.True(t, ok)
	assert.Equal(t, succ, got)
}

// TestActionsSetBindingTakeoverDuplicateIdempotent: a duplicate SET_BINDING
// for the SAME UID on the same IP stays an idempotent no-op (not a takeover).
func TestActionsSetBindingTakeoverDuplicateIdempotent(t *testing.T) {
	srv, reg, nft := fastSandboxTestServer(t)
	input := `{"defaultAction":"deny"}`
	s := setBindingAndReady(t, srv, reg, testUID, testIP, &input)
	nft.mu.Lock()
	denyFirstAfterSetup := len(nft.denyFirst)
	nft.mu.Unlock()

	// retried delivery of the same SET_BINDING (same fence, same key)
	rec := doAction(t, srv, bindBody(t, testUID, testIP, testFenceR, testFenceA, 1, &input))
	require.Equal(t, http.StatusOK, rec.Code)
	require.Equal(t, subject.StateActive, mustState(reg.Get(s)))
	nft.mu.Lock()
	assert.Equal(t, denyFirstAfterSetup, len(nft.denyFirst), "no deny-first re-install")
	assert.Empty(t, nft.removed, "own-key re-claim must not tear anything down")
	nft.mu.Unlock()
}

// TestActionsSetBindingTakeoverNftFailureFailClosed: a failed stale teardown
// fails the takeover; the retry converges.
func TestActionsSetBindingTakeoverNftFailureFailClosed(t *testing.T) {
	srv, reg, nft := fastSandboxTestServer(t)
	staleUID, takeUID := "u-stale", "u-take"
	stale, succ := subject.FromSandboxUID(staleUID), subject.FromSandboxUID(takeUID)
	input := `{"defaultAction":"deny"}`
	setBindingAndReady(t, srv, reg, staleUID, testIP, &input)
	_, err := srv.vaultFor(stale).Create(credentialvault.CreateRequest{}, reg.EffectivePolicy(stale))
	require.NoError(t, err)

	nft.mu.Lock()
	nft.removeErr = errors.New("nft busy")
	nft.mu.Unlock()
	rec := doAction(t, srv, bindBody(t, takeUID, testIP, "runtime-2", "attachment-2", 1, strPtr(`{"defaultAction":"deny"}`)))
	require.Equal(t, http.StatusInternalServerError, rec.Code)

	// fail-closed: the stale subject keeps the slot
	require.Equal(t, subject.StateActive, mustState(reg.Get(stale)))
	_, ok := reg.Get(succ)
	assert.False(t, ok)
	got, ok := reg.Resolve(subject.SubjectKey{SourceIP: netip.MustParseAddr(testIP)})
	require.True(t, ok)
	assert.Equal(t, stale, got)
	srv.mu.Lock()
	_, hasStaleVault := srv.vaults[stale]
	srv.mu.Unlock()
	assert.True(t, hasStaleVault, "the stale vault must be untouched")

	// the retry converges
	nft.mu.Lock()
	nft.removeErr = nil
	nft.mu.Unlock()
	rec = doAction(t, srv, bindBody(t, takeUID, testIP, "runtime-2", "attachment-2", 1, strPtr(`{"defaultAction":"deny"}`)))
	require.Equal(t, http.StatusOK, rec.Code)
	require.Equal(t, subject.StateDenying, mustState(reg.Get(succ)))
	_, ok = reg.Get(stale)
	assert.False(t, ok)
}

// TestActionsSetBindingTakeoverGatewayRefcountConsistent: gateway refcount
// and MITM table stay consistent across the swap.
func TestActionsSetBindingTakeoverGatewayRefcountConsistent(t *testing.T) {
	reg := subject.NewRegistry(nil, nil)
	nft := &fakeNft{}
	srv := newFastSandboxPolicyServer(context.Background(), reg, nft)
	wireTestPolicyPersistence(t, srv)
	dns := newFakeGatewayDNSRedirectApplier()
	srv.dnsRedirectApply = dns.apply
	inst := &fakeMitmInstaller{}
	srv.mitmInstall = inst.install
	srv.mitmRemove = inst.remove

	staleUID, takeUID := "u-stale", "u-take"
	input := `{"defaultAction":"deny"}`
	rec := doAction(t, srv, bindBody(t, staleUID, testIP, testFenceR, testFenceA, 1, &input))
	require.Equal(t, http.StatusOK, rec.Code)
	require.Equal(t, map[netip.Addr]struct{}{netip.MustParseAddr(testGW): {}}, dns.snapshot())

	rec = doAction(t, srv, bindBody(t, takeUID, testIP, "runtime-2", "attachment-2", 1, &input))
	require.Equal(t, http.StatusOK, rec.Code)

	assert.Equal(t, 3, dns.callCount(), "stale release and successor install each reconcile the full set")
	assert.Equal(t, map[netip.Addr]struct{}{netip.MustParseAddr(testGW): {}}, dns.snapshot())
	srv.gwMu.Lock()
	_, hasStale := srv.gatewayDNSRefs[subject.FromSandboxUID(staleUID)]
	succ, hasSucc := srv.gatewayDNSRefs[subject.FromSandboxUID(takeUID)]
	srv.gwMu.Unlock()
	assert.False(t, hasStale)
	assert.True(t, hasSucc)
	assert.Equal(t, netip.MustParseAddr(testGW), succ)
	require.Equal(t, []iptables.MitmRedirectEntry{{SandboxIP: netip.MustParseAddr(testIP), Gateway: netip.MustParseAddr(testGW)}}, inst.snapshot())
}

// ---------------------------------------------------------------------------
// Envelope validation
// ---------------------------------------------------------------------------

func TestActionsInvalidEnvelopesRejected(t *testing.T) {
	srv, _, _ := fastSandboxTestServer(t)
	cases := []struct {
		name string
		body string
	}{
		{"bad apiVersion", `{"apiVersion":"v1","operation":"SET_BINDING","sandbox":{"uid":"u"},"revision":{"runtimeInstanceId":"r","attachmentId":"a"},"attachment":{"network":{"ip":"10.0.0.5","gateway":"10.0.0.1","hostVeth":"v"}},"binding":{"input":"{}"}}`},
		{"unknown operation", `{"apiVersion":"sandbox.fast.io/actions/v1","operation":"EXPLODE","sandbox":{"uid":"u"},"revision":{"runtimeInstanceId":"r","attachmentId":"a"}}`},
		{"missing uid", `{"apiVersion":"sandbox.fast.io/actions/v1","operation":"SET_BINDING","revision":{"runtimeInstanceId":"r","attachmentId":"a"}}`},
		{"set_binding without binding", `{"apiVersion":"sandbox.fast.io/actions/v1","operation":"SET_BINDING","sandbox":{"uid":"u"},"revision":{"runtimeInstanceId":"r","attachmentId":"a"}}`},
		{"set_binding non-string input", `{"apiVersion":"sandbox.fast.io/actions/v1","operation":"SET_BINDING","sandbox":{"uid":"u"},"revision":{"runtimeInstanceId":"r","attachmentId":"a"},"attachment":{"network":{"ip":"10.0.0.5","gateway":"10.0.0.1","hostVeth":"v"}},"binding":{"input":42}}`},
		{"set_binding without attachment", `{"apiVersion":"sandbox.fast.io/actions/v1","operation":"SET_BINDING","sandbox":{"uid":"u"},"revision":{"runtimeInstanceId":"r","attachmentId":"a"},"binding":{"input":"{}"}}`},
		{"hook without hook", `{"apiVersion":"sandbox.fast.io/actions/v1","operation":"LIFECYCLE_HOOK","sandbox":{"uid":"u"},"revision":{"runtimeInstanceId":"r","attachmentId":"a"}}`},
		{"unknown hook", `{"apiVersion":"sandbox.fast.io/actions/v1","operation":"LIFECYCLE_HOOK","sandbox":{"uid":"u"},"revision":{"runtimeInstanceId":"r","attachmentId":"a"},"hook":{"name":"sandbox.before-delete"}}`},
		{"missing fence", `{"apiVersion":"sandbox.fast.io/actions/v1","operation":"SET_BINDING","sandbox":{"uid":"u"},"attachment":{"network":{"ip":"10.0.0.5","gateway":"10.0.0.1","hostVeth":"v"}},"binding":{"input":"{}"}}`},
		{"not json", `{`},
	}
	for _, tc := range cases {
		t.Run(tc.name, func(t *testing.T) {
			rec := doAction(t, srv, tc.body)
			require.Equal(t, http.StatusBadRequest, rec.Code)
		})
	}
}

// ---------------------------------------------------------------------------
// Superseded proxy-route surfaces (removed per OSEP-0022 revision)
// ---------------------------------------------------------------------------

// TestFastSandboxServerSupersededRoutesGone: the UID-header-routed /policy and
// /credential-vault surfaces are removed — policy rides the SET_BINDING
// binding input at create and the relay surface /policy/{sandboxUid} at
// runtime, and the vault moves to the relay-only /vault/{sandboxUid} surface.
// The bare /policy path redirects into the relay subtree; /credential-vault
// must stay gone.
func TestFastSandboxServerSupersededRoutesGone(t *testing.T) {
	srv, _, _ := fastSandboxTestServer(t)
	// The bare /policy path redirects into the relay subtree; the old
	// UID-header surface stays gone. The exact 3xx code varies by method
	// (and Go version) — assert the redirect class and the Location target.
	for _, tc := range []struct {
		method, path string
	}{
		{http.MethodGet, "/policy"},
		{http.MethodPut, "/policy"},
		{http.MethodPost, "/policy"},
		{http.MethodDelete, "/policy"},
	} {
		rec := doRequest(t, srv, tc.method, tc.path, `{}`)
		require.True(t, rec.Code >= 300 && rec.Code < 400, "%s %s: want a redirect, got %d", tc.method, tc.path, rec.Code)
		assert.Equal(t, "/policy/", rec.Header().Get("Location"), "%s %s", tc.method, tc.path)
	}
	for _, tc := range []struct {
		method, path string
	}{
		{http.MethodGet, "/credential-vault"},
		{http.MethodPost, "/credential-vault"},
		{http.MethodPatch, "/credential-vault"},
		{http.MethodDelete, "/credential-vault"},
	} {
		rec := doRequest(t, srv, tc.method, tc.path, `{}`)
		require.Equal(t, http.StatusNotFound, rec.Code, "%s %s must be gone", tc.method, tc.path)
	}
}

func TestFastSandboxServerHealthz(t *testing.T) {
	srv, _, _ := fastSandboxTestServer(t)
	rec := doRequest(t, srv, http.MethodGet, "/healthz", "")
	require.Equal(t, http.StatusOK, rec.Code)
}

// ---------------------------------------------------------------------------
// MITM interception redirects
// ---------------------------------------------------------------------------

type fakeMitmInstaller struct {
	mu      sync.Mutex
	entries []iptables.MitmRedirectEntry
	err     error
	removed bool
}

func (f *fakeMitmInstaller) install(entries []iptables.MitmRedirectEntry) error {
	f.mu.Lock()
	defer f.mu.Unlock()
	if f.err != nil {
		return f.err
	}
	f.entries = append([]iptables.MitmRedirectEntry(nil), entries...)
	return nil
}

func (f *fakeMitmInstaller) remove() error {
	f.mu.Lock()
	defer f.mu.Unlock()
	f.removed = true
	f.entries = nil
	return nil
}

func (f *fakeMitmInstaller) snapshot() []iptables.MitmRedirectEntry {
	f.mu.Lock()
	defer f.mu.Unlock()
	return append([]iptables.MitmRedirectEntry(nil), f.entries...)
}

func mitmAtt(t *testing.T, ip, gw string) actionhandler.NetworkAttachment {
	t.Helper()
	return actionhandler.NetworkAttachment{
		IP:          netip.MustParseAddr(ip),
		Gateway:     netip.MustParseAddr(gw),
		PrivateCIDR: netip.MustParsePrefix("10.0.0.0/24"),
		HostVeth:    "v",
	}
}

func TestFastSandboxServerMitmRedirectInstalledOnRegistration(t *testing.T) {
	srv, _, _ := fastSandboxTestServer(t)
	inst := &fakeMitmInstaller{}
	srv.SetMitm(nil, 18081, []int{80, 443})
	srv.mitmInstall = inst.install
	srv.mitmRemove = inst.remove

	s := subject.FromSandboxUID("u-1")
	require.NoError(t, srv.OnRegistered(s, mitmAtt(t, "10.0.0.5", "10.0.0.1")))
	require.Equal(t, []iptables.MitmRedirectEntry{{SandboxIP: netip.MustParseAddr("10.0.0.5"), Gateway: netip.MustParseAddr("10.0.0.1")}}, inst.snapshot())

	// a second subject rebuilds with both entries
	s2 := subject.FromSandboxUID("u-2")
	require.NoError(t, srv.OnRegistered(s2, mitmAtt(t, "10.0.0.6", "10.0.0.1")))
	require.ElementsMatch(t, []iptables.MitmRedirectEntry{
		{SandboxIP: netip.MustParseAddr("10.0.0.5"), Gateway: netip.MustParseAddr("10.0.0.1")},
		{SandboxIP: netip.MustParseAddr("10.0.0.6"), Gateway: netip.MustParseAddr("10.0.0.1")},
	}, inst.snapshot())

	// unload rebuilds without the subject
	require.NoError(t, srv.OnUnloaded(s, mitmAtt(t, "10.0.0.5", "10.0.0.1")))
	require.Equal(t, []iptables.MitmRedirectEntry{{SandboxIP: netip.MustParseAddr("10.0.0.6"), Gateway: netip.MustParseAddr("10.0.0.1")}}, inst.snapshot())
}

func TestFastSandboxServerMitmRedirectFailClosesRegistration(t *testing.T) {
	srv, _, nft := fastSandboxTestServer(t)
	inst := &fakeMitmInstaller{err: fmt.Errorf("nft unavailable")}
	srv.SetMitm(nil, 18081, []int{80, 443})
	srv.mitmInstall = inst.install

	s := subject.FromSandboxUID("u-1")
	require.Error(t, srv.OnRegistered(s, mitmAtt(t, "10.0.0.5", "10.0.0.1")))
	// the entry was rolled back: nothing left for the next rebuild
	srv.mitmMu.Lock()
	require.Len(t, srv.mitmEntries, 0)
	srv.mitmMu.Unlock()
	// the subject stays denying (deny-first ran before the failed redirect)
	require.Len(t, nft.denyFirst, 1)
}

func TestFastSandboxServerMitmRedirectCrossFamilyRejected(t *testing.T) {
	srv, _, _ := fastSandboxTestServer(t)
	inst := &fakeMitmInstaller{}
	srv.SetMitm(nil, 18081, []int{80, 443})
	srv.mitmInstall = inst.install

	s := subject.FromSandboxUID("u-1")
	att := mitmAtt(t, "10.0.0.5", "10.0.0.1")
	// v6 sandbox IP with a v4 gateway: an illegal nft expression that would
	// abort the whole transactional rebuild — must be rejected up front
	att.IP = netip.MustParseAddr("fd00::5")
	require.Error(t, srv.OnRegistered(s, att))
	require.Nil(t, inst.snapshot())
	// invalid gateway likewise
	att = mitmAtt(t, "10.0.0.5", "10.0.0.1")
	att.Gateway = netip.Addr{}
	require.Error(t, srv.OnRegistered(s, att))
	require.Nil(t, inst.snapshot())
}

func TestFastSandboxServerMitmDisabledSkipsInterception(t *testing.T) {
	srv, _, _ := fastSandboxTestServer(t)
	inst := &fakeMitmInstaller{}
	// SetMitm never called with a gate: hooks must not install anything
	srv.mitmInstall = inst.install

	s := subject.FromSandboxUID("u-1")
	require.NoError(t, srv.OnRegistered(s, mitmAtt(t, "10.0.0.5", "10.0.0.1")))
	require.NoError(t, srv.OnUnloaded(s, mitmAtt(t, "10.0.0.5", "10.0.0.1")))
	require.Nil(t, inst.snapshot())
}

func TestFastSandboxServerHealthzMitmGate(t *testing.T) {
	t.Setenv(constants.EnvMitmproxyTransparent, "true")
	srv, _, _ := fastSandboxTestServer(t)
	gate := mitmproxy.NewHealthGate()
	srv.mitmGate = gate

	gate.SetReady(false)
	rec := doRequest(t, srv, http.MethodGet, "/healthz", "")
	require.Equal(t, http.StatusServiceUnavailable, rec.Code)
	require.Contains(t, rec.Body.String(), "mitmproxy-not-ready")

	gate.SetReady(true)
	rec = doRequest(t, srv, http.MethodGet, "/healthz", "")
	require.Equal(t, http.StatusOK, rec.Code)
}

func TestFastSandboxServerHealthzAddonDown(t *testing.T) {
	t.Setenv(constants.EnvMitmproxyTransparent, "true")
	srv, _, _ := fastSandboxTestServer(t)
	gate := mitmproxy.NewHealthGate()
	gate.SetCAFingerprint("abc123")
	gate.MarkDown("pinned CA unavailable: missing seed")
	// Addon-down assembly: gate wired, no interceptors (port <= 0).
	srv.SetMitm(gate, 0, nil)
	require.Nil(t, srv.mitmInstall, "addon-down must not wire interception installers")

	rec := doRequest(t, srv, http.MethodGet, "/healthz", "")
	require.Equal(t, http.StatusServiceUnavailable, rec.Code)
	require.Contains(t, rec.Body.String(), `"status":"mitm-addon-down"`)
	require.Contains(t, rec.Body.String(), `"mitmCaDown":true`)
	require.Contains(t, rec.Body.String(), "pinned CA unavailable")

	rec = doRequest(t, srv, http.MethodGet, constants.ActionsStatusPath, "")
	require.Equal(t, http.StatusOK, rec.Code)
	require.Contains(t, rec.Body.String(), `"ready":false`)
	require.Contains(t, rec.Body.String(), `"mitmCaDown":true`)
	require.Contains(t, rec.Body.String(), "pinned CA unavailable")
}

func TestFastSandboxServerStatusPinnedCaObservability(t *testing.T) {
	srv, _, _ := fastSandboxTestServer(t)
	gate := mitmproxy.NewHealthGate()
	gate.SetCAFingerprint("deadbeef")
	srv.SetMitm(gate, 18081, []int{80, 443})

	rec := doRequest(t, srv, http.MethodGet, constants.ActionsStatusPath, "")
	require.Equal(t, http.StatusOK, rec.Code)
	require.Contains(t, rec.Body.String(), `"ready":true`)
	require.Contains(t, rec.Body.String(), `"mitmCaFingerprint":"deadbeef"`)
	require.NotContains(t, rec.Body.String(), "mitmCaDown")

	// MITM disabled (nil gate): no CA fields at all.
	srv2, _, _ := fastSandboxTestServer(t)
	rec = doRequest(t, srv2, http.MethodGet, constants.ActionsStatusPath, "")
	require.Equal(t, http.StatusOK, rec.Code)
	require.NotContains(t, rec.Body.String(), "mitmCa")
}

// ---------------------------------------------------------------------------
// Subject-aware active vault API
// ---------------------------------------------------------------------------

func TestFastSandboxServerActiveVaultClientIPDispatch(t *testing.T) {
	srv, reg, _ := fastSandboxTestServer(t)
	sA := subject.FromSandboxUID("a")
	sB := subject.FromSandboxUID("b")
	reg.Register(sA, subject.SubjectKey{SourceIP: netip.MustParseAddr("10.0.0.5")}, subject.Fencing{RuntimeInstanceID: "r-a", AttachmentID: "a-a"})
	reg.Register(sB, subject.SubjectKey{SourceIP: netip.MustParseAddr("10.0.0.6")}, subject.Fencing{RuntimeInstanceID: "r-b", AttachmentID: "a-b"})

	// activate subject A (bindings require an effective policy that allows
	// the bound host)
	rec := doAction(t, srv, bindBody(t, "a", "10.0.0.5", "r-a", "a-a", 1,
		strPtr(`{"defaultAction":"deny","egress":[{"action":"allow","target":"example.com"}]}`)))
	require.Equal(t, http.StatusOK, rec.Code)
	rec = doAction(t, srv, hookBody(t, "a", "r-a", "a-a", constants.HookDataPlaneReady))
	require.Equal(t, http.StatusOK, rec.Code)

	// Populate subject A's vault through the store API (the relay-only
	// surface that owns vault writes is tracked separately in OSEP-0022).
	vault := srv.vaultFor(sA)
	_, err := vault.Create(credentialvault.CreateRequest{
		Credentials: []credentialvault.Credential{
			{Name: "k", Source: json.RawMessage(`{"type":"inline","value":"v"}`)},
		},
		Bindings: []credentialvault.Binding{
			{Name: "b", Match: credentialvault.Match{Hosts: []string{"example.com"}}, Auth: credentialvault.Auth{Type: "apiKey", Name: "X-API-Key", Credential: "k"}},
		},
	}, reg.EffectivePolicy(sA))
	require.NoError(t, err)

	// client IP -> subject A vault (the active endpoint is served on the
	// unix socket; here the handler is exercised directly)
	rec = httptest.NewRecorder()
	srv.handleCredentialVaultActive(rec, httptest.NewRequest(http.MethodGet, "/credential-vault/_active?clientIp=10.0.0.5", nil))
	require.Equal(t, http.StatusOK, rec.Code)
	require.Contains(t, rec.Body.String(), `"revision":1`)
	initialTag := rec.Header().Get("ETag")
	require.NotEmpty(t, initialTag)

	// A conditional lookup for the active snapshot tag avoids rendering and
	// retransmitting the subject's credentials.
	req := httptest.NewRequest(http.MethodGet, "/credential-vault/_active?clientIp=10.0.0.5", nil)
	req.Header.Set("If-None-Match", initialTag)
	rec = httptest.NewRecorder()
	srv.handleCredentialVaultActive(rec, req)
	require.Equal(t, http.StatusNotModified, rec.Code)
	require.Empty(t, rec.Body.String())
	require.Equal(t, initialTag, rec.Header().Get("ETag"))

	// subject B has no vault: 404 (the addon treats it as no-vault)
	rec = httptest.NewRecorder()
	srv.handleCredentialVaultActive(rec, httptest.NewRequest(http.MethodGet, "/credential-vault/_active?clientIp=10.0.0.6", nil))
	require.Equal(t, http.StatusNotFound, rec.Code)

	// unknown IP: 404
	rec = httptest.NewRecorder()
	srv.handleCredentialVaultActive(rec, httptest.NewRequest(http.MethodGet, "/credential-vault/_active?clientIp=10.0.0.99", nil))
	require.Equal(t, http.StatusNotFound, rec.Code)

	// missing / malformed clientIp: 400
	rec = httptest.NewRecorder()
	srv.handleCredentialVaultActive(rec, httptest.NewRequest(http.MethodGet, "/credential-vault/_active", nil))
	require.Equal(t, http.StatusBadRequest, rec.Code)
	rec = httptest.NewRecorder()
	srv.handleCredentialVaultActive(rec, httptest.NewRequest(http.MethodGet, "/credential-vault/_active?clientIp=not-an-ip", nil))
	require.Equal(t, http.StatusBadRequest, rec.Code)

	// the HTTP mux must NOT expose the active endpoint (sidecar parity: the
	// addon is the only client, over the unix socket)
	rec = doRequest(t, srv, http.MethodGet, "/credential-vault/_active?clientIp=10.0.0.5", "")
	require.Equal(t, http.StatusNotFound, rec.Code)
}

// TestActionsCreateThenConfigureEndToEnd exercises the actions-driven
// create-then-configure spine: SET_BINDING installs deny-first with the
// binding policy held pending, data-plane-ready activates it, and
// REMOVE_BINDING tears everything down.
func TestActionsCreateThenConfigureEndToEnd(t *testing.T) {
	srv, reg, nft := fastSandboxTestServer(t)
	uid := "e2e-1"
	s := subject.FromSandboxUID(uid)

	// SET_BINDING with the policy: deny-first, policy pending
	rec := doAction(t, srv, bindBody(t, uid, testIP, testFenceR, testFenceA, 1, strPtr(`{"defaultAction":"deny","egress":[{"action":"allow","target":"example.com"}]}`)))
	require.Equal(t, http.StatusOK, rec.Code)
	require.Equal(t, subject.StateDenying, mustState(reg.Get(s)))
	assert.Nil(t, reg.EffectivePolicy(s), "fail-closed: DNS denies while the policy is pending")

	// data-plane-ready activates the policy
	rec = doAction(t, srv, hookBody(t, uid, testFenceR, testFenceA, constants.HookDataPlaneReady))
	require.Equal(t, http.StatusOK, rec.Code)
	require.Equal(t, subject.StateActive, mustState(reg.Get(s)))
	assert.Equal(t, "allow", reg.EffectivePolicy(s).Evaluate("example.com"))
	require.Equal(t, 1, nft.appliedCount())

	// REMOVE_BINDING tears everything down
	rec = doAction(t, srv, removeBody(t, uid, testFenceR, testFenceA))
	require.Equal(t, http.StatusOK, rec.Code)
	_, ok := reg.Get(s)
	assert.False(t, ok)
	require.Len(t, nft.removed, 1)
}

// TestFastSandboxServerAlwaysRulesReachNft: allow.always/deny.always must be
// enforced at the IP layer too, not only in DNS dispatch. The nft swap must
// receive the always-merged effective policy.
func TestFastSandboxServerAlwaysRulesReachNft(t *testing.T) {
	alwaysDeny, err := policy.ParseValidatedEgressRule("deny", "203.0.113.0/24")
	require.NoError(t, err)
	alwaysAllow, err := policy.ParseValidatedEgressRule("allow", "198.51.100.7")
	require.NoError(t, err)

	reg := subject.NewRegistry([]policy.EgressRule{alwaysDeny}, []policy.EgressRule{alwaysAllow})
	nft := &fakeNft{}
	srv := newFastSandboxPolicyServer(context.Background(), reg, nft)
	wireTestPolicyPersistence(t, srv)
	srv.dnsRedirectApply = func(context.Context, []netip.Addr, int) error { return nil }

	// activation through the action protocol; the binding policy reaches nft
	// merged with the always overlay
	rec := doAction(t, srv, bindBody(t, "u-1", testIP, testFenceR, testFenceA, 1, strPtr(`{"defaultAction":"deny","egress":[{"action":"allow","target":"example.com"}]}`)))
	require.Equal(t, http.StatusOK, rec.Code)
	rec = doAction(t, srv, hookBody(t, "u-1", testFenceR, testFenceA, constants.HookDataPlaneReady))
	require.Equal(t, http.StatusOK, rec.Code)

	nft.mu.Lock()
	applied := nft.lastPolicy
	nft.mu.Unlock()
	require.NotNil(t, applied, "nft must receive the merged policy")
	allowV4, _, denyV4, _ := applied.StaticIPSets()
	require.Contains(t, denyV4, "203.0.113.0/24", "always-deny CIDR must reach the nft deny set")
	require.Contains(t, allowV4, "198.51.100.7", "always-allow IP must reach the nft allow set")
}

// TestFastSandboxServerNftFailureKeepsRegistryState: a failed nft apply must leave
// the registry (DNS dispatch) on the PREVIOUS policy — nft commits before registry
// state, and the relay write NACKs with last-good kept on both sides.
func TestFastSandboxServerNftFailureKeepsRegistryState(t *testing.T) {
	srv, reg, nft := fastSandboxTestServer(t)
	input := `{"defaultAction":"deny"}`
	s := setBindingAndReady(t, srv, reg, "u-1", testIP, &input)
	require.Equal(t, subject.StateActive, mustState(reg.Get(s)))

	// relay write fails at nft: 500, the registry must stay on the FIRST
	// policy (the record was seeded at revision 1; the write targeted
	// revision 2)
	nft.mu.Lock()
	nft.policyErr = errors.New("nft busy")
	nft.mu.Unlock()
	rec := doPolicy(t, srv, http.MethodPut, "u-1", policyPutDoc(
		`{"defaultAction":"deny","egress":[{"action":"allow","target":"example.com"}]}`, 1, false, true), "")
	require.Equal(t, http.StatusInternalServerError, rec.Code)

	eff := reg.EffectivePolicy(s)
	require.NotNil(t, eff)
	assert.Equal(t, "deny", eff.Evaluate("example.com"), "failed nft apply must not publish the new policy")

	// a same-fence binding replay never carries the new policy either
	rec = doAction(t, srv, bindBody(t, "u-1", testIP, testFenceR, testFenceA, 2,
		strPtr(`{"defaultAction":"deny","egress":[{"action":"allow","target":"example.com"}]}`)))
	require.Equal(t, http.StatusOK, rec.Code)
	eff = reg.EffectivePolicy(s)
	require.NotNil(t, eff)
	assert.Equal(t, "deny", eff.Evaluate("example.com"), "the recorded policy is authoritative over binding replays")

	// the retry with a working nft lands
	nft.mu.Lock()
	nft.policyErr = nil
	nft.mu.Unlock()
	rec = doPolicy(t, srv, http.MethodPut, "u-1", policyPutDoc(
		`{"defaultAction":"deny","egress":[{"action":"allow","target":"example.com"}]}`, 1, false, true), "")
	require.Equal(t, http.StatusOK, rec.Code)
	assert.Equal(t, "allow", reg.EffectivePolicy(s).Evaluate("example.com"))
}

// TestFastSandboxServerGatewayRedirectRefcounted: the shared prerouting table
// keeps every active gateway and removes only gateways without owners.
func TestFastSandboxServerGatewayRedirectRefcounted(t *testing.T) {
	reg := subject.NewRegistry(nil, nil)
	nft := &fakeNft{}
	srv := newFastSandboxPolicyServer(context.Background(), reg, nft)
	wireTestPolicyPersistence(t, srv)
	dns := newFakeGatewayDNSRedirectApplier()
	srv.dnsRedirectApply = dns.apply

	attA := mitmAtt(t, "10.0.0.5", "10.10.0.1")
	attB := mitmAtt(t, "10.0.0.6", "10.10.0.1")
	attC := mitmAtt(t, "10.0.0.7", "10.20.0.1")

	require.NoError(t, srv.OnRegistered(subject.FromSandboxUID("a"), attA))
	require.NoError(t, srv.OnRegistered(subject.FromSandboxUID("b"), attB))
	require.NoError(t, srv.OnRegistered(subject.FromSandboxUID("c"), attC))
	wantBoth := map[netip.Addr]struct{}{
		netip.MustParseAddr("10.10.0.1"): {},
		netip.MustParseAddr("10.20.0.1"): {},
	}
	assert.Equal(t, wantBoth, dns.snapshot())

	require.NoError(t, srv.OnUnloaded(subject.FromSandboxUID("a"), attA))
	assert.Equal(t, wantBoth, dns.snapshot(), "a non-final shared-gateway release preserves the whole table")
	require.NoError(t, srv.OnUnloaded(subject.FromSandboxUID("b"), attB))
	assert.Equal(t, map[netip.Addr]struct{}{netip.MustParseAddr("10.20.0.1"): {}}, dns.snapshot(), "removing gw1 must preserve gw2")

	require.NoError(t, srv.OnUnloaded(subject.FromSandboxUID("c"), attC))
	assert.Empty(t, dns.snapshot(), "the final release removes the DNS table contents")
}

func TestFastSandboxServerGatewayRedirectUsesLifecycleDeadline(t *testing.T) {
	srv, _, _ := fastSandboxTestServer(t)
	var contexts []context.Context
	srv.dnsRedirectApply = func(ctx context.Context, _ []netip.Addr, _ int) error {
		contexts = append(contexts, ctx)
		return nil
	}
	subj := subject.FromSandboxUID("dns-context")
	att := mitmAtt(t, "10.0.0.5", "10.10.0.1")

	require.NoError(t, srv.OnRegistered(subj, att))
	require.NoError(t, srv.OnUnloaded(subj, att))
	require.Len(t, contexts, 2)
	for i, maxTimeout := range []time.Duration{policyApplyTimeout, 30 * time.Second} {
		deadline, ok := contexts[i].Deadline()
		require.True(t, ok, "lifecycle operation %d must have a deadline", i)
		remaining := time.Until(deadline)
		assert.Positive(t, remaining, "lifecycle context should still be within its deadline")
		assert.LessOrEqual(t, remaining, maxTimeout, "lifecycle context must preserve its existing timeout")
	}
}

func TestFastSandboxServerReleaseCleansUncommittedInstallAfterLostResult(t *testing.T) {
	srv, _, _ := fastSandboxTestServer(t)
	dns := newFakeGatewayDNSRedirectApplier()
	srv.dnsRedirectApply = dns.apply
	subj, gw := subject.FromSandboxUID("uncommitted"), netip.MustParseAddr("10.10.0.1")
	dns.failOnCall(1, errors.New("result lost"), true)
	require.Error(t, srv.installGatewayDNSRedirect(context.Background(), subj, gw))
	srv.gwMu.Lock()
	assert.Empty(t, srv.gatewayDNSRefs)
	srv.gwMu.Unlock()
	assert.Equal(t, map[netip.Addr]struct{}{gw: {}}, dns.snapshot())

	require.NoError(t, srv.releaseGatewayDNSRedirect(context.Background(), subj))
	assert.Empty(t, dns.snapshot(), "release reconciles even though install ownership was not committed")
}

func TestActionsSetAndRemoveRetryAfterGatewayDNSFailures(t *testing.T) {
	srv, reg, _ := fastSandboxTestServer(t)
	dns := newFakeGatewayDNSRedirectApplier()
	srv.dnsRedirectApply = dns.apply
	input := `{"defaultAction":"deny"}`
	subj := subject.FromSandboxUID("dns-remove")
	dns.failOnCall(1, errors.New("nft busy"), false)
	rec := doAction(t, srv, bindBody(t, "dns-remove", testIP, testFenceR, testFenceA, 1, strPtr(input)))
	require.Equal(t, http.StatusInternalServerError, rec.Code)
	assert.Equal(t, subject.StateDenying, mustState(reg.Get(subj)))
	assert.Empty(t, dns.snapshot())
	srv.gwMu.Lock()
	assert.Empty(t, srv.gatewayDNSRefs, "failed install does not commit ownership")
	srv.gwMu.Unlock()

	dns.clearFailures()
	rec = doAction(t, srv, bindBody(t, "dns-remove", testIP, testFenceR, testFenceA, 1, strPtr(input)))
	require.Equal(t, http.StatusOK, rec.Code)
	assert.Equal(t, map[netip.Addr]struct{}{netip.MustParseAddr(testGW): {}}, dns.snapshot())
	dns.failOnCall(3, errors.New("nft result lost"), true)

	rec = doAction(t, srv, removeBody(t, "dns-remove", testFenceR, testFenceA))
	require.Equal(t, http.StatusInternalServerError, rec.Code)
	state, ok := reg.Get(subj)
	require.True(t, ok, "REMOVE_BINDING must not unregister after a failed DNS reconcile")
	assert.Equal(t, subject.StateDenying, state)
	srv.gwMu.Lock()
	assert.Equal(t, netip.MustParseAddr(testGW), srv.gatewayDNSRefs[subj], "failed release retains retry ownership")
	srv.gwMu.Unlock()
	assert.Empty(t, dns.snapshot(), "the apply-then-error fake makes the kernel result ambiguous")

	dns.clearFailures()
	rec = doAction(t, srv, removeBody(t, "dns-remove", testFenceR, testFenceA))
	require.Equal(t, http.StatusOK, rec.Code)
	_, ok = reg.Get(subj)
	assert.False(t, ok)
	assert.Empty(t, dns.snapshot())
	srv.gwMu.Lock()
	assert.Empty(t, srv.gatewayDNSRefs)
	srv.gwMu.Unlock()
}

func TestActionsSetBindingTakeoverRetriesAfterGatewayDNSFailure(t *testing.T) {
	srv, reg, _ := fastSandboxTestServer(t)
	dns := newFakeGatewayDNSRedirectApplier()
	srv.dnsRedirectApply = dns.apply
	staleUID, successorUID := "dns-stale", "dns-successor"
	stale, successor := subject.FromSandboxUID(staleUID), subject.FromSandboxUID(successorUID)
	input := `{"defaultAction":"deny"}`
	setBindingAndReady(t, srv, reg, staleUID, testIP, &input)
	dns.failOnCall(2, errors.New("nft result lost"), true)

	rec := doAction(t, srv, bindBody(t, successorUID, testIP, "runtime-2", "attachment-2", 1, strPtr(input)))
	require.Equal(t, http.StatusInternalServerError, rec.Code)
	require.Equal(t, subject.StateActive, mustState(reg.Get(stale)), "failed DNS release must not evict the old owner")
	_, ok := reg.Get(successor)
	assert.False(t, ok, "takeover must not publish its successor before DNS cleanup succeeds")
	srv.gwMu.Lock()
	assert.Equal(t, netip.MustParseAddr(testGW), srv.gatewayDNSRefs[stale])
	srv.gwMu.Unlock()

	dns.clearFailures()
	rec = doAction(t, srv, bindBody(t, successorUID, testIP, "runtime-2", "attachment-2", 1, strPtr(input)))
	require.Equal(t, http.StatusOK, rec.Code)
	_, ok = reg.Get(stale)
	assert.False(t, ok)
	require.Equal(t, subject.StateDenying, mustState(reg.Get(successor)))
	assert.Equal(t, map[netip.Addr]struct{}{netip.MustParseAddr(testGW): {}}, dns.snapshot())
}

func TestActionsMITMFailureRollbackAndTerminalCleanup(t *testing.T) {
	srv, reg, _ := fastSandboxTestServer(t)
	dns := newFakeGatewayDNSRedirectApplier()
	srv.dnsRedirectApply = dns.apply
	srv.mitmInstall = func([]iptables.MitmRedirectEntry) error { return errors.New("mitm failed") }
	input := `{"defaultAction":"deny"}`

	rec := doAction(t, srv, bindBody(t, "mitm-dns", testIP, testFenceR, testFenceA, 1, strPtr(input)))
	require.Equal(t, http.StatusInternalServerError, rec.Code)
	assert.Contains(t, rec.Body.String(), "mitm failed")
	subj := subject.FromSandboxUID("mitm-dns")
	state, ok := reg.Get(subj)
	require.True(t, ok, "failed registration remains denying and retryable")
	assert.Equal(t, subject.StateDenying, state)
	srv.gwMu.Lock()
	assert.Empty(t, srv.gatewayDNSRefs, "successful rollback commits the empty ownership map")
	srv.gwMu.Unlock()
	assert.Empty(t, dns.snapshot(), "successful rollback removes the newly installed redirect")

	// Retry the same deny-first registration; this time rollback applies but
	// returns an error, so ownership remains available to terminal cleanup.
	dns.failOnCall(4, errors.New("DNS rollback failed"), true)
	rec = doAction(t, srv, bindBody(t, "mitm-dns", testIP, testFenceR, testFenceA, 1, strPtr(input)))
	require.Equal(t, http.StatusInternalServerError, rec.Code)
	assert.Contains(t, rec.Body.String(), "mitm failed")
	assert.Contains(t, rec.Body.String(), "DNS rollback failed")
	srv.gwMu.Lock()
	assert.Equal(t, netip.MustParseAddr(testGW), srv.gatewayDNSRefs[subj], "failed rollback retains ownership")
	srv.gwMu.Unlock()
	assert.Empty(t, dns.snapshot(), "the rollback result is ambiguous")

	dns.clearFailures()
	rec = doAction(t, srv, removeBody(t, "mitm-dns", testFenceR, testFenceA))
	require.Equal(t, http.StatusOK, rec.Code)
	_, ok = reg.Get(subj)
	assert.False(t, ok)
	assert.Empty(t, dns.snapshot())
	srv.gwMu.Lock()
	assert.Empty(t, srv.gatewayDNSRefs)
	srv.gwMu.Unlock()
}

func strPtr(s string) *string { return &s }

func mustState(st subject.State, ok bool) subject.State {
	if !ok {
		panic("subject absent")
	}
	return st
}

// ---------------------------------------------------------------------------
// Review fixes: lifecycle-barrier retry semantics (PR #1678 comments)
// ---------------------------------------------------------------------------

// TestActionsDataPlaneReadyRetriesAfterNftFailure (review fix): a transient
// nft failure on data-plane-ready must NOT consume the pending policy — the
// Fastlet's retry of the same Hook succeeds instead of 409-ing forever with
// the subject permanently deny-first.
func TestActionsDataPlaneReadyRetriesAfterNftFailure(t *testing.T) {
	srv, reg, nft := fastSandboxTestServer(t)
	s := subject.FromSandboxUID("u-1")
	input := `{"defaultAction":"deny","egress":[{"action":"allow","target":"example.com"}]}`
	rec := doAction(t, srv, bindBody(t, "u-1", testIP, testFenceR, testFenceA, 1, &input))
	require.Equal(t, http.StatusOK, rec.Code)
	require.Equal(t, subject.StateDenying, mustState(reg.Get(s)))

	// first data-plane-ready fails at nft: 500, pending policy retained
	nft.mu.Lock()
	nft.policyErr = errors.New("nft busy")
	nft.mu.Unlock()
	rec = doAction(t, srv, hookBody(t, "u-1", testFenceR, testFenceA, constants.HookDataPlaneReady))
	require.Equal(t, http.StatusInternalServerError, rec.Code)
	require.Equal(t, subject.StateDenying, mustState(reg.Get(s)))

	// retry of the SAME Hook succeeds with the retained pending policy
	nft.mu.Lock()
	nft.policyErr = nil
	nft.mu.Unlock()
	rec = doAction(t, srv, hookBody(t, "u-1", testFenceR, testFenceA, constants.HookDataPlaneReady))
	require.Equal(t, http.StatusOK, rec.Code)
	require.Equal(t, subject.StateActive, mustState(reg.Get(s)))
	require.Equal(t, 1, nft.appliedCount())
	assert.Equal(t, "allow", reg.EffectivePolicy(s).Evaluate("example.com"))
}

// TestActionsStaleHookFenceRejected (review fix): a delayed data-plane-ready
// from a previous instance must not consume the replacement sandbox's pending
// policy or activate it before its own data plane is ready.
func TestActionsStaleHookFenceRejected(t *testing.T) {
	srv, reg, nft := fastSandboxTestServer(t)
	s := subject.FromSandboxUID("u-1")
	input := `{"defaultAction":"deny","egress":[{"action":"allow","target":"new.com"}]}`
	rec := doAction(t, srv, bindBody(t, "u-1", testIP, "runtime-2", "attachment-2", 1, &input))
	require.Equal(t, http.StatusOK, rec.Code)
	require.Equal(t, subject.StateDenying, mustState(reg.Get(s)))

	// stale Hook (old runtime identity): 409, pending policy untouched, the
	// subject stays deny-first
	rec = doAction(t, srv, hookBody(t, "u-1", "runtime-old", "attachment-old", constants.HookDataPlaneReady))
	require.Equal(t, http.StatusConflict, rec.Code)
	require.Equal(t, subject.StateDenying, mustState(reg.Get(s)))
	assert.Equal(t, 0, nft.appliedCount())

	// the genuine Hook (current fence) succeeds
	rec = doAction(t, srv, hookBody(t, "u-1", "runtime-2", "attachment-2", constants.HookDataPlaneReady))
	require.Equal(t, http.StatusOK, rec.Code)
	require.Equal(t, subject.StateActive, mustState(reg.Get(s)))
	assert.Equal(t, "allow", reg.EffectivePolicy(s).Evaluate("new.com"))
}

// TestActionsRemoveBindingRetriesAfterNftFailure (review fix): a transient
// nft removal failure must keep the subject registered so the retried
// REMOVE_BINDING resumes cleanup — otherwise it would succeed with stale
// rules left for a possibly-reused IP.
func TestActionsRemoveBindingRetriesAfterNftFailure(t *testing.T) {
	srv, reg, nft := fastSandboxTestServer(t)
	input := `{"defaultAction":"deny"}`
	s := setBindingAndReady(t, srv, reg, "u-1", testIP, &input)

	// first REMOVE_BINDING fails at nft: 500, the subject STAYS registered
	nft.mu.Lock()
	nft.removeErr = errors.New("nft busy")
	nft.mu.Unlock()
	rec := doAction(t, srv, removeBody(t, "u-1", testFenceR, testFenceA))
	require.Equal(t, http.StatusInternalServerError, rec.Code)
	state, ok := reg.Get(s)
	require.True(t, ok, "subject must stay registered after a failed removal")
	assert.Equal(t, subject.StateActive, state)

	// retried REMOVE_BINDING resumes cleanup and completes (the fake only
	// records successful removes; the registry state below proves the retry
	// re-ran the enforcement removal instead of short-circuiting)
	nft.mu.Lock()
	nft.removeErr = nil
	nft.mu.Unlock()
	rec = doAction(t, srv, removeBody(t, "u-1", testFenceR, testFenceA))
	require.Equal(t, http.StatusOK, rec.Code)
	_, ok = reg.Get(s)
	assert.False(t, ok)
	require.Len(t, nft.removed, 1, "the retry must re-attempt and complete the nft removal")
}

// TestFastSandboxServerGatewayRedirectDuplicateSetBindingIdempotent (review fix):
// at-least-once SET_BINDING delivery must not double-count the gateway — a
// duplicate registration is a no-op, and one unload fully releases it.
func TestFastSandboxServerGatewayRedirectDuplicateSetBindingIdempotent(t *testing.T) {
	reg := subject.NewRegistry(nil, nil)
	nft := &fakeNft{}
	srv := newFastSandboxPolicyServer(context.Background(), reg, nft)
	wireTestPolicyPersistence(t, srv)
	dns := newFakeGatewayDNSRedirectApplier()
	srv.dnsRedirectApply = dns.apply

	att := mitmAtt(t, "10.0.0.5", "10.10.0.1")
	s := subject.FromSandboxUID("a")
	dns.failOnCall(1, errors.New("result lost"), true)
	require.Error(t, srv.OnRegistered(s, att))
	srv.gwMu.Lock()
	assert.Empty(t, srv.gatewayDNSRefs, "failed install does not commit the map")
	srv.gwMu.Unlock()
	assert.Equal(t, map[netip.Addr]struct{}{netip.MustParseAddr("10.10.0.1"): {}}, dns.snapshot())
	dns.clearFailures()
	require.NoError(t, srv.OnRegistered(s, att), "the same desired set must be reconciled after an ambiguous error")
	require.NoError(t, srv.OnRegistered(s, att), "duplicate SET_BINDING delivery")
	assert.Equal(t, 3, dns.callCount(), "duplicate registration still reconciles the complete desired set")
	assert.Equal(t, map[netip.Addr]struct{}{netip.MustParseAddr("10.10.0.1"): {}}, dns.snapshot())

	require.NoError(t, srv.OnUnloaded(s, att))
	assert.Empty(t, dns.snapshot(), "one unload must fully release the gateway")
}

// TestFastSandboxServerGatewayRedirectRebindMovesGateway: a rebind that moved the
// subject to a different gateway releases the old gateway's redirect.
func TestFastSandboxServerGatewayRedirectRebindMovesGateway(t *testing.T) {
	reg := subject.NewRegistry(nil, nil)
	nft := &fakeNft{}
	srv := newFastSandboxPolicyServer(context.Background(), reg, nft)
	wireTestPolicyPersistence(t, srv)
	dns := newFakeGatewayDNSRedirectApplier()
	srv.dnsRedirectApply = dns.apply

	s := subject.FromSandboxUID("a")
	att1 := mitmAtt(t, "10.0.0.5", "10.10.0.1")
	att2 := mitmAtt(t, "10.0.0.5", "10.20.0.1")
	require.NoError(t, srv.OnRegistered(s, att1))
	assert.Equal(t, map[netip.Addr]struct{}{netip.MustParseAddr("10.10.0.1"): {}}, dns.snapshot())
	dns.failOnCall(2, errors.New("replacement rejected"), false)
	require.Error(t, srv.OnRegistered(s, att2))
	srv.gwMu.Lock()
	assert.Equal(t, netip.MustParseAddr("10.10.0.1"), srv.gatewayDNSRefs[s], "failed replacement does not commit the new gateway")
	srv.gwMu.Unlock()
	assert.Equal(t, map[netip.Addr]struct{}{netip.MustParseAddr("10.10.0.1"): {}}, dns.snapshot(), "a rejected batch keeps the prior table")
	dns.clearFailures()
	require.NoError(t, srv.OnRegistered(s, att2), "rebind with a new gateway")
	assert.Equal(t, map[netip.Addr]struct{}{netip.MustParseAddr("10.20.0.1"): {}}, dns.snapshot(), "one replacement drops the old gateway")

	require.NoError(t, srv.OnUnloaded(s, att2))
	assert.Empty(t, dns.snapshot(), "the re-bound gateway must be released")
}