Skip to content

Agent Mesh

57.5 KB
Raw Blame
// Copyright 2026 Google LLC
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package meshnet

import (
	"bufio"
	"bytes"
	"context"
	"crypto/ed25519"
	"crypto/rand"
	"encoding/base64"
	"encoding/json"
	"errors"
	"fmt"
	"io"
	"net/http"
	"net/http/httptest"
	"strings"
	"sync"
	"sync/atomic"
	"testing"
	"time"

	"github.com/a2aproject/a2a-go/v2/a2a"
	"github.com/eclipse-biscuit/biscuit-go/v2"
	"github.com/google/agentmesh/api"
	"github.com/libp2p/go-libp2p"
	"github.com/libp2p/go-libp2p/core/network"
	"github.com/libp2p/go-libp2p/core/peer"
	"github.com/libp2p/go-msgio"
	"github.com/modelcontextprotocol/go-sdk/mcp"
	"github.com/prometheus/client_golang/prometheus"
	dto "github.com/prometheus/client_model/go"
	"google.golang.org/protobuf/proto"
	"google.golang.org/protobuf/types/known/timestamppb"
)

func TestAttenuateToken(t *testing.T) {
	ctx, cancel := context.WithTimeout(context.Background(), 15*time.Second)
	defer cancel()

	t.Run("rejects_before_up_and_nil_rule", func(t *testing.T) {
		var unstarted Server
		if _, _, err := unstarted.AttenuateToken(&api.TaskAuthorizationRule{Name: "t1"}, time.Minute, false); err == nil {
			t.Fatal("AttenuateToken succeeded before Up, want error")
		}
		env := newStartedTestServer(t, ctx)
		if _, _, err := env.server.AttenuateToken(nil, time.Minute, false); err == nil {
			t.Fatal("AttenuateToken(nil) succeeded, want error")
		}
	})

	t.Run("attenuates_caps_ttl_and_seals", func(t *testing.T) {
		env := newStartedTestServer(t, ctx)
		rule := &api.TaskAuthorizationRule{
			Name:        "review-pr-42",
			DisplayName: "Review PR 42",
			Rules: []*api.TaskRule{
				{
					AllowedServices: []string{"a2a://coder"},
					Operation:       &api.TaskOperation{AllowedMethods: []string{"POST"}},
				},
			},
		}

		unsealedB64, exp, err := env.server.AttenuateToken(rule, 5*time.Minute, false)
		if err != nil {
			t.Fatalf("AttenuateToken(seal=false): %v", err)
		}
		if d := time.Until(exp); d <= 0 || d > 5*time.Minute+time.Second {
			t.Errorf("attenuated expiry delta = %v, want (0, 5m]", d)
		}
		rawUnsealed, err := base64.RawURLEncoding.DecodeString(unsealedB64)
		if err != nil {
			t.Fatalf("DecodeString: %v", err)
		}
		unsealedTok, err := biscuit.Unmarshal(rawUnsealed)
		if err != nil {
			t.Fatalf("biscuit.Unmarshal(unsealed): %v", err)
		}
		codes := unsealedTok.Code()
		if len(codes) != 1 {
			t.Fatalf("len(unsealedTok.Code()) = %d, want 1 appended block", len(codes))
		}
		inner := strings.TrimSpace(strings.TrimSuffix(strings.TrimPrefix(strings.TrimSpace(codes[0]), "Block {"), "}"))
		decodedRule, err := api.ParseTARBlockSource(inner)
		if err != nil || decodedRule.GetName() != "review-pr-42" {
			t.Fatalf("ParseTARBlockSource(%q) = %v, err=%v; want review-pr-42", inner, decodedRule, err)
		}

		// A second attenuation block can be appended to an unsealed token.
		nextFact, err := api.EncodeTARBlockFact(&api.TaskAuthorizationRule{
			Name:       "narrower",
			ExpireTime: decodedRule.GetExpireTime(),
			Rules:      decodedRule.GetRules(),
		})
		if err != nil {
			t.Fatalf("EncodeTARBlockFact: %v", err)
		}
		bb := unsealedTok.CreateBlock()
		if err := bb.AddFact(nextFact); err != nil {
			t.Fatalf("AddFact: %v", err)
		}
		if _, err := unsealedTok.Append(rand.Reader, bb.Build()); err != nil {
			t.Fatalf("expected unsealed token to allow further attenuation: %v", err)
		}

		// With seal=true, downstream holders cannot append another block.
		sealedB64, _, err := env.server.AttenuateToken(rule, 5*time.Minute, true)
		if err != nil {
			t.Fatalf("AttenuateToken(seal=true): %v", err)
		}
		rawSealed, err := base64.RawURLEncoding.DecodeString(sealedB64)
		if err != nil {
			t.Fatalf("DecodeString(sealed): %v", err)
		}
		sealedTok, err := biscuit.Unmarshal(rawSealed)
		if err != nil {
			t.Fatalf("biscuit.Unmarshal(sealed): %v", err)
		}
		forgedBB := sealedTok.CreateBlock()
		if err := forgedBB.AddFact(nextFact); err != nil {
			t.Fatalf("AddFact(forged): %v", err)
		}
		if _, err := sealedTok.Append(rand.Reader, forgedBB.Build()); err == nil {
			t.Fatal("Append succeeded on a sealed token, want error")
		}
	})
}

func TestSanitizeCallerHeadersAndAgentCardRelativize(t *testing.T) {
	peerID := "12D3KooWExamplePeer"
	privRoot := "/mesh/" + peerID + "/a2a/coder"
	pubRoot := "/ingress/" + peerID + "/a2a/coder"
	roots := []string{pubRoot, privRoot, ""}

	orig := &a2a.AgentCard{
		Name:        "coder",
		Description: "Laptop coder",
		Version:     "1.0.0",
		SupportedInterfaces: []*a2a.AgentInterface{
			{URL: "https://bananas.sam-mesh.dev" + pubRoot + "/invoke?mode=fast", ProtocolBinding: a2a.TransportProtocolJSONRPC, ProtocolVersion: "1.0"},
			{URL: "http://agentmesh.local" + privRoot, ProtocolBinding: a2a.TransportProtocolHTTPJSON, ProtocolVersion: "1.0"},
			{URL: "http://127.0.0.1:9090/v1/rpc", ProtocolBinding: a2a.TransportProtocolJSONRPC, ProtocolVersion: "1.0"},
		},
		Signatures: []a2a.AgentCardSignature{{Protected: "hdr", Signature: "sig"}},
	}
	rawOrig, err := json.Marshal(orig)
	if err != nil {
		t.Fatal(err)
	}

	req := httptest.NewRequest(http.MethodGet, "http://agentmesh.local/.well-known/agent-card.json", nil)
	resp := &http.Response{
		StatusCode:    http.StatusOK,
		Header:        http.Header{"Content-Type": {"application/json"}},
		Body:          io.NopCloser(bytes.NewReader(rawOrig)),
		ContentLength: int64(len(rawOrig)),
		Request:       req,
	}
	if err := (&Server{}).relativizeAgentCardMultiRoot("coder", roots)(resp); err != nil {
		t.Fatalf("relativizeAgentCardMultiRoot: %v", err)
	}
	relBytes, err := io.ReadAll(resp.Body)
	_ = resp.Body.Close()
	if err != nil {
		t.Fatal(err)
	}
	var relCard a2a.AgentCard
	if err := json.Unmarshal(relBytes, &relCard); err != nil {
		t.Fatalf("Unmarshal relativized card: %v", err)
	}
	if got := relCard.SupportedInterfaces[0].URL; got != "invoke?mode=fast" {
		t.Errorf("interface[0].URL = %q, want %q", got, "invoke?mode=fast")
	}
	if got := relCard.SupportedInterfaces[1].URL; got != "" {
		t.Errorf("interface[1].URL = %q, want empty relative root", got)
	}
	if got := relCard.SupportedInterfaces[2].URL; got != "v1/rpc" {
		t.Errorf("interface[2].URL = %q, want %q", got, "v1/rpc")
	}

	inHdr := http.Header{
		"Authorization":                    {"Bearer app-token"},
		"Cookie":                           {"sid=secret"},
		"X-Mesh-Principal":                 {"spoofed"},
		api.HeaderPeerID:                   {"spoofed-peer"},
		api.HeaderMeshRequiredLabels:       {"env=prod"},
		api.HeaderMeshAuthentication:       {"Bearer task-token"},
		api.HeaderMeshIngressCallerBiscuit: {"spoofed-caller"},
	}
	sanitized := sanitizeCallerMeshHeaders(inHdr)
	if sanitized.Get("Cookie") != "" || sanitized.Get("X-Mesh-Principal") != "" || sanitized.Get(api.HeaderPeerID) != "" || sanitized.Get(api.HeaderMeshIngressCallerBiscuit) != "" {
		t.Errorf("sanitizeCallerMeshHeaders leaked restricted headers: %v", sanitized)
	}
	if sanitized.Get("Authorization") != "Bearer app-token" {
		t.Errorf("Authorization header was dropped: %v", sanitized)
	}
	if sanitized.Get(api.HeaderMeshRequiredLabels) != "env=prod" {
		t.Errorf("X-Mesh-Required-Labels was dropped: %v", sanitized)
	}
	if sanitized.Get(api.HeaderMeshAuthentication) != "Bearer task-token" {
		t.Errorf("X-Mesh-Authentication was dropped: %v", sanitized)
	}
}

func TestHTTPClientPathAndRequiredLabelValidation(t *testing.T) {
	ctx, cancel := context.WithTimeout(context.Background(), 20*time.Second)
	defer cancel()

	env := newStartedTestServer(t, ctx)
	hc := env.server.HTTPClient()
	validBase := fmt.Sprintf("http://agentmesh.local/mesh/%s/a2a/coder", env.routerHost.ID().String())

	for _, tc := range []struct {
		name   string
		url    string
		labels string
	}{
		{name: "dot_segment", url: validBase + "/../admin"},
		{name: "non_mesh_prefix", url: "http://agentmesh.local/v1/models"},
		{name: "invalid_peer_id", url: "http://agentmesh.local/mesh/not-a-peer/a2a/coder"},
		{name: "invalid_service_type", url: fmt.Sprintf("http://agentmesh.local/mesh/%s/unknown/coder", env.routerHost.ID().String())},
		{name: "lone_commas_in_required_labels", url: validBase + "/invoke", labels: ",,"},
		{name: "missing_equals_in_required_labels", url: validBase + "/invoke", labels: "env-prod"},
		{name: "duplicate_key_in_required_labels", url: validBase + "/invoke", labels: "env=prod,env=staging"},
		{name: "invalid_key_in_required_labels", url: validBase + "/invoke", labels: "BAD KEY!=val"},
	} {
		t.Run(tc.name, func(t *testing.T) {
			req, err := http.NewRequestWithContext(ctx, http.MethodPost, tc.url, strings.NewReader(`{}`))
			if err != nil {
				t.Fatal(err)
			}
			if tc.labels != "" {
				req.Header.Set(api.HeaderMeshRequiredLabels, tc.labels)
			}
			resp, err := hc.Do(req)
			if err != nil {
				t.Fatalf("hc.Do(%s): %v", tc.url, err)
			}
			_ = resp.Body.Close()
			if resp.StatusCode != http.StatusBadRequest {
				t.Fatalf("status = %d, want 400 Bad Request", resp.StatusCode)
			}
		})
	}
}

func TestListenA2AWhoIsDiscoverAndHTTPClient(t *testing.T) {
	ctx, cancel := context.WithTimeout(context.Background(), 20*time.Second)
	defer cancel()

	mesh := newInProcessMeshPair(t, ctx)
	provider := mesh.provider
	caller := mesh.caller

	ep, err := provider.ListenA2A(ctx, A2AOptions{
		Name:        "coder",
		Description: "In-process coding agent",
	})
	if err != nil {
		t.Fatalf("ListenA2A: %v", err)
	}
	if ep.PublicURL != nil {
		t.Fatalf("PublicURL = %v, want nil when Public=false", ep.PublicURL)
	}

	var seenCaller CallerInfo
	var seenRemoteAddr string
	var callerMu sync.Mutex

	mux := http.NewServeMux()
	mux.HandleFunc("/.well-known/agent-card.json", func(w http.ResponseWriter, _ *http.Request) {
		card := &a2a.AgentCard{
			Name:        "coder",
			Description: "In-process coding agent",
			Version:     "1.0.0",
			SupportedInterfaces: []*a2a.AgentInterface{
				{URL: ep.PrivateURL.String() + "/invoke", ProtocolBinding: a2a.TransportProtocolJSONRPC, ProtocolVersion: "1.0"},
			},
			Signatures: []a2a.AgentCardSignature{{Protected: "hdr", Signature: "sig"}},
		}
		w.Header().Set("Content-Type", "application/json")
		_ = json.NewEncoder(w).Encode(card)
	})
	mux.HandleFunc("/invoke", func(w http.ResponseWriter, r *http.Request) {
		info, ok := provider.WhoIs(r.RemoteAddr)
		if !ok || info == nil {
			http.Error(w, "missing WhoIs", http.StatusInternalServerError)
			return
		}
		// Mutate returned copy to verify WhoIs returns a defensive copy.
		if len(info.Roles) > 0 {
			info.Roles[0] = "mutated"
		}
		info.Labels["env"] = "mutated"

		fresh, ok := provider.WhoIs(r.RemoteAddr)
		if !ok {
			http.Error(w, "missing second WhoIs", http.StatusInternalServerError)
			return
		}
		callerMu.Lock()
		seenCaller = *fresh
		seenRemoteAddr = r.RemoteAddr
		callerMu.Unlock()

		w.Header().Set("Content-Type", "application/json")
		_, _ = w.Write([]byte(`{"ok":true}`))
	})
	httpSrv := &http.Server{Handler: mux}
	go func() { _ = httpSrv.Serve(ep.PrivateListener) }()
	t.Cleanup(func() {
		_ = httpSrv.Close()
		_ = ep.Close()
	})

	// 1. Targeted discovery & type-wide catalog discovery.
	byName, err := caller.Discover(ctx, api.ServiceType_SERVICE_TYPE_A2A, "coder")
	if err != nil {
		t.Fatalf("Discover(by name): %v", err)
	}
	if len(byName) != 1 || byName[0].GetLocalProxyUrl() != ep.PrivateURL.String() {
		t.Fatalf("Discover(by name) = %v, want [%s]", byName, ep.PrivateURL.String())
	}

	byType, err := caller.Discover(ctx, api.ServiceType_SERVICE_TYPE_A2A, "")
	if err != nil {
		t.Fatalf("Discover(by type): %v", err)
	}
	if len(byType) != 1 || byType[0].GetSrvDescription() != "In-process coding agent" {
		t.Fatalf("Discover(by type) = %v, want description 'In-process coding agent'", byType)
	}

	// 2. AgentCard fetch via caller.HTTPClient() rewrites URL & supports ETag/304.
	hc := caller.HTTPClient()
	cardResp, err := hc.Get(ep.PrivateURL.String() + "/.well-known/agent-card.json")
	if err != nil {
		t.Fatalf("GET agent-card.json: %v", err)
	}
	cardBody, err := io.ReadAll(cardResp.Body)
	_ = cardResp.Body.Close()
	if err != nil {
		t.Fatalf("io.ReadAll(agent-card.json): %v", err)
	}
	if cardResp.StatusCode != http.StatusOK {
		t.Fatalf("agent-card status = %d: %s", cardResp.StatusCode, cardBody)
	}
	var gotCard a2a.AgentCard
	if err := json.Unmarshal(cardBody, &gotCard); err != nil {
		t.Fatalf("unmarshal card: %v", err)
	}
	if len(gotCard.SupportedInterfaces) != 1 || gotCard.SupportedInterfaces[0].URL != ep.PrivateURL.String()+"/invoke" {
		t.Fatalf("SupportedInterfaces = %+v, want %s/invoke", gotCard.SupportedInterfaces, ep.PrivateURL.String())
	}
	if len(gotCard.Signatures) != 0 {
		t.Errorf("Signatures = %v, want stripped on regenerated mesh card", gotCard.Signatures)
	}
	etag := cardResp.Header.Get("ETag")
	if etag == "" {
		t.Fatal("missing ETag header on regenerated AgentCard")
	}

	req304, err := http.NewRequestWithContext(ctx, http.MethodGet, ep.PrivateURL.String()+"/.well-known/agent-card.json", nil)
	if err != nil {
		t.Fatalf("NewRequestWithContext(304): %v", err)
	}
	req304.Header.Set("If-None-Match", etag)
	resp304, err := hc.Do(req304)
	if err != nil {
		t.Fatalf("GET agent-card with If-None-Match: %v", err)
	}
	_ = resp304.Body.Close()
	if resp304.StatusCode != http.StatusNotModified {
		t.Fatalf("If-None-Match status = %d, want 304", resp304.StatusCode)
	}

	// 3. POST /invoke verifies WhoIs caller identity, defensive copy, and post-close cleanup.
	invReq, err := http.NewRequestWithContext(ctx, http.MethodPost, ep.PrivateURL.String()+"/invoke", strings.NewReader(`{}`))
	if err != nil {
		t.Fatalf("NewRequestWithContext(invoke): %v", err)
	}
	invReq.Header.Set(api.HeaderMeshRequiredLabels, "env=prod")
	invResp, err := hc.Do(invReq)
	if err != nil {
		t.Fatalf("POST /invoke: %v", err)
	}
	_, readErr := io.ReadAll(invResp.Body)
	_ = invResp.Body.Close()
	if readErr != nil {
		t.Fatalf("io.ReadAll(invoke): %v", readErr)
	}
	if invResp.StatusCode != http.StatusOK {
		t.Fatalf("POST /invoke status = %d, want 200", invResp.StatusCode)
	}

	callerMu.Lock()
	gotCaller := seenCaller
	remoteAddr := seenRemoteAddr
	callerMu.Unlock()

	if gotCaller.PeerID != caller.Status().PeerID {
		t.Errorf("WhoIs PeerID = %q, want %q", gotCaller.PeerID, caller.Status().PeerID)
	}
	if gotCaller.Principal != "[email protected]" {
		t.Errorf("WhoIs Principal = %q, want %q", gotCaller.Principal, "[email protected]")
	}
	if len(gotCaller.Roles) != 1 || gotCaller.Roles[0] != api.RoleNode {
		t.Errorf("WhoIs Roles = %v, want [%s]", gotCaller.Roles, api.RoleNode)
	}
	if gotCaller.Labels["env"] != "prod" {
		t.Errorf("WhoIs Labels[env] = %q, want prod", gotCaller.Labels["env"])
	}

	// Wait briefly for trackedConn.Close() to evict remoteAddr from activeConns.
	deadline := time.Now().Add(2 * time.Second)
	for time.Now().Before(deadline) {
		if _, still := provider.WhoIs(remoteAddr); !still {
			break
		}
		time.Sleep(10 * time.Millisecond)
	}
	if _, still := provider.WhoIs(remoteAddr); still {
		t.Errorf("WhoIs(%q) still present after connection closed", remoteAddr)
	}

	// 4. Required-label mismatch fails closed BEFORE sending request bytes.
	badLabelReq, err := http.NewRequestWithContext(ctx, http.MethodPost, ep.PrivateURL.String()+"/invoke", strings.NewReader(`{}`))
	if err != nil {
		t.Fatalf("NewRequestWithContext(badLabel): %v", err)
	}
	badLabelReq.Header.Set(api.HeaderMeshRequiredLabels, "env=staging")
	badLabelResp, err := hc.Do(badLabelReq)
	if err != nil {
		t.Fatalf("POST with mismatched required label: %v", err)
	}
	_ = badLabelResp.Body.Close()
	if badLabelResp.StatusCode != http.StatusForbidden {
		t.Fatalf("mismatched required label status = %d, want 403", badLabelResp.StatusCode)
	}
}

func TestHTTPClientAuthorizerRunLimitReturns504And500(t *testing.T) {
	ctx, cancel := context.WithTimeout(context.Background(), 20*time.Second)
	defer cancel()

	mesh := newInProcessMeshPair(t, ctx)
	ep, err := mesh.provider.ListenA2A(ctx, A2AOptions{Name: "coder"})
	if err != nil {
		t.Fatalf("ListenA2A: %v", err)
	}
	httpSrv := &http.Server{
		Handler: http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
			w.WriteHeader(http.StatusOK)
		}),
	}
	go func() { _ = httpSrv.Serve(ep.PrivateListener) }()
	t.Cleanup(func() {
		_ = httpSrv.Close()
		_ = ep.Close()
	})

	hc := mesh.caller.HTTPClient()

	// 1. Transient Datalog evaluation timeout -> 504 Gateway Timeout (not 403).
	mesh.caller.node.BiscuitTimeout = time.Nanosecond
	resp504, err := hc.Post(ep.PrivateURL.String()+"/invoke", "application/json", strings.NewReader(`{}`))
	if err != nil {
		t.Fatalf("POST with 1ns BiscuitTimeout: %v", err)
	}
	body504, readErr := io.ReadAll(resp504.Body)
	_ = resp504.Body.Close()
	if readErr != nil {
		t.Fatalf("io.ReadAll(resp504): %v", readErr)
	}
	if resp504.StatusCode != http.StatusGatewayTimeout {
		t.Fatalf("authorizer timeout status = %d (%s), want 504 Gateway Timeout", resp504.StatusCode, body504)
	}

	// 2. Deterministic Datalog max-facts limit -> 500 Internal Server Error (not 403).
	mesh.caller.node.BiscuitTimeout = time.Second
	texts := make([]string, 0, 1100)
	for i := range 1100 {
		texts = append(texts, fmt.Sprintf("limit_fact_%d(true) <- role(%q)", i, api.RoleNode))
	}
	rules, err := api.ParseDatalogRules(texts)
	if err != nil {
		t.Fatalf("ParseDatalogRules: %v", err)
	}
	mesh.caller.node.MeshPolicyMu.Lock()
	mesh.caller.node.MeshPolicyRules = rules
	mesh.caller.node.MeshPolicyMu.Unlock()

	resp500, err := hc.Post(ep.PrivateURL.String()+"/invoke", "application/json", strings.NewReader(`{}`))
	if err != nil {
		t.Fatalf("POST over Datalog fact limit: %v", err)
	}
	body500, readErr := io.ReadAll(resp500.Body)
	_ = resp500.Body.Close()
	if readErr != nil {
		t.Fatalf("io.ReadAll(resp500): %v", readErr)
	}
	if resp500.StatusCode != http.StatusInternalServerError {
		t.Fatalf("authorizer max-facts status = %d (%s), want 500 Internal Server Error", resp500.StatusCode, body500)
	}
}

func TestListenA2APublicNoAnnounceReturnsNilPublicURL(t *testing.T) {
	ctx, cancel := context.WithTimeout(context.Background(), 20*time.Second)
	defer cancel()

	cpPub, cpPriv, err := ed25519.GenerateKey(rand.Reader)
	if err != nil {
		t.Fatal(err)
	}
	rh, err := libp2p.New(libp2p.ListenAddrStrings("/ip4/127.0.0.1/tcp/0"))
	if err != nil {
		t.Fatal(err)
	}
	t.Cleanup(func() { _ = rh.Close() })

	routerBiscuit := mintTestBiscuit(t, cpPriv, rh.ID().String(), api.RoleRouter, time.Now().Add(time.Hour))
	rh.SetStreamHandler(api.AuthProtocolID, func(st network.Stream) {
		defer func() { _ = st.Close() }()
		reader := msgio.NewVarintReaderSize(st, 64*1024)
		msg, rErr := reader.ReadMsg()
		if rErr != nil {
			return
		}
		reader.ReleaseMsg(msg)
		reply, _ := proto.Marshal(&api.AuthResponse{Success: true, Biscuit: routerBiscuit})
		_ = msgio.NewVarintWriter(st).WriteMsg(reply)
	})
	rh.SetStreamHandler(api.DirectoryProtocolID, func(st network.Stream) {
		defer func() { _ = st.Close() }()
		reader := msgio.NewVarintReaderSize(st, 64*1024)
		msg, rErr := reader.ReadMsg()
		if rErr != nil {
			return
		}
		reader.ReleaseMsg(msg)
		reply, _ := proto.Marshal(&api.DirectoryResponse{})
		_ = msgio.NewVarintWriter(st).WriteMsg(reply)
	})

	var routerAddrs []string
	for _, a := range rh.Addrs() {
		routerAddrs = append(routerAddrs, a.String()+"/p2p/"+rh.ID().String())
	}
	cp := newFakeControlPlaneHTTPWithKeys(t, cpPub, cpPriv, routerAddrs)
	cp.mu.Lock()
	cp.routers = []*api.RouterInfo{
		{
			PeerId:     rh.ID().String(),
			Addresses:  routerAddrs,
			IngressUrl: "https://bananas.sam-mesh.dev/ingress",
		},
	}
	cp.mu.Unlock()

	s := &Server{
		StateDir:        t.TempDir(),
		ControlPlaneURL: cp.srv.URL,
		BootstrapToken:  "boot-secret",
		ListenAddrs:     []string{"/ip4/127.0.0.1/tcp/0"},
		AllowInsecure:   true,
	}
	if _, err := s.Up(ctx); err != nil {
		t.Fatalf("Server.Up: %v", err)
	}
	t.Cleanup(func() { _ = s.Close() })

	unannounced, err := s.ListenA2A(ctx, A2AOptions{
		Name:       "unannounced-agent",
		Public:     true,
		NoAnnounce: true,
	})
	if err != nil {
		t.Fatalf("ListenA2A(NoAnnounce=true): %v", err)
	}
	t.Cleanup(func() { _ = unannounced.Close() })
	if unannounced.PublicListener == nil {
		t.Fatal("PublicListener is nil, want non-nil when Public=true")
	}
	if unannounced.PublicURL != nil {
		t.Fatalf("PublicURL = %v, want nil when NoAnnounce=true (routers have no directory route)", unannounced.PublicURL)
	}

	announced, err := s.ListenA2A(ctx, A2AOptions{
		Name:   "announced-agent",
		Public: true,
	})
	if err != nil {
		t.Fatalf("ListenA2A(NoAnnounce=false): %v", err)
	}
	t.Cleanup(func() { _ = announced.Close() })
	wantPubURL := fmt.Sprintf("https://bananas.sam-mesh.dev/ingress/%s/a2a/announced-agent", s.Status().PeerID)
	if announced.PublicURL == nil || announced.PublicURL.String() != wantPubURL {
		t.Fatalf("PublicURL = %v, want %s", announced.PublicURL, wantPubURL)
	}
}

func TestServerMetricsRegistererNoCollisionAndCleanup(t *testing.T) {
	ctx, cancel := context.WithTimeout(context.Background(), 20*time.Second)
	defer cancel()

	cpPub, cpPriv, err := ed25519.GenerateKey(rand.Reader)
	if err != nil {
		t.Fatal(err)
	}
	rh, err := libp2p.New(libp2p.ListenAddrStrings("/ip4/127.0.0.1/tcp/0"))
	if err != nil {
		t.Fatal(err)
	}
	t.Cleanup(func() { _ = rh.Close() })

	routerBiscuit := mintTestBiscuit(t, cpPriv, rh.ID().String(), api.RoleRouter, time.Now().Add(time.Hour))
	rh.SetStreamHandler(api.AuthProtocolID, func(st network.Stream) {
		defer func() { _ = st.Close() }()
		reader := msgio.NewVarintReaderSize(st, 64*1024)
		msg, rErr := reader.ReadMsg()
		if rErr != nil {
			return
		}
		reader.ReleaseMsg(msg)
		reply, _ := proto.Marshal(&api.AuthResponse{Success: true, Biscuit: routerBiscuit})
		_ = msgio.NewVarintWriter(st).WriteMsg(reply)
	})
	rh.SetStreamHandler(api.DirectoryProtocolID, func(st network.Stream) {
		defer func() { _ = st.Close() }()
		reader := msgio.NewVarintReaderSize(st, 64*1024)
		msg, rErr := reader.ReadMsg()
		if rErr != nil {
			return
		}
		reader.ReleaseMsg(msg)
		reply, _ := proto.Marshal(&api.DirectoryResponse{})
		_ = msgio.NewVarintWriter(st).WriteMsg(reply)
	})

	var routerAddrs []string
	for _, a := range rh.Addrs() {
		routerAddrs = append(routerAddrs, a.String()+"/p2p/"+rh.ID().String())
	}
	cp := newFakeControlPlaneHTTPWithKeys(t, cpPub, cpPriv, routerAddrs)

	reg := prometheus.NewRegistry()
	dir1 := t.TempDir()
	s1 := &Server{
		StateDir:        dir1,
		ControlPlaneURL: cp.srv.URL,
		BootstrapToken:  "boot-secret",
		ListenAddrs:     []string{"/ip4/127.0.0.1/tcp/0"},
		AllowInsecure:   true,
		Registerer:      reg,
	}
	st1, err := s1.Up(ctx)
	if err != nil {
		t.Fatalf("s1.Up: %v", err)
	}
	t.Cleanup(func() { _ = s1.Close() })

	s2 := &Server{
		StateDir:        t.TempDir(),
		ControlPlaneURL: cp.srv.URL,
		BootstrapToken:  "boot-secret",
		ListenAddrs:     []string{"/ip4/127.0.0.1/tcp/0"},
		AllowInsecure:   true,
		Registerer:      reg,
	}
	st2, err := s2.Up(ctx)
	if err != nil {
		t.Fatalf("s2.Up (shared Registerer): %v", err)
	}
	t.Cleanup(func() { _ = s2.Close() })

	// Trigger a gated stream refusal on s1 and verify s1's counter increments.
	stranger, err := libp2p.New(libp2p.ListenAddrStrings("/ip4/127.0.0.1/tcp/0"))
	if err != nil {
		t.Fatal(err)
	}
	t.Cleanup(func() { _ = stranger.Close() })
	if err := stranger.Connect(ctx, peer.AddrInfo{ID: s1.node.Host.ID(), Addrs: s1.node.Host.Addrs()}); err != nil {
		t.Fatal(err)
	}
	st, err := stranger.NewStream(ctx, s1.node.Host.ID(), api.IngressProtocolID)
	if err != nil {
		t.Fatal(err)
	}
	defer func() { _ = st.Reset() }()
	if _, err := st.Write([]byte("GET / HTTP/1.1\r\nHost: x\r\n\r\n")); err != nil {
		t.Fatalf("write: %v", err)
	}
	buf := make([]byte, 64)
	_ = st.SetReadDeadline(time.Now().Add(time.Second))
	if n, err := st.Read(buf); err == nil {
		t.Fatalf("%s answered %q to an unauthenticated peer", api.IngressProtocolID, buf[:n])
	}
	if err := stranger.Network().ClosePeer(s1.node.Host.ID()); err != nil {
		t.Fatal(err)
	}
	waitForCount(t, func() uint64 {
		return s1.refusedStreamCount("unauthenticated", api.IngressProtocolID)
	}, 1, 5*time.Second)

	families, err := reg.Gather()
	if err != nil {
		t.Fatalf("reg.Gather: %v", err)
	}
	if v, ok := findMetricValue(families, "agentmesh_meshnet_ready", map[string]string{"peer_id": st1.PeerID}); !ok || v != 1 {
		t.Errorf("s1 agentmesh_meshnet_ready = %v (ok=%v), want 1", v, ok)
	}
	if v, ok := findMetricValue(families, "agentmesh_meshnet_ready", map[string]string{"peer_id": st2.PeerID}); !ok || v != 1 {
		t.Errorf("s2 agentmesh_meshnet_ready = %v (ok=%v), want 1", v, ok)
	}
	if v, ok := findMetricValue(families, "agentmesh_meshnet_routers", map[string]string{"peer_id": st1.PeerID, "state": "attached"}); !ok || v < 1 {
		t.Errorf("s1 attached routers gauge = %v (ok=%v), want >= 1", v, ok)
	}
	if v, ok := findMetricValue(families, "agentmesh_meshnet_streams_refused_total", map[string]string{
		"peer_id":  st1.PeerID,
		"scope":    "unauthenticated",
		"protocol": string(api.IngressProtocolID),
	}); !ok || v < 1 {
		t.Errorf("s1 streams_refused_total = %v (ok=%v), want >= 1", v, ok)
	}

	// Closing s1 unregisters its collectors so the same PeerID can re-register cleanly.
	if err := s1.Close(); err != nil {
		t.Fatalf("s1.Close: %v", err)
	}
	afterClose, err := reg.Gather()
	if err != nil {
		t.Fatalf("reg.Gather after s1.Close: %v", err)
	}
	if _, ok := findMetricValue(afterClose, "agentmesh_meshnet_ready", map[string]string{"peer_id": st1.PeerID}); ok {
		t.Fatal("s1 metrics remained registered after Close")
	}

	s1Restarted := &Server{
		StateDir:        dir1,
		ControlPlaneURL: cp.srv.URL,
		ListenAddrs:     []string{"/ip4/127.0.0.1/tcp/0"},
		AllowInsecure:   true,
		Registerer:      reg,
	}
	if _, err := s1Restarted.Up(ctx); err != nil {
		t.Fatalf("s1Restarted.Up with same PeerID and Registerer: %v", err)
	}
	t.Cleanup(func() { _ = s1Restarted.Close() })
}

func TestCredentialLapseExponentialBackoffAndSignal(t *testing.T) {
	ctx, cancel := context.WithTimeout(context.Background(), 20*time.Second)
	defer cancel()

	cpPub, cpPriv, err := ed25519.GenerateKey(rand.Reader)
	if err != nil {
		t.Fatal(err)
	}
	rh, err := libp2p.New(libp2p.ListenAddrStrings("/ip4/127.0.0.1/tcp/0"))
	if err != nil {
		t.Fatal(err)
	}
	t.Cleanup(func() { _ = rh.Close() })

	routerBiscuit := mintTestBiscuit(t, cpPriv, rh.ID().String(), api.RoleRouter, time.Now().Add(time.Hour))
	rh.SetStreamHandler(api.AuthProtocolID, func(st network.Stream) {
		defer func() { _ = st.Close() }()
		reader := msgio.NewVarintReaderSize(st, 64*1024)
		msg, rErr := reader.ReadMsg()
		if rErr != nil {
			return
		}
		reader.ReleaseMsg(msg)
		reply, _ := proto.Marshal(&api.AuthResponse{Success: true, Biscuit: routerBiscuit})
		_ = msgio.NewVarintWriter(st).WriteMsg(reply)
	})
	rh.SetStreamHandler(api.DirectoryProtocolID, func(st network.Stream) {
		defer func() { _ = st.Close() }()
		reader := msgio.NewVarintReaderSize(st, 64*1024)
		msg, rErr := reader.ReadMsg()
		if rErr != nil {
			return
		}
		reader.ReleaseMsg(msg)
		reply, _ := proto.Marshal(&api.DirectoryResponse{})
		_ = msgio.NewVarintWriter(st).WriteMsg(reply)
	})

	var routerAddrs []string
	for _, a := range rh.Addrs() {
		routerAddrs = append(routerAddrs, a.String()+"/p2p/"+rh.ID().String())
	}

	var allowTokenSource atomic.Bool
	allowTokenSource.Store(true)

	// Dedicated fake CP whose /refresh blocks until the initial short-lived
	// credential reaches its expiry boundary, then fails immediately so the
	// very first failed renewal lands right at expiry with initial backoff.
	var expUnix atomic.Int64
	mux := http.NewServeMux()
	var cpURL string
	mux.HandleFunc("/keys", func(w http.ResponseWriter, _ *http.Request) {
		resp := &api.KeysResponse{PublicKeys: [][]byte{cpPub}}
		_ = api.SignKeysResponse(resp, []ed25519.PrivateKey{cpPriv}, time.Now())
		writeProto(w, resp)
	})
	mux.HandleFunc("/info", func(w http.ResponseWriter, _ *http.Request) {
		writeProto(w, &api.ControlPlaneInfoResponse{RouterAddresses: routerAddrs})
	})
	mux.HandleFunc("/policies", func(w http.ResponseWriter, _ *http.Request) {
		compiled, _ := api.BuildPolicyRules([]*api.PolicyRole{
			{Name: api.RoleNode, AllowedTargets: []string{"*"}, AllowedServices: []string{"*"}, AllowedPublish: []string{"*"}, AllowedIngress: []*api.IngressGrant{{Services: []string{"*"}}}},
		}, nil)
		texts := make([]string, 0, len(compiled))
		for _, r := range compiled {
			texts = append(texts, r.Text)
		}
		writeProto(w, &api.PolicyConfigGetResponse{DatalogRules: texts})
	})
	mux.HandleFunc("/revocations", func(w http.ResponseWriter, _ *http.Request) {
		writeProto(w, &api.RevocationsResponse{})
	})
	mux.HandleFunc("/register", func(w http.ResponseWriter, r *http.Request) {
		var req api.EnrollRequest
		if !readProto(w, r, &req) {
			return
		}
		var exp time.Time
		if req.GetJwt() == "jwt-initial" {
			exp = time.Now().Truncate(time.Second).Add(2 * time.Second)
			expUnix.Store(exp.Unix())
		} else {
			exp = time.Now().Add(time.Hour)
			expUnix.Store(exp.Unix())
		}
		tok := mintTestBiscuit(t, cpPriv, req.GetPeerId(), api.RoleNode, exp)
		writeProto(w, &api.EnrollResponse{
			BiscuitToken:          tok,
			ControlPlanePublicKey: cpPub,
			ExpireTime:            timestamppb.New(exp),
			RouterAddresses:       routerAddrs,
		})
	})
	mux.HandleFunc("/refresh", func(w http.ResponseWriter, r *http.Request) {
		target := time.Unix(expUnix.Load(), 0)
		if wait := time.Until(target); wait > 0 {
			timer := time.NewTimer(wait)
			select {
			case <-r.Context().Done():
				timer.Stop()
				return
			case <-timer.C:
			}
		}
		http.Error(w, "refresh unavailable", http.StatusServiceUnavailable)
	})
	cpSrv := httptest.NewServer(mux)
	cpURL = cpSrv.URL
	t.Cleanup(cpSrv.Close)

	var lapseMu sync.Mutex
	var lapseTimes []time.Time
	lapseCh := make(chan struct{}, 8)

	reg := prometheus.NewRegistry()
	s := &Server{
		StateDir:        t.TempDir(),
		ControlPlaneURL: cpURL,
		TokenSource: func(context.Context) (string, error) {
			if !allowTokenSource.Load() {
				return "", errors.New("idp unavailable")
			}
			if expUnix.Load() == 0 {
				return "jwt-initial", nil
			}
			return "jwt-recovered", nil
		},
		ListenAddrs:           []string{"/ip4/127.0.0.1/tcp/0"},
		AllowInsecure:         true,
		Registerer:            reg,
		renewalInitialBackoff: 80 * time.Millisecond,
		renewalMaxBackoff:     time.Second,
		OnCredentialLapse: func(err error) {
			if !errors.Is(err, ErrCredentialLapsed) {
				t.Errorf("OnCredentialLapse error %v does not wrap ErrCredentialLapsed", err)
			}
			lapseMu.Lock()
			lapseTimes = append(lapseTimes, time.Now())
			lapseMu.Unlock()
			select {
			case lapseCh <- struct{}{}:
			default:
			}
		},
	}
	st, err := s.Up(ctx)
	if err != nil {
		t.Fatalf("s.Up: %v", err)
	}
	t.Cleanup(func() { _ = s.Close() })

	// Break TokenSource before expiry so fallback re-enrollment also fails.
	allowTokenSource.Store(false)

	select {
	case <-s.Done():
	case <-ctx.Done():
		t.Fatal("timed out waiting for s.Done() on credential lapse")
	}
	if !errors.Is(s.Err(), ErrCredentialLapsed) {
		t.Fatalf("s.Err() = %v, want ErrCredentialLapsed", s.Err())
	}
	if !s.Status().CredentialLapsed {
		t.Fatal("Status().CredentialLapsed = false, want true")
	}

	// Collect 3 consecutive lapse callbacks and verify exponential backoff (80ms -> 160ms).
	for range 3 {
		select {
		case <-lapseCh:
		case <-ctx.Done():
			t.Fatal("timed out waiting for 3 exponential-backoff lapse callbacks")
		}
	}
	lapseMu.Lock()
	t0, t1, t2 := lapseTimes[0], lapseTimes[1], lapseTimes[2]
	lapseMu.Unlock()

	gap1 := t1.Sub(t0)
	gap2 := t2.Sub(t1)
	if gap1 < 50*time.Millisecond {
		t.Errorf("first backoff gap = %v, want >= 50ms (initial 80ms)", gap1)
	}
	if gap2 < gap1+40*time.Millisecond {
		t.Errorf("second backoff gap = %v, did not grow exponentially over first gap %v", gap2, gap1)
	}

	families, err := reg.Gather()
	if err != nil {
		t.Fatalf("reg.Gather: %v", err)
	}
	if v, ok := findMetricValue(families, "agentmesh_meshnet_credential_lapsed", map[string]string{"peer_id": st.PeerID}); !ok || v != 1 {
		t.Errorf("agentmesh_meshnet_credential_lapsed = %v (ok=%v), want 1", v, ok)
	}
	if v, ok := findMetricValue(families, "agentmesh_meshnet_credential_lapses_total", map[string]string{"peer_id": st.PeerID}); !ok || v < 3 {
		t.Errorf("agentmesh_meshnet_credential_lapses_total = %v (ok=%v), want >= 3", v, ok)
	}

	// Restore TokenSource and verify the next backoff retry recovers Status().CredentialLapsed.
	allowTokenSource.Store(true)
	deadline := time.Now().Add(3 * time.Second)
	for time.Now().Before(deadline) {
		if !s.Status().CredentialLapsed {
			break
		}
		time.Sleep(25 * time.Millisecond)
	}
	if s.Status().CredentialLapsed {
		t.Fatal("Status().CredentialLapsed remained true after TokenSource recovered")
	}
}

func findMetricValue(families []*dto.MetricFamily, name string, wantLabels map[string]string) (float64, bool) {
	for _, mf := range families {
		if mf.GetName() != name {
			continue
		}
		for _, m := range mf.GetMetric() {
			matched := true
			for k, want := range wantLabels {
				got := ""
				for _, lp := range m.GetLabel() {
					if lp.GetName() == k {
						got = lp.GetValue()
						break
					}
				}
				if got != want {
					matched = false
					break
				}
			}
			if !matched {
				continue
			}
			if m.Gauge != nil {
				return m.Gauge.GetValue(), true
			}
			if m.Counter != nil {
				return m.Counter.GetValue(), true
			}
		}
	}
	return 0, false
}

func TestLocalMCPAndInferenceHandlers(t *testing.T) {
	ctx, cancel := context.WithTimeout(context.Background(), 20*time.Second)
	defer cancel()

	mesh := newInProcessMeshPair(t, ctx)
	_, err := mesh.provider.ListenA2A(ctx, A2AOptions{Name: "helper", Description: "Helper agent"})
	if err != nil {
		t.Fatalf("ListenA2A: %v", err)
	}

	// 1. Local MCPServer + MCPHandler over StreamableHTTP.
	if mesh.caller.MCPServer() == nil {
		t.Fatal("MCPServer() returned nil")
	}
	mcpHTTP := httptest.NewServer(mesh.caller.MCPHandler())
	t.Cleanup(mcpHTTP.Close)

	client := mcp.NewClient(&mcp.Implementation{Name: "test-caller", Version: "1.0.0"}, nil)
	sess, err := client.Connect(ctx, &mcp.StreamableClientTransport{Endpoint: mcpHTTP.URL + "/mcp"}, nil)
	if err != nil {
		t.Fatalf("MCP Connect: %v", err)
	}
	t.Cleanup(func() { _ = sess.Close() })

	toolsRes, err := sess.ListTools(ctx, nil)
	if err != nil {
		t.Fatalf("ListTools: %v", err)
	}
	if len(toolsRes.Tools) != 6 {
		t.Fatalf("len(Tools) = %d, want 6 mesh tools", len(toolsRes.Tools))
	}

	infoRes, err := sess.CallTool(ctx, &mcp.CallToolParams{Name: "get_mesh_info"})
	if err != nil || infoRes.IsError {
		t.Fatalf("get_mesh_info failed: err=%v res=%+v", err, infoRes)
	}

	discRes, err := sess.CallTool(ctx, &mcp.CallToolParams{
		Name:      "discover_remote_services",
		Arguments: map[string]any{"type": "a2a"},
	})
	if err != nil || discRes.IsError {
		t.Fatalf("discover_remote_services failed: err=%v res=%+v", err, discRes)
	}
	discText := discRes.Content[0].(*mcp.TextContent).Text
	if !strings.Contains(discText, "helper") {
		t.Errorf("discover_remote_services = %s, want helper", discText)
	}

	localRes, err := sess.CallTool(ctx, &mcp.CallToolParams{Name: "list_local_services"})
	if err != nil || localRes.IsError {
		t.Fatalf("list_local_services failed: err=%v res=%+v", err, localRes)
	}

	// CIDv1 representation of self PeerID is canonicalized and rejected by find_remote_tools.
	selfCIDv1 := peer.ToCid(mesh.caller.peerID).String()
	selfToolRes, selfToolErr := sess.CallTool(ctx, &mcp.CallToolParams{
		Name:      "find_remote_tools",
		Arguments: map[string]any{"peer_id": selfCIDv1},
	})
	if selfToolErr == nil && (selfToolRes == nil || !selfToolRes.IsError) {
		t.Fatalf("find_remote_tools accepted CIDv1 self peer_id %q, want error", selfCIDv1)
	}

	// 2. Local InferenceHandler routes and validation.
	infHTTP := httptest.NewServer(mesh.caller.InferenceHandler())
	t.Cleanup(infHTTP.Close)

	modelsResp, err := http.Get(infHTTP.URL + "/v1/models")
	if err != nil {
		t.Fatalf("GET /v1/models: %v", err)
	}
	modelsBody, err := io.ReadAll(modelsResp.Body)
	_ = modelsResp.Body.Close()
	if err != nil {
		t.Fatalf("io.ReadAll(/v1/models): %v", err)
	}
	if modelsResp.StatusCode != http.StatusOK || !strings.Contains(string(modelsBody), `"object":"list"`) {
		t.Fatalf("GET /v1/models = %d: %s", modelsResp.StatusCode, modelsBody)
	}

	resp405, err := http.Post(infHTTP.URL+"/v1/models", "application/json", strings.NewReader(`{}`))
	if err != nil {
		t.Fatal(err)
	}
	_ = resp405.Body.Close()
	if resp405.StatusCode != http.StatusMethodNotAllowed {
		t.Errorf("POST /v1/models = %d, want 405", resp405.StatusCode)
	}

	resp404, err := http.Get(infHTTP.URL + "/v1/unknown")
	if err != nil {
		t.Fatal(err)
	}
	_ = resp404.Body.Close()
	if resp404.StatusCode != http.StatusNotFound {
		t.Errorf("GET /v1/unknown = %d, want 404", resp404.StatusCode)
	}

	resp400, err := http.Post(infHTTP.URL+"/v1/chat/completions", "application/json", strings.NewReader(`{}`))
	if err != nil {
		t.Fatal(err)
	}
	_ = resp400.Body.Close()
	if resp400.StatusCode != http.StatusBadRequest {
		t.Errorf("POST /v1/chat/completions missing model = %d, want 400", resp400.StatusCode)
	}

	respUnknownModel, err := http.Post(infHTTP.URL+"/v1/chat/completions", "application/json", bytes.NewReader([]byte(`{"model":"gpt-4o"}`)))
	if err != nil {
		t.Fatal(err)
	}
	_ = respUnknownModel.Body.Close()
	if respUnknownModel.StatusCode != http.StatusNotFound {
		t.Errorf("POST /v1/chat/completions with unknown model = %d, want 404", respUnknownModel.StatusCode)
	}
}

func TestExternalIngressCanonicalizesActorNodePeerID(t *testing.T) {
	ctx, cancel := context.WithTimeout(context.Background(), 20*time.Second)
	defer cancel()

	env := newStartedTestServer(t, ctx)
	s := env.server

	ep, err := s.ListenA2A(ctx, A2AOptions{
		Name:        "coder",
		Public:      true,
		IngressAuth: api.IngressAuthMode_INGRESS_AUTH_MODE_EDGE,
		NoAnnounce:  true,
	})
	if err != nil {
		t.Fatalf("ListenA2A: %v", err)
	}
	t.Cleanup(func() { _ = ep.Close() })

	var gotPeerID string
	var mu sync.Mutex
	pubSrv := &http.Server{
		Handler: http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
			if info, ok := s.WhoIs(r.RemoteAddr); ok && info != nil {
				mu.Lock()
				gotPeerID = info.PeerID
				mu.Unlock()
			}
			w.WriteHeader(http.StatusOK)
		}),
	}
	go func() { _ = pubSrv.Serve(ep.PublicListener) }()
	t.Cleanup(func() { _ = pubSrv.Close() })

	actorHost, err := libp2p.New(libp2p.ListenAddrStrings("/ip4/127.0.0.1/tcp/0"))
	if err != nil {
		t.Fatal(err)
	}
	t.Cleanup(func() { _ = actorHost.Close() })
	cidv1Actor := peer.ToCid(actorHost.ID()).String()
	if cidv1Actor == actorHost.ID().String() {
		t.Fatalf("peer.ToCid(%s) produced identical string", actorHost.ID())
	}

	routerPeer := newAdmittedTestPeer(t, ctx, s, env.cpPriv, api.RoleRouter)
	routerBiscuit := mintTestBiscuit(t, env.cpPriv, routerPeer.ID().String(), api.RoleRouter, time.Now().Add(time.Hour))

	b := biscuit.NewBuilder(env.cpPriv)
	for _, f := range []biscuit.Fact{
		{Predicate: biscuit.Predicate{Name: api.FactActorNode, IDs: []biscuit.Term{biscuit.String(cidv1Actor)}}},
		{Predicate: biscuit.Predicate{Name: api.FactRole, IDs: []biscuit.Term{biscuit.String(api.RoleNode)}}},
		{Predicate: biscuit.Predicate{Name: api.FactEmail, IDs: []biscuit.Term{biscuit.String("[email protected]")}}},
		{Predicate: biscuit.Predicate{Name: api.FactExpiration, IDs: []biscuit.Term{biscuit.Date(time.Now().Add(time.Hour))}}},
		api.MarkerFact(api.FactTargetUnrestricted),
	} {
		if err := b.AddAuthorityFact(f); err != nil {
			t.Fatalf("AddAuthorityFact: %v", err)
		}
	}
	built, err := b.Build()
	if err != nil {
		t.Fatalf("Build: %v", err)
	}
	callerBytes, err := built.Serialize()
	if err != nil {
		t.Fatalf("Serialize: %v", err)
	}

	req, err := http.NewRequestWithContext(ctx, http.MethodPost, "http://mesh/a2a/coder/invoke", strings.NewReader(`{}`))
	if err != nil {
		t.Fatalf("NewRequestWithContext: %v", err)
	}
	req.Header.Set(api.HeaderMeshBiscuit, base64.RawURLEncoding.EncodeToString(routerBiscuit))
	req.Header.Set(api.HeaderMeshIngress, api.IngressAuthModeStringEdge)
	req.Header.Set(api.HeaderMeshIngressCallerBiscuit, base64.RawURLEncoding.EncodeToString(callerBytes))

	st, err := routerPeer.NewStream(ctx, s.node.Host.ID(), api.IngressProtocolID)
	if err != nil {
		t.Fatalf("NewStream(%s): %v", api.IngressProtocolID, err)
	}
	defer func() { _ = st.Close() }()
	if err := req.Write(st); err != nil {
		t.Fatalf("req.Write: %v", err)
	}
	_ = st.SetReadDeadline(time.Now().Add(5 * time.Second))
	resp, err := http.ReadResponse(bufio.NewReader(st), req)
	if err != nil {
		t.Fatalf("ReadResponse: %v", err)
	}
	respBody, readErr := io.ReadAll(resp.Body)
	_ = resp.Body.Close()
	if readErr != nil {
		t.Fatalf("io.ReadAll(resp): %v", readErr)
	}
	if resp.StatusCode != http.StatusOK {
		t.Fatalf("external ingress status = %d (%s), want 200", resp.StatusCode, string(respBody))
	}

	mu.Lock()
	defer mu.Unlock()
	if gotPeerID != actorHost.ID().String() {
		t.Fatalf("WhoIs PeerID = %q, want canonical %q (from CIDv1 %q)", gotPeerID, actorHost.ID().String(), cidv1Actor)
	}
}

func TestWhoIsUnverifiedExternalAndOIDCUserPeerID(t *testing.T) {
	ctx, cancel := context.WithTimeout(context.Background(), 20*time.Second)
	defer cancel()

	env := newStartedTestServer(t, ctx)
	s := env.server

	edgeEP, err := s.ListenA2A(ctx, A2AOptions{
		Name:        "edge-agent",
		Public:      true,
		IngressAuth: api.IngressAuthMode_INGRESS_AUTH_MODE_EDGE,
		NoAnnounce:  true,
	})
	if err != nil {
		t.Fatalf("ListenA2A(edge-agent): %v", err)
	}
	t.Cleanup(func() { _ = edgeEP.Close() })

	passEP, err := s.ListenA2A(ctx, A2AOptions{
		Name:        "pass-agent",
		Public:      true,
		IngressAuth: api.IngressAuthMode_INGRESS_AUTH_MODE_PASSTHROUGH,
		NoAnnounce:  true,
	})
	if err != nil {
		t.Fatalf("ListenA2A(pass-agent): %v", err)
	}
	t.Cleanup(func() { _ = passEP.Close() })

	type whoIsObs struct {
		ok   bool
		info *CallerInfo
	}
	var obsMu sync.Mutex
	var lastObs whoIsObs

	recordHandler := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
		info, ok := s.WhoIs(r.RemoteAddr)
		obsMu.Lock()
		lastObs = whoIsObs{ok: ok, info: info}
		obsMu.Unlock()
		w.Header().Set("Content-Type", "application/json")
		_, _ = w.Write([]byte(`{"name":"agent"}`))
	})

	edgePubSrv := &http.Server{Handler: recordHandler}
	passPubSrv := &http.Server{Handler: recordHandler}
	go func() { _ = edgePubSrv.Serve(edgeEP.PublicListener) }()
	go func() { _ = passPubSrv.Serve(passEP.PublicListener) }()
	t.Cleanup(func() {
		_ = edgePubSrv.Close()
		_ = passPubSrv.Close()
	})

	routerPeer := newAdmittedTestPeer(t, ctx, s, env.cpPriv, api.RoleRouter)
	routerBiscuit := mintTestBiscuit(t, env.cpPriv, routerPeer.ID().String(), api.RoleRouter, time.Now().Add(time.Hour))

	sendIngressReq := func(t *testing.T, method, rawURL string, hdr http.Header) {
		t.Helper()
		req, err := http.NewRequestWithContext(ctx, method, rawURL, strings.NewReader(`{}`))
		if err != nil {
			t.Fatalf("NewRequestWithContext: %v", err)
		}
		for k, vals := range hdr {
			for _, v := range vals {
				req.Header.Add(k, v)
			}
		}
		req.Header.Set(api.HeaderMeshBiscuit, base64.RawURLEncoding.EncodeToString(routerBiscuit))

		st, err := routerPeer.NewStream(ctx, s.node.Host.ID(), api.IngressProtocolID)
		if err != nil {
			t.Fatalf("NewStream: %v", err)
		}
		defer func() { _ = st.Close() }()
		if err := req.Write(st); err != nil {
			t.Fatalf("req.Write: %v", err)
		}
		_ = st.SetReadDeadline(time.Now().Add(5 * time.Second))
		resp, err := http.ReadResponse(bufio.NewReader(st), req)
		if err != nil {
			t.Fatalf("ReadResponse: %v", err)
		}
		respBody, readErr := io.ReadAll(resp.Body)
		_ = resp.Body.Close()
		if readErr != nil {
			t.Fatalf("io.ReadAll: %v", readErr)
		}
		if resp.StatusCode != http.StatusOK {
			t.Fatalf("status = %d (%s), want 200", resp.StatusCode, string(respBody))
		}
	}

	// 1. Anonymous GET /.well-known/agent-card.json in EDGE mode -> WhoIs returns (nil, false).
	sendIngressReq(t, http.MethodGet, "http://mesh/a2a/edge-agent/.well-known/agent-card.json", http.Header{
		api.HeaderMeshIngress: {api.IngressAuthModeStringEdge},
	})
	obsMu.Lock()
	anonObs := lastObs
	obsMu.Unlock()
	if anonObs.ok || anonObs.info != nil {
		t.Fatalf("anonymous agent-card WhoIs = (%+v, %v), want (nil, false)", anonObs.info, anonObs.ok)
	}

	// 2. PASSTHROUGH request -> WhoIs returns (nil, false), never router's PeerID.
	sendIngressReq(t, http.MethodPost, "http://mesh/a2a/pass-agent/invoke", http.Header{
		api.HeaderMeshIngress: {api.IngressAuthModeStringPassthrough},
		"Authorization":       {"Bearer pair-token"},
	})
	obsMu.Lock()
	passObs := lastObs
	obsMu.Unlock()
	if passObs.ok || passObs.info != nil {
		t.Fatalf("PASSTHROUGH WhoIs = (%+v, %v), want (nil, false)", passObs.info, passObs.ok)
	}

	// 3. Verified standalone external OIDC user Biscuit from /oauth/token (actor_node bound to
	// control-plane Biscuit authority signing key, not a mesh node) -> WhoIs returns ok=true with PeerID == "".
	cpAuthorityPeerID := authorityKeyPeerID(env.cpPub)
	if cpAuthorityPeerID == "" {
		t.Fatal("authorityKeyPeerID(env.cpPub) returned empty string")
	}
	userBuilder := biscuit.NewBuilder(env.cpPriv)
	for _, f := range []biscuit.Fact{
		{Predicate: biscuit.Predicate{Name: api.FactActorNode, IDs: []biscuit.Term{biscuit.String(cpAuthorityPeerID)}}},
		{Predicate: biscuit.Predicate{Name: api.FactClientPeerID, IDs: []biscuit.Term{biscuit.String(cpAuthorityPeerID)}}},
		{Predicate: biscuit.Predicate{Name: api.FactRole, IDs: []biscuit.Term{biscuit.String(api.RoleNode)}}},
		{Predicate: biscuit.Predicate{Name: api.FactEmail, IDs: []biscuit.Term{biscuit.String("[email protected]")}}},
		{Predicate: biscuit.Predicate{Name: api.FactExpiration, IDs: []biscuit.Term{biscuit.Date(time.Now().Add(time.Hour))}}},
		api.MarkerFact(api.FactTargetUnrestricted),
	} {
		if err := userBuilder.AddAuthorityFact(f); err != nil {
			t.Fatalf("AddAuthorityFact: %v", err)
		}
	}
	userTok, err := userBuilder.Build()
	if err != nil {
		t.Fatalf("Build: %v", err)
	}
	userBytes, err := userTok.Serialize()
	if err != nil {
		t.Fatalf("Serialize: %v", err)
	}
	sendIngressReq(t, http.MethodPost, "http://mesh/a2a/edge-agent/invoke", http.Header{
		api.HeaderMeshIngress:              {api.IngressAuthModeStringEdge},
		api.HeaderMeshIngressCallerBiscuit: {base64.RawURLEncoding.EncodeToString(userBytes)},
	})
	obsMu.Lock()
	userObs := lastObs
	obsMu.Unlock()
	if !userObs.ok || userObs.info == nil {
		t.Fatal("verified external OIDC user WhoIs returned (nil, false), want verified CallerInfo")
	}
	if userObs.info.Principal != "[email protected]" {
		t.Errorf("user WhoIs Principal = %q, want [email protected]", userObs.info.Principal)
	}
	if userObs.info.PeerID != "" {
		t.Errorf("user WhoIs PeerID = %q, want empty string (must not leak router %q or authority key %q)", userObs.info.PeerID, routerPeer.ID(), cpAuthorityPeerID)
	}

	// 4. Delegated Biscuit exchanged by an actual mesh actor node -> WhoIs preserves actor node PeerID.
	actorHost, err := libp2p.New(libp2p.NoListenAddrs)
	if err != nil {
		t.Fatalf("libp2p.New(actorHost): %v", err)
	}
	t.Cleanup(func() { _ = actorHost.Close() })
	actorBuilder := biscuit.NewBuilder(env.cpPriv)
	for _, f := range []biscuit.Fact{
		{Predicate: biscuit.Predicate{Name: api.FactActorNode, IDs: []biscuit.Term{biscuit.String(actorHost.ID().String())}}},
		{Predicate: biscuit.Predicate{Name: api.FactClientPeerID, IDs: []biscuit.Term{biscuit.String(actorHost.ID().String())}}},
		{Predicate: biscuit.Predicate{Name: api.FactRole, IDs: []biscuit.Term{biscuit.String(api.RoleNode)}}},
		{Predicate: biscuit.Predicate{Name: api.FactEmail, IDs: []biscuit.Term{biscuit.String("[email protected]")}}},
		{Predicate: biscuit.Predicate{Name: api.FactExpiration, IDs: []biscuit.Term{biscuit.Date(time.Now().Add(time.Hour))}}},
		api.MarkerFact(api.FactTargetUnrestricted),
	} {
		if err := actorBuilder.AddAuthorityFact(f); err != nil {
			t.Fatalf("AddAuthorityFact(actor): %v", err)
		}
	}
	actorTok, err := actorBuilder.Build()
	if err != nil {
		t.Fatalf("Build(actor): %v", err)
	}
	actorBytes, err := actorTok.Serialize()
	if err != nil {
		t.Fatalf("Serialize(actor): %v", err)
	}
	sendIngressReq(t, http.MethodPost, "http://mesh/a2a/edge-agent/invoke", http.Header{
		api.HeaderMeshIngress:              {api.IngressAuthModeStringEdge},
		api.HeaderMeshIngressCallerBiscuit: {base64.RawURLEncoding.EncodeToString(actorBytes)},
	})
	obsMu.Lock()
	actorObs := lastObs
	obsMu.Unlock()
	if !actorObs.ok || actorObs.info == nil {
		t.Fatal("actor-delegated WhoIs returned (nil, false), want verified CallerInfo")
	}
	if actorObs.info.PeerID != actorHost.ID().String() {
		t.Errorf("actor-delegated WhoIs PeerID = %q, want %q", actorObs.info.PeerID, actorHost.ID().String())
	}
}

func TestHTTPClientCanonicalizesCIDv1PeerIDAndRecoversAbort(t *testing.T) {
	t.Run("canonicalizes_cidv1_peer_id_on_egress", func(t *testing.T) {
		ctx, cancel := context.WithTimeout(context.Background(), 20*time.Second)
		defer cancel()

		mesh := newInProcessMeshPair(t, ctx)
		ep, err := mesh.provider.ListenA2A(ctx, A2AOptions{Name: "coder"})
		if err != nil {
			t.Fatalf("ListenA2A: %v", err)
		}
		srv := &http.Server{
			Handler: http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
				_, _ = w.Write([]byte("cidv1-ok"))
			}),
		}
		go func() { _ = srv.Serve(ep.PrivateListener) }()
		t.Cleanup(func() {
			_ = srv.Close()
			_ = ep.Close()
		})

		cidv1Provider := peer.ToCid(mesh.provider.peerID).String()
		if cidv1Provider == mesh.provider.peerID.String() {
			t.Fatalf("peer.ToCid(%s) did not produce distinct CIDv1 string", mesh.provider.peerID)
		}

		resp, err := mesh.caller.HTTPClient().Get(fmt.Sprintf("http://agentmesh.local/mesh/%s/a2a/coder/ping", cidv1Provider))
		if err != nil {
			t.Fatalf("GET with CIDv1 peer ID: %v", err)
		}
		body, readErr := io.ReadAll(resp.Body)
		_ = resp.Body.Close()
		if readErr != nil {
			t.Fatalf("io.ReadAll: %v", readErr)
		}
		if resp.StatusCode != http.StatusOK || string(body) != "cidv1-ok" {
			t.Fatalf("CIDv1 egress = %d %q, want 200 cidv1-ok", resp.StatusCode, string(body))
		}
	})

	t.Run("recovers_ErrAbortHandler_before_headers", func(t *testing.T) {
		req := httptest.NewRequest(http.MethodGet, "http://agentmesh.local/mesh/test", nil)
		pr, pw := io.Pipe()
		rw := newPipeResponseWriter(req, pw)

		go serveEgressSafe(&Server{}, req.URL.String(), req, rw, http.HandlerFunc(func(http.ResponseWriter, *http.Request) {
			panic(http.ErrAbortHandler)
		}))

		resp, err := rw.waitResponse(pr)
		if !errors.Is(err, io.ErrUnexpectedEOF) {
			t.Fatalf("waitResponse err = %v (resp=%v), want io.ErrUnexpectedEOF", err, resp)
		}
	})

	t.Run("recovers_ErrAbortHandler_mid_stream", func(t *testing.T) {
		req := httptest.NewRequest(http.MethodGet, "http://agentmesh.local/mesh/test", nil)
		pr, pw := io.Pipe()
		rw := newPipeResponseWriter(req, pw)

		go serveEgressSafe(&Server{}, req.URL.String(), req, rw, http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
			w.WriteHeader(http.StatusOK)
			_, _ = w.Write([]byte("partial-chunk"))
			panic(http.ErrAbortHandler)
		}))

		resp, err := rw.waitResponse(pr)
		if err != nil {
			t.Fatalf("waitResponse after headers: %v", err)
		}
		body, readErr := io.ReadAll(resp.Body)
		_ = resp.Body.Close()
		if string(body) != "partial-chunk" {
			t.Errorf("partial body = %q, want partial-chunk", string(body))
		}
		if !errors.Is(readErr, io.ErrUnexpectedEOF) {
			t.Fatalf("io.ReadAll after mid-stream abort = %v, want io.ErrUnexpectedEOF", readErr)
		}
	})

	t.Run("recovers_unexpected_handler_panic", func(t *testing.T) {
		req := httptest.NewRequest(http.MethodGet, "http://agentmesh.local/mesh/test", nil)
		pr, pw := io.Pipe()
		rw := newPipeResponseWriter(req, pw)

		go serveEgressSafe(&Server{}, req.URL.String(), req, rw, http.HandlerFunc(func(http.ResponseWriter, *http.Request) {
			panic("unexpected-boom")
		}))

		resp, err := rw.waitResponse(pr)
		if err == nil || !strings.Contains(err.Error(), "unexpected-boom") {
			t.Fatalf("waitResponse on unexpected panic = (%v, %v), want error containing unexpected-boom", resp, err)
		}
	})
}

func TestServerConcurrentStatusMetricsAndClose(t *testing.T) {
	ctx, cancel := context.WithTimeout(context.Background(), 20*time.Second)
	defer cancel()

	env := newStartedTestServer(t, ctx)
	reg := prometheus.NewRegistry()
	s := &Server{
		ControlPlaneURL: env.cp.srv.URL,
		BootstrapToken:  "boot-secret",
		ListenAddrs:     []string{"/ip4/127.0.0.1/tcp/0"},
		AllowInsecure:   true,
		Registerer:      reg,
	}
	if _, err := s.Up(ctx); err != nil {
		t.Fatalf("s.Up: %v", err)
	}
	t.Cleanup(func() { _ = s.Close() })

	var wg sync.WaitGroup
	for range 4 {
		wg.Add(1)
		go func() {
			defer wg.Done()
			for range 50 {
				_ = s.Status()
				_, _ = reg.Gather()
				_, _, _ = s.AttenuateToken(&api.TaskAuthorizationRule{Name: "race-check"}, time.Minute, false)
				s.onStreamRefused("unauthenticated", string(api.IngressProtocolID))
			}
		}()
	}
	wg.Add(1)
	go func() {
		defer wg.Done()
		time.Sleep(5 * time.Millisecond)
		_ = s.Close()
	}()
	wg.Wait()
}

type inProcessMeshPair struct {
	provider *Server
	caller   *Server
}

func newInProcessMeshPair(t *testing.T, ctx context.Context) *inProcessMeshPair {
	t.Helper()
	cpPub, cpPriv, err := ed25519.GenerateKey(rand.Reader)
	if err != nil {
		t.Fatal(err)
	}

	rh, err := libp2p.New(libp2p.ListenAddrStrings("/ip4/127.0.0.1/tcp/0"))
	if err != nil {
		t.Fatal(err)
	}
	t.Cleanup(func() { _ = rh.Close() })

	var dirMu sync.Mutex
	providersByKey := map[string]map[peer.ID]*api.PeerRecord{}
	peersByID := map[peer.ID]*api.PeerRecord{}

	routerBiscuit := mintTestBiscuit(t, cpPriv, rh.ID().String(), api.RoleRouter, time.Now().Add(24*time.Hour))
	rh.SetStreamHandler(api.AuthProtocolID, func(st network.Stream) {
		defer func() { _ = st.Close() }()
		reader := msgio.NewVarintReaderSize(st, 64*1024)
		msg, err := reader.ReadMsg()
		if err != nil {
			return
		}
		reader.ReleaseMsg(msg)
		reply, _ := proto.Marshal(&api.AuthResponse{Success: true, Biscuit: routerBiscuit})
		_ = msgio.NewVarintWriter(st).WriteMsg(reply)
	})
	rh.SetStreamHandler(api.DirectoryProtocolID, func(st network.Stream) {
		defer func() { _ = st.Close() }()
		remote := st.Conn().RemotePeer()

		reader := msgio.NewVarintReaderSize(st, 64*1024)
		msg, err := reader.ReadMsg()
		if err != nil {
			return
		}
		var req api.DirectoryRequest
		_ = proto.Unmarshal(msg, &req)
		reader.ReleaseMsg(msg)

		resp := &api.DirectoryResponse{}
		dirMu.Lock()
		entry := peersByID[remote]
		if entry == nil {
			var addrs []string
			for _, ma := range rh.Peerstore().Addrs(remote) {
				addrs = append(addrs, ma.String())
			}
			entry = &api.PeerRecord{PeerId: remote.String(), Addrs: addrs}
			peersByID[remote] = entry
		}
		switch op := req.GetRequest().(type) {
		case *api.DirectoryRequest_Provide:
			k := string(op.Provide.GetKey())
			if providersByKey[k] == nil {
				providersByKey[k] = map[peer.ID]*api.PeerRecord{}
			}
			providersByKey[k][remote] = entry
		case *api.DirectoryRequest_Withdraw:
			delete(providersByKey[string(op.Withdraw.GetKey())], remote)
		case *api.DirectoryRequest_FindProviders:
			for _, p := range providersByKey[string(op.FindProviders.GetKey())] {
				resp.Providers = append(resp.Providers, p)
			}
		case *api.DirectoryRequest_FindPeer:
			if pid, err := peer.Decode(op.FindPeer.GetPeerId()); err == nil {
				resp.Peer = peersByID[pid]
			}
		}
		dirMu.Unlock()

		reply, _ := proto.Marshal(resp)
		_ = msgio.NewVarintWriter(st).WriteMsg(reply)
	})

	var routerAddrs []string
	for _, a := range rh.Addrs() {
		routerAddrs = append(routerAddrs, a.String()+"/p2p/"+rh.ID().String())
	}
	cp := newFakeControlPlaneHTTPWithKeys(t, cpPub, cpPriv, routerAddrs)

	startOne := func() *Server {
		s := &Server{
			StateDir:        t.TempDir(),
			ControlPlaneURL: cp.srv.URL,
			BootstrapToken:  "boot-secret",
			ListenAddrs:     []string{"/ip4/127.0.0.1/tcp/0"},
			AllowInsecure:   true,
		}
		if _, err := s.Up(ctx); err != nil {
			t.Fatalf("Server.Up: %v", err)
		}
		t.Cleanup(func() { _ = s.Close() })
		return s
	}

	p := startOne()
	c := startOne()
	// Ensure router peerstore has listen multiaddrs of both peers for FindProviders/FindPeer.
	dirMu.Lock()
	for _, srv := range []*Server{p, c} {
		var maddrs []string
		for _, ma := range srv.node.Host.Addrs() {
			maddrs = append(maddrs, ma.String())
		}
		peersByID[srv.node.Host.ID()] = &api.PeerRecord{PeerId: srv.node.Host.ID().String(), Addrs: maddrs}
	}
	dirMu.Unlock()

	return &inProcessMeshPair{provider: p, caller: c}
}

// TestDeliverHTTPHandlerThatIgnoresBody drives deliverHTTP over its net.Pipe
// against a handler that answers without reading the request body, as an
// agent-card GET or an early refusal does. The transport must keep the
// connection alive so net/http's server drains the body before it closes;
// with Connection: close the server closed first and the transport's body
// write raced the response, surfacing as a 502 under load.
func TestDeliverHTTPHandlerThatIgnoresBody(t *testing.T) {
	s := &Server{}
	ln := newInprocListener("meshnet:test:ignore-body")
	srv := &http.Server{Handler: http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
		w.Header().Set("Content-Type", "application/json")
		_, _ = w.Write([]byte(`{"ok":true}`))
	})}
	go func() { _ = srv.Serve(ln) }()
	t.Cleanup(func() {
		_ = srv.Close()
		_ = ln.Close()
	})

	body := bytes.Repeat([]byte("x"), 64<<10)
	caller := &CallerInfo{PeerID: "12D3KooWTestPeer", Principal: "node:test"}
	for i := 0; i < 200; i++ {
		req := httptest.NewRequest(http.MethodPost, "http://agentmesh.local/invoke", bytes.NewReader(body))
		rec := httptest.NewRecorder()
		s.deliverHTTP(rec, req, ln, caller, nil)
		if rec.Code != http.StatusOK {
			t.Fatalf("request %d: status = %d, body = %q, want 200", i, rec.Code, rec.Body.String())
		}
	}

	deadline := time.Now().Add(2 * time.Second)
	for time.Now().Before(deadline) {
		live := 0
		s.activeConns.Range(func(_, _ any) bool { live++; return true })
		if live == 0 {
			return
		}
		time.Sleep(10 * time.Millisecond)
	}
	s.activeConns.Range(func(k, _ any) bool {
		t.Errorf("WhoIs entry %v still present after its connection closed", k)
		return true
	})
}