Agent Mesh
Go a2aa2a-protocolagent-meshagentic-aiinferencemcpmeshmesh-networknetwork-securitynetworking-aip2p-network
feat(node): provider scorer with region constraints and retry-next-best
Antonio Ojeac2159e3b04c8
diff --git a/internal/node/inference_metrics.go b/internal/node/inference_metrics.goindex 84d3af99..66ac8e37 100644--- a/internal/node/inference_metrics.go+++ b/internal/node/inference_metrics.go@@ -27,8 +27,32 @@ var (},[]string{"peer_id", "model", "token_type"},)++ facadeRejectionsTotal = promauto.NewCounterVec(+ prometheus.CounterOpts{+ Name: "sam_node_facade_rejections_total",+ Help: "Providers excluded by the OpenAI facade scorer, by reason",+ },+ []string{"reason"},+ )++ facadeRetriesTotal = promauto.NewCounter(+ prometheus.CounterOpts{+ Name: "sam_node_facade_retries_total",+ Help: "Completion attempts retried on another provider after a retryable failure",+ },+ ))+// recordFacadeRejection accounts one excluded provider by reason.+func recordFacadeRejection(reason string) {+ facadeRejectionsTotal.WithLabelValues(reason).Inc()+}++func recordFacadeRetry() {+ facadeRetriesTotal.Inc()+}+// recordTokens increments the prompt and completion token metrics.func recordTokens(peerID, model string, promptTokens, completionTokens int) {if model == "" {diff --git a/internal/node/openai_facade.go b/internal/node/openai_facade.goindex 14d90e2c..939cde91 100644--- a/internal/node/openai_facade.go+++ b/internal/node/openai_facade.go@@ -51,10 +51,13 @@ type modelLister interface {}// modelProvider locates one service able to serve a model.-// An empty peerID means the service runs on this node.+// An empty peerID means the service runs on this node. region and active+// are known only for gossip-observed providers; zero values mean unknown.type modelProvider struct {peerID stringservice string+ region string+ active uint32}// openAIFacade exposes an OpenAI-compatible surface on the sidecar@@ -72,11 +75,17 @@ type openAIFacade struct {// and interest registration so announcements start flowing for a model.viewProviders func(model string) []modelProviderensureInterest func(model string)+ // Scorer seams (may be nil): revocation check and this node's region.+ isRevoked func(peerID string) bool+ localRegion func() stringttl time.Durationmu sync.Mutexmodels map[string][]modelProviderexpires time.Time++ backoffMu sync.Mutex+ backoff map[string]time.Time}func newOpenAIFacade(node *SamNode, egress http.Handler) *openAIFacade {@@ -106,7 +115,12 @@ func newOpenAIFacade(node *SamNode, egress http.Handler) *openAIFacade {}var out []modelProviderfor _, p := range node.Discovery.Providers(api.ServiceType_SERVICE_TYPE_INFERENCE, model) {- out = append(out, modelProvider{peerID: p.PeerID, service: p.Service})+ out = append(out, modelProvider{+ peerID: p.PeerID,+ service: p.Service,+ region: p.Labels[api.LabelRegion],+ active: p.Load.ActiveRequests,+ })}return out},@@ -115,6 +129,10 @@ func newOpenAIFacade(node *SamNode, egress http.Handler) *openAIFacade {node.Discovery.Ensure(api.ServiceType_SERVICE_TYPE_INFERENCE, model)}},+ isRevoked: func(peerID string) bool {+ return node.revokedPeers != nil && node.revokedPeers.Contains(peerID)+ },+ localRegion: func() string { return node.config.Region },}}@@ -262,18 +280,9 @@ func (f *openAIFacade) collectModels(ctx context.Context) map[string][]modelProvreturn models}-// facadeRR spreads load across remote providers of the same model.+// facadeRR spreads load across equally ranked remote providers.var facadeRR atomic.Uint64-// pickProvider prefers a local provider (locals sort first in collectModels),-// otherwise round-robins across remotes.-func pickProvider(providers []modelProvider) modelProvider {- if providers[0].peerID == "" {- return providers[0]- }- return providers[facadeRR.Add(1)%uint64(len(providers))]-}-func (f *openAIFacade) handleModels(w http.ResponseWriter, r *http.Request) {if r.Method != http.MethodGet {writeOpenAIError(w, http.StatusMethodNotAllowed, "method_not_allowed", "only GET is supported")@@ -335,25 +344,53 @@ func (f *openAIFacade) handleCompletions(w http.ResponseWriter, r *http.Request)fmt.Sprintf("model %q is not served by any provider on the mesh", req.Model))return}- p := pickProvider(providers)- logger.Debugf("[OpenAIFacade] model %q -> provider peer=%q service=%q (%d candidates)",- req.Model, p.peerID, p.service, len(providers))++ requiredRegions := parseRequiredRegions(r.Header.Get(api.HeaderSamRequiredRegion))+ ranked := f.rankProviders(providers, requiredRegions)+ if len(ranked) == 0 {+ recordFacadeRejection(reasonNoEligible)+ writeOpenAIError(w, http.StatusServiceUnavailable, "no_eligible_provider",+ fmt.Sprintf("model %q has no provider satisfying the request constraints", req.Model))+ return+ }// The gate already stripped the credential it consumed; a surviving// Authorization header is the destination backend's own credential and// passes through, mirroring the egress path. X-Sam-Authentication is- // local-only and must never travel off-node.+ // local-only, and the region requirement is an instruction to this+ // sidecar, not to the provider.r.Header.Del(api.HeaderSamAuthentication)- r.Body = io.NopCloser(bytes.NewReader(body))- r.ContentLength = int64(len(body))+ r.Header.Del(api.HeaderSamRequiredRegion)- if p.peerID == "" {- f.serveLocal(w, r, p.service)- return+ maxAttempts := min(3, len(ranked))+ for i := range maxAttempts {+ p := ranked[i]+ logger.Debugf("[OpenAIFacade] model %q attempt %d/%d -> provider peer=%q service=%q (%d eligible)",+ req.Model, i+1, maxAttempts, p.peerID, p.service, len(ranked))++ attempt := r.Clone(r.Context())+ attempt.Body = io.NopCloser(bytes.NewReader(body))+ attempt.ContentLength = int64(len(body))+ aw := newAttemptWriter(w)++ if p.peerID == "" {+ f.serveLocal(aw, attempt, p.service)+ } else {+ attempt.URL.Path = fmt.Sprintf("/sam/%s/%s/%s%s", p.peerID, api.ServiceTypeStringInference, p.service, r.URL.Path)+ attempt.URL.RawPath = ""+ f.forward.ServeHTTP(aw, attempt)+ }+ if !aw.retryable {+ return // response delivered (success or non-retryable error)+ }+ recordFacadeRetry()+ f.recordBackoff(p)+ logger.Warnf("[OpenAIFacade] provider peer=%q service=%q returned %d for model %q; trying next",+ p.peerID, p.service, aw.status, req.Model)}- r.URL.Path = fmt.Sprintf("/sam/%s/%s/%s%s", p.peerID, api.ServiceTypeStringInference, p.service, r.URL.Path)- r.URL.RawPath = ""- f.forward.ServeHTTP(w, r)+ recordFacadeRejection(reasonAttemptsExceeded)+ writeOpenAIError(w, http.StatusServiceUnavailable, "no_available_provider",+ fmt.Sprintf("all providers for model %q are currently unavailable", req.Model))}func (f *openAIFacade) serveLocal(w http.ResponseWriter, r *http.Request, serviceName string) {diff --git a/internal/node/openai_facade_test.go b/internal/node/openai_facade_test.goindex 462ba237..281f9637 100644--- a/internal/node/openai_facade_test.go+++ b/internal/node/openai_facade_test.go@@ -382,20 +382,240 @@ func TestFacade_EmptyRegistryIsNotCached(t *testing.T) {}}-func TestPickProvider(t *testing.T) {+func TestRankProviders(t *testing.T) {local := modelProvider{service: "local-llm"}- remoteA := modelProvider{peerID: "peerA", service: "srvA"}- remoteB := modelProvider{peerID: "peerB", service: "srvB"}+ remoteEU := modelProvider{peerID: "peerEU", service: "srv", region: "eu", active: 5}+ remoteUS := modelProvider{peerID: "peerUS", service: "srv", region: "us", active: 0}+ remoteNoLabel := modelProvider{peerID: "peerX", service: "srv", active: 1}++ t.Run("locals first, remotes by ascending load", func(t *testing.T) {+ f := newTestFacade()+ got := f.rankProviders([]modelProvider{remoteEU, remoteUS, local}, nil)+ if len(got) != 3 || got[0].service != "local-llm" || got[1].peerID != "peerUS" || got[2].peerID != "peerEU" {+ t.Fatalf("unexpected ranking: %+v", got)+ }+ })- if got := pickProvider([]modelProvider{local, remoteA}); got != local {- t.Errorf("pickProvider with local first: got %+v, want local", got)+ t.Run("region requirement is fail-closed", func(t *testing.T) {+ f := newTestFacade()+ got := f.rankProviders([]modelProvider{remoteEU, remoteUS, remoteNoLabel, local}, []string{"eu"})+ // Unlabeled remote and unlabeled local must both be excluded.+ if len(got) != 1 || got[0].peerID != "peerEU" {+ t.Fatalf("unexpected ranking under region requirement: %+v", got)+ }+ })++ t.Run("local matches its declared region", func(t *testing.T) {+ f := newTestFacade()+ f.localRegion = func() string { return "eu" }+ got := f.rankProviders([]modelProvider{local, remoteUS}, []string{"eu"})+ if len(got) != 1 || got[0].service != "local-llm" {+ t.Fatalf("local with matching region should survive: %+v", got)+ }+ })++ t.Run("revoked peers are excluded", func(t *testing.T) {+ f := newTestFacade()+ f.isRevoked = func(peerID string) bool { return peerID == "peerUS" }+ got := f.rankProviders([]modelProvider{remoteEU, remoteUS}, nil)+ if len(got) != 1 || got[0].peerID != "peerEU" {+ t.Fatalf("revoked peer should be excluded: %+v", got)+ }+ })++ t.Run("backoff excludes and expires", func(t *testing.T) {+ f := newTestFacade()+ f.recordBackoff(remoteUS)+ if got := f.rankProviders([]modelProvider{remoteUS}, nil); len(got) != 0 {+ t.Fatalf("backed-off provider should be excluded: %+v", got)+ }+ f.backoffMu.Lock()+ f.backoff[backoffKey(remoteUS)] = time.Now().Add(-time.Second)+ f.backoffMu.Unlock()+ if got := f.rankProviders([]modelProvider{remoteUS}, nil); len(got) != 1 {+ t.Fatal("expired backoff should be cleared")+ }+ })++ t.Run("equal load rotates", func(t *testing.T) {+ f := newTestFacade()+ a := modelProvider{peerID: "peerA", service: "srv"}+ b := modelProvider{peerID: "peerB", service: "srv"}+ seen := map[string]bool{}+ for range 10 {+ seen[f.rankProviders([]modelProvider{a, b}, nil)[0].peerID] = true+ }+ if !seen["peerA"] || !seen["peerB"] {+ t.Errorf("rotation should spread across equals, got %v", seen)+ }+ })+}++func TestParseRequiredRegions(t *testing.T) {+ if got := parseRequiredRegions(""); got != nil {+ t.Errorf("empty header: got %v, want nil", got)+ }+ if got := parseRequiredRegions(" eu , us ,,"); len(got) != 2 || got[0] != "eu" || got[1] != "us" {+ t.Errorf("parse: got %v", got)+ }+}++func TestFacade_Completions_RetriesNextProvider(t *testing.T) {+ f := newTestFacade()+ f.discover = func(_ context.Context) ([]*api.DiscoveredProvider, error) {+ return []*api.DiscoveredProvider{+ {PeerId: "peerA", SrvName: "srvA"},+ {PeerId: "peerB", SrvName: "srvB"},+ }, nil}+ f.remoteModels = func(_ context.Context, _, _ string) ([]string, error) {+ return []string{"m1"}, nil+ }+ var attempts []string+ f.forward = http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {+ attempts = append(attempts, r.URL.Path)+ if len(attempts) == 1 {+ http.Error(w, "overloaded", http.StatusServiceUnavailable)+ return+ }+ w.WriteHeader(http.StatusOK)+ _, _ = w.Write([]byte(`{"ok":true}`))+ })- seen := map[string]bool{}- for range 10 {- seen[pickProvider([]modelProvider{remoteA, remoteB}).peerID] = true+ req := httptest.NewRequest(http.MethodPost, "/v1/chat/completions", strings.NewReader(`{"model":"m1"}`))+ rec := httptest.NewRecorder()+ f.handleCompletions(rec, req)++ if rec.Code != http.StatusOK {+ t.Fatalf("status: got %d, want %d (attempts: %v)", rec.Code, http.StatusOK, attempts)+ }+ if len(attempts) != 2 {+ t.Fatalf("attempts: got %v, want 2", attempts)+ }+ // The failed provider must be in backoff now.+ f.backoffMu.Lock()+ n := len(f.backoff)+ f.backoffMu.Unlock()+ if n != 1 {+ t.Errorf("backoff entries: got %d, want 1", n)}- if !seen["peerA"] || !seen["peerB"] {- t.Errorf("round-robin should hit both remotes, got %v", seen)+}++func TestFacade_Completions_AllProvidersFail(t *testing.T) {+ f := newTestFacade()+ f.discover = func(_ context.Context) ([]*api.DiscoveredProvider, error) {+ return []*api.DiscoveredProvider{{PeerId: "peerA", SrvName: "srvA"}}, nil}+ f.remoteModels = func(_ context.Context, _, _ string) ([]string, error) {+ return []string{"m1"}, nil+ }+ f.forward = http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {+ http.Error(w, "down", http.StatusBadGateway)+ })++ req := httptest.NewRequest(http.MethodPost, "/v1/chat/completions", strings.NewReader(`{"model":"m1"}`))+ rec := httptest.NewRecorder()+ f.handleCompletions(rec, req)++ if rec.Code != http.StatusServiceUnavailable {+ t.Fatalf("status: got %d, want %d", rec.Code, http.StatusServiceUnavailable)+ }+ if !strings.Contains(rec.Body.String(), "no_available_provider") {+ t.Errorf("body missing no_available_provider: %s", rec.Body.String())+ }+}++func TestFacade_Completions_RegionRequirement(t *testing.T) {+ f := newTestFacade()+ f.viewProviders = func(model string) []modelProvider {+ return []modelProvider{+ {peerID: "peerEU", service: "srvEU", region: "eu"},+ {peerID: "peerUS", service: "srvUS", region: "us"},+ }+ }+ var gotPath, gotRegionHeader string+ f.forward = http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {+ gotPath = r.URL.Path+ gotRegionHeader = r.Header.Get(api.HeaderSamRequiredRegion)+ })++ req := httptest.NewRequest(http.MethodPost, "/v1/chat/completions", strings.NewReader(`{"model":"m1"}`))+ req.Header.Set(api.HeaderSamRequiredRegion, "eu")+ rec := httptest.NewRecorder()+ f.handleCompletions(rec, req)++ if want := "/sam/peerEU/inference/srvEU/v1/chat/completions"; gotPath != want {+ t.Errorf("forwarded path: got %q, want %q", gotPath, want)+ }+ if gotRegionHeader != "" {+ t.Errorf("%s must not travel to the provider, got %q", api.HeaderSamRequiredRegion, gotRegionHeader)+ }++ // No provider satisfies the requirement.+ req = httptest.NewRequest(http.MethodPost, "/v1/chat/completions", strings.NewReader(`{"model":"m1"}`))+ req.Header.Set(api.HeaderSamRequiredRegion, "mars")+ rec = httptest.NewRecorder()+ f.handleCompletions(rec, req)+ if rec.Code != http.StatusServiceUnavailable || !strings.Contains(rec.Body.String(), "no_eligible_provider") {+ t.Errorf("unsatisfiable region: got status %d, body %s", rec.Code, rec.Body.String())+ }+}++func TestAttemptWriter(t *testing.T) {+ t.Run("retryable status swallowed", func(t *testing.T) {+ rec := httptest.NewRecorder()+ aw := newAttemptWriter(rec)+ aw.Header().Set("Content-Type", "application/json")+ aw.WriteHeader(http.StatusServiceUnavailable)+ _, _ = aw.Write([]byte("error body"))+ if !aw.retryable {+ t.Fatal("expected retryable")+ }+ if rec.Body.Len() != 0 || rec.Header().Get("Content-Type") != "" {+ t.Errorf("nothing must reach the client on a retryable status, got body %q", rec.Body.String())+ }+ })++ t.Run("success streams through with headers", func(t *testing.T) {+ rec := httptest.NewRecorder()+ aw := newAttemptWriter(rec)+ aw.Header().Set("Content-Type", "text/event-stream")+ aw.WriteHeader(http.StatusOK)+ _, _ = aw.Write([]byte("data: x\n\n"))+ aw.Flush()+ if aw.retryable {+ t.Fatal("must not be retryable")+ }+ if rec.Code != http.StatusOK || rec.Body.String() != "data: x\n\n" {+ t.Errorf("unexpected passthrough: code %d, body %q", rec.Code, rec.Body.String())+ }+ if rec.Header().Get("Content-Type") != "text/event-stream" {+ t.Errorf("headers must be copied, got %v", rec.Header())+ }+ if !rec.Flushed {+ t.Error("flush must reach the destination")+ }+ })++ t.Run("implicit 200 on first write", func(t *testing.T) {+ rec := httptest.NewRecorder()+ aw := newAttemptWriter(rec)+ _, _ = aw.Write([]byte("hello"))+ if rec.Code != http.StatusOK || rec.Body.String() != "hello" {+ t.Errorf("implicit 200: code %d, body %q", rec.Code, rec.Body.String())+ }+ })++ t.Run("non-retryable error passes through", func(t *testing.T) {+ rec := httptest.NewRecorder()+ aw := newAttemptWriter(rec)+ aw.WriteHeader(http.StatusNotFound)+ _, _ = aw.Write([]byte("nope"))+ if aw.retryable {+ t.Fatal("404 must not be retryable")+ }+ if rec.Code != http.StatusNotFound || rec.Body.String() != "nope" {+ t.Errorf("passthrough: code %d, body %q", rec.Code, rec.Body.String())+ }+ })}diff --git a/internal/node/openai_scorer.go b/internal/node/openai_scorer.gonew file mode 100644index 00000000..6dda6167--- /dev/null+++ b/internal/node/openai_scorer.go@@ -0,0 +1,198 @@+// Copyright 2026 Google LLC+//+// Licensed under the Apache License, Version 2.0 (the "License");+// you may not use this file except in compliance with the License.+// You may obtain a copy of the License at+//+// http://www.apache.org/licenses/LICENSE-2.0+//+// Unless required by applicable law or agreed to in writing, software+// distributed under the License is distributed on an "AS IS" BASIS,+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+// See the License for the specific language governing permissions and+// limitations under the License.++package node++import (+ "net/http"+ "slices"+ "sort"+ "strings"+ "time"+)++// providerBackoff is how long a provider is skipped after a retryable failure.+const providerBackoff = 15 * time.Second++// Rejection reasons for facade provider filtering (metric label values).+const (+ reasonPeerRevoked = "peer_revoked"+ reasonProviderBackoff = "provider_backoff"+ reasonRegionMismatch = "region_mismatch"+ reasonNoEligible = "no_eligible_provider"+ reasonAttemptsExceeded = "attempts_exceeded"+)++// parseRequiredRegions splits the X-Sam-Required-Region header value.+func parseRequiredRegions(h string) []string {+ if h == "" {+ return nil+ }+ var out []string+ for _, part := range strings.Split(h, ",") {+ if p := strings.TrimSpace(part); p != "" {+ out = append(out, p)+ }+ }+ return out+}++// rankProviders applies hard constraints then orders the survivors: eligible+// locals first, then remotes by ascending advertised load. Providers with no+// load information look idle — without telemetry this degrades to rotation+// among equals. Every exclusion is accounted per reason.+func (f *openAIFacade) rankProviders(providers []modelProvider, requiredRegions []string) []modelProvider {+ var locals, remotes []modelProvider+ for _, p := range providers {+ region := p.region+ if p.peerID == "" && f.localRegion != nil {+ region = f.localRegion()+ }+ // Region is fail-closed: a requirement never matches an unlabeled+ // provider. Labels are routing hints until biscuit-attested.+ if len(requiredRegions) > 0 && !slices.Contains(requiredRegions, region) {+ recordFacadeRejection(reasonRegionMismatch)+ continue+ }+ if p.peerID == "" {+ locals = append(locals, p)+ continue+ }+ if f.isRevoked != nil && f.isRevoked(p.peerID) {+ recordFacadeRejection(reasonPeerRevoked)+ continue+ }+ if f.inBackoff(p) {+ recordFacadeRejection(reasonProviderBackoff)+ continue+ }+ remotes = append(remotes, p)+ }++ sort.SliceStable(remotes, func(i, j int) bool { return remotes[i].active < remotes[j].active })+ rotateEqualHead(remotes)+ return append(locals, remotes...)+}++// rotateEqualHead rotates the group of equally loaded head providers so+// repeated requests spread across them instead of hammering the first.+func rotateEqualHead(remotes []modelProvider) {+ if len(remotes) < 2 {+ return+ }+ group := 1+ for group < len(remotes) && remotes[group].active == remotes[0].active {+ group+++ }+ if group < 2 {+ return+ }+ offset := int(facadeRR.Add(1) % uint64(group))+ head := append([]modelProvider{}, remotes[:group]...)+ for i := range group {+ remotes[i] = head[(i+offset)%group]+ }+}++func backoffKey(p modelProvider) string { return p.peerID + "|" + p.service }++func (f *openAIFacade) inBackoff(p modelProvider) bool {+ f.backoffMu.Lock()+ defer f.backoffMu.Unlock()+ until, ok := f.backoff[backoffKey(p)]+ if !ok {+ return false+ }+ if time.Now().After(until) {+ delete(f.backoff, backoffKey(p))+ return false+ }+ return true+}++func (f *openAIFacade) recordBackoff(p modelProvider) {+ if p.peerID == "" {+ return // locals are not backed off; the service registry owns their lifecycle+ }+ f.backoffMu.Lock()+ defer f.backoffMu.Unlock()+ if f.backoff == nil {+ f.backoff = map[string]time.Time{}+ }+ f.backoff[backoffKey(p)] = time.Now().Add(providerBackoff)+}++// retryableStatus reports provider responses that are safe to retry+// elsewhere: the provider produced no model output.+func retryableStatus(code int) bool {+ switch code {+ case http.StatusTooManyRequests, http.StatusBadGateway,+ http.StatusServiceUnavailable, http.StatusGatewayTimeout:+ return true+ }+ return false+}++// attemptWriter defers committing to the client until the provider's status+// is known: retryable statuses are swallowed (body discarded) so the request+// can be retried on the next-ranked provider; anything else streams through.+type attemptWriter struct {+ dst http.ResponseWriter+ header http.Header+ status int+ retryable bool+ started bool+}++func newAttemptWriter(dst http.ResponseWriter) *attemptWriter {+ return &attemptWriter{dst: dst, header: http.Header{}}+}++func (a *attemptWriter) Header() http.Header { return a.header }++func (a *attemptWriter) WriteHeader(code int) {+ if a.started || a.retryable {+ return+ }+ a.status = code+ if retryableStatus(code) {+ a.retryable = true+ return+ }+ dst := a.dst.Header()+ for k, vv := range a.header {+ dst[k] = vv+ }+ a.dst.WriteHeader(code)+ a.started = true+}++func (a *attemptWriter) Write(p []byte) (int, error) {+ if a.retryable {+ return len(p), nil // discard failed attempt's error body+ }+ if !a.started {+ a.WriteHeader(http.StatusOK)+ }+ return a.dst.Write(p)+}++func (a *attemptWriter) Flush() {+ if !a.started {+ return+ }+ if fl, ok := a.dst.(http.Flusher); ok {+ fl.Flush()+ }+}diff --git a/tests/integration/catalog_test.go b/tests/integration/catalog_test.goindex ad88f67d..7c856a2c 100644--- a/tests/integration/catalog_test.go+++ b/tests/integration/catalog_test.go@@ -50,7 +50,8 @@ func startBackgroundNode(t *testing.T, nodeBin string, routerAddr string, homeDifunc waitForMCPAddr(t *testing.T, logPath string) string {t.Helper()- deadline := time.Now().Add(5 * time.Second)+ // Generous under CI load; polling returns as soon as the line appears.+ deadline := time.Now().Add(10 * time.Second)for time.Now().Before(deadline) {data, _ := os.ReadFile(logPath)lines := strings.Split(string(data), "\n")@@ -64,7 +65,8 @@ func waitForMCPAddr(t *testing.T, logPath string) string {}time.Sleep(100 * time.Millisecond)}- t.Fatalf("timeout waiting for MCP addr in log: %s", logPath)+ data, _ := os.ReadFile(logPath)+ t.Fatalf("timeout waiting for MCP addr in log: %s\n--- log contents ---\n%s", logPath, string(data))return ""}