Skip to content

Agent Mesh

feat(node): provider scorer with region constraints and retry-next-best

Antonio Ojeac2159e3b04c8
diff --git a/internal/node/inference_metrics.go b/internal/node/inference_metrics.go
index 84d3af99..66ac8e37 100644
--- a/internal/node/inference_metrics.go
+++ b/internal/node/inference_metrics.go
@@ -27,8 +27,32 @@ var (
},
[]string{"peer_id", "model", "token_type"},
)
+
+ facadeRejectionsTotal = promauto.NewCounterVec(
+ prometheus.CounterOpts{
+ Name: "sam_node_facade_rejections_total",
+ Help: "Providers excluded by the OpenAI facade scorer, by reason",
+ },
+ []string{"reason"},
+ )
+
+ facadeRetriesTotal = promauto.NewCounter(
+ prometheus.CounterOpts{
+ Name: "sam_node_facade_retries_total",
+ Help: "Completion attempts retried on another provider after a retryable failure",
+ },
+ )
)
+// recordFacadeRejection accounts one excluded provider by reason.
+func recordFacadeRejection(reason string) {
+ facadeRejectionsTotal.WithLabelValues(reason).Inc()
+}
+
+func recordFacadeRetry() {
+ facadeRetriesTotal.Inc()
+}
+
// recordTokens increments the prompt and completion token metrics.
func recordTokens(peerID, model string, promptTokens, completionTokens int) {
if model == "" {
diff --git a/internal/node/openai_facade.go b/internal/node/openai_facade.go
index 14d90e2c..939cde91 100644
--- a/internal/node/openai_facade.go
+++ b/internal/node/openai_facade.go
@@ -51,10 +51,13 @@ type modelLister interface {
}
// modelProvider locates one service able to serve a model.
-// An empty peerID means the service runs on this node.
+// An empty peerID means the service runs on this node. region and active
+// are known only for gossip-observed providers; zero values mean unknown.
type modelProvider struct {
peerID string
service string
+ region string
+ active uint32
}
// openAIFacade exposes an OpenAI-compatible surface on the sidecar
@@ -72,11 +75,17 @@ type openAIFacade struct {
// and interest registration so announcements start flowing for a model.
viewProviders func(model string) []modelProvider
ensureInterest func(model string)
+ // Scorer seams (may be nil): revocation check and this node's region.
+ isRevoked func(peerID string) bool
+ localRegion func() string
ttl time.Duration
mu sync.Mutex
models map[string][]modelProvider
expires time.Time
+
+ backoffMu sync.Mutex
+ backoff map[string]time.Time
}
func newOpenAIFacade(node *SamNode, egress http.Handler) *openAIFacade {
@@ -106,7 +115,12 @@ func newOpenAIFacade(node *SamNode, egress http.Handler) *openAIFacade {
}
var out []modelProvider
for _, p := range node.Discovery.Providers(api.ServiceType_SERVICE_TYPE_INFERENCE, model) {
- out = append(out, modelProvider{peerID: p.PeerID, service: p.Service})
+ out = append(out, modelProvider{
+ peerID: p.PeerID,
+ service: p.Service,
+ region: p.Labels[api.LabelRegion],
+ active: p.Load.ActiveRequests,
+ })
}
return out
},
@@ -115,6 +129,10 @@ func newOpenAIFacade(node *SamNode, egress http.Handler) *openAIFacade {
node.Discovery.Ensure(api.ServiceType_SERVICE_TYPE_INFERENCE, model)
}
},
+ isRevoked: func(peerID string) bool {
+ return node.revokedPeers != nil && node.revokedPeers.Contains(peerID)
+ },
+ localRegion: func() string { return node.config.Region },
}
}
@@ -262,18 +280,9 @@ func (f *openAIFacade) collectModels(ctx context.Context) map[string][]modelProv
return models
}
-// facadeRR spreads load across remote providers of the same model.
+// facadeRR spreads load across equally ranked remote providers.
var facadeRR atomic.Uint64
-// pickProvider prefers a local provider (locals sort first in collectModels),
-// otherwise round-robins across remotes.
-func pickProvider(providers []modelProvider) modelProvider {
- if providers[0].peerID == "" {
- return providers[0]
- }
- return providers[facadeRR.Add(1)%uint64(len(providers))]
-}
-
func (f *openAIFacade) handleModels(w http.ResponseWriter, r *http.Request) {
if r.Method != http.MethodGet {
writeOpenAIError(w, http.StatusMethodNotAllowed, "method_not_allowed", "only GET is supported")
@@ -335,25 +344,53 @@ func (f *openAIFacade) handleCompletions(w http.ResponseWriter, r *http.Request)
fmt.Sprintf("model %q is not served by any provider on the mesh", req.Model))
return
}
- p := pickProvider(providers)
- logger.Debugf("[OpenAIFacade] model %q -> provider peer=%q service=%q (%d candidates)",
- req.Model, p.peerID, p.service, len(providers))
+
+ requiredRegions := parseRequiredRegions(r.Header.Get(api.HeaderSamRequiredRegion))
+ ranked := f.rankProviders(providers, requiredRegions)
+ if len(ranked) == 0 {
+ recordFacadeRejection(reasonNoEligible)
+ writeOpenAIError(w, http.StatusServiceUnavailable, "no_eligible_provider",
+ fmt.Sprintf("model %q has no provider satisfying the request constraints", req.Model))
+ return
+ }
// The gate already stripped the credential it consumed; a surviving
// Authorization header is the destination backend's own credential and
// passes through, mirroring the egress path. X-Sam-Authentication is
- // local-only and must never travel off-node.
+ // local-only, and the region requirement is an instruction to this
+ // sidecar, not to the provider.
r.Header.Del(api.HeaderSamAuthentication)
- r.Body = io.NopCloser(bytes.NewReader(body))
- r.ContentLength = int64(len(body))
+ r.Header.Del(api.HeaderSamRequiredRegion)
- if p.peerID == "" {
- f.serveLocal(w, r, p.service)
- return
+ maxAttempts := min(3, len(ranked))
+ for i := range maxAttempts {
+ p := ranked[i]
+ logger.Debugf("[OpenAIFacade] model %q attempt %d/%d -> provider peer=%q service=%q (%d eligible)",
+ req.Model, i+1, maxAttempts, p.peerID, p.service, len(ranked))
+
+ attempt := r.Clone(r.Context())
+ attempt.Body = io.NopCloser(bytes.NewReader(body))
+ attempt.ContentLength = int64(len(body))
+ aw := newAttemptWriter(w)
+
+ if p.peerID == "" {
+ f.serveLocal(aw, attempt, p.service)
+ } else {
+ attempt.URL.Path = fmt.Sprintf("/sam/%s/%s/%s%s", p.peerID, api.ServiceTypeStringInference, p.service, r.URL.Path)
+ attempt.URL.RawPath = ""
+ f.forward.ServeHTTP(aw, attempt)
+ }
+ if !aw.retryable {
+ return // response delivered (success or non-retryable error)
+ }
+ recordFacadeRetry()
+ f.recordBackoff(p)
+ logger.Warnf("[OpenAIFacade] provider peer=%q service=%q returned %d for model %q; trying next",
+ p.peerID, p.service, aw.status, req.Model)
}
- r.URL.Path = fmt.Sprintf("/sam/%s/%s/%s%s", p.peerID, api.ServiceTypeStringInference, p.service, r.URL.Path)
- r.URL.RawPath = ""
- f.forward.ServeHTTP(w, r)
+ recordFacadeRejection(reasonAttemptsExceeded)
+ writeOpenAIError(w, http.StatusServiceUnavailable, "no_available_provider",
+ fmt.Sprintf("all providers for model %q are currently unavailable", req.Model))
}
func (f *openAIFacade) serveLocal(w http.ResponseWriter, r *http.Request, serviceName string) {
diff --git a/internal/node/openai_facade_test.go b/internal/node/openai_facade_test.go
index 462ba237..281f9637 100644
--- a/internal/node/openai_facade_test.go
+++ b/internal/node/openai_facade_test.go
@@ -382,20 +382,240 @@ func TestFacade_EmptyRegistryIsNotCached(t *testing.T) {
}
}
-func TestPickProvider(t *testing.T) {
+func TestRankProviders(t *testing.T) {
local := modelProvider{service: "local-llm"}
- remoteA := modelProvider{peerID: "peerA", service: "srvA"}
- remoteB := modelProvider{peerID: "peerB", service: "srvB"}
+ remoteEU := modelProvider{peerID: "peerEU", service: "srv", region: "eu", active: 5}
+ remoteUS := modelProvider{peerID: "peerUS", service: "srv", region: "us", active: 0}
+ remoteNoLabel := modelProvider{peerID: "peerX", service: "srv", active: 1}
+
+ t.Run("locals first, remotes by ascending load", func(t *testing.T) {
+ f := newTestFacade()
+ got := f.rankProviders([]modelProvider{remoteEU, remoteUS, local}, nil)
+ if len(got) != 3 || got[0].service != "local-llm" || got[1].peerID != "peerUS" || got[2].peerID != "peerEU" {
+ t.Fatalf("unexpected ranking: %+v", got)
+ }
+ })
- if got := pickProvider([]modelProvider{local, remoteA}); got != local {
- t.Errorf("pickProvider with local first: got %+v, want local", got)
+ t.Run("region requirement is fail-closed", func(t *testing.T) {
+ f := newTestFacade()
+ got := f.rankProviders([]modelProvider{remoteEU, remoteUS, remoteNoLabel, local}, []string{"eu"})
+ // Unlabeled remote and unlabeled local must both be excluded.
+ if len(got) != 1 || got[0].peerID != "peerEU" {
+ t.Fatalf("unexpected ranking under region requirement: %+v", got)
+ }
+ })
+
+ t.Run("local matches its declared region", func(t *testing.T) {
+ f := newTestFacade()
+ f.localRegion = func() string { return "eu" }
+ got := f.rankProviders([]modelProvider{local, remoteUS}, []string{"eu"})
+ if len(got) != 1 || got[0].service != "local-llm" {
+ t.Fatalf("local with matching region should survive: %+v", got)
+ }
+ })
+
+ t.Run("revoked peers are excluded", func(t *testing.T) {
+ f := newTestFacade()
+ f.isRevoked = func(peerID string) bool { return peerID == "peerUS" }
+ got := f.rankProviders([]modelProvider{remoteEU, remoteUS}, nil)
+ if len(got) != 1 || got[0].peerID != "peerEU" {
+ t.Fatalf("revoked peer should be excluded: %+v", got)
+ }
+ })
+
+ t.Run("backoff excludes and expires", func(t *testing.T) {
+ f := newTestFacade()
+ f.recordBackoff(remoteUS)
+ if got := f.rankProviders([]modelProvider{remoteUS}, nil); len(got) != 0 {
+ t.Fatalf("backed-off provider should be excluded: %+v", got)
+ }
+ f.backoffMu.Lock()
+ f.backoff[backoffKey(remoteUS)] = time.Now().Add(-time.Second)
+ f.backoffMu.Unlock()
+ if got := f.rankProviders([]modelProvider{remoteUS}, nil); len(got) != 1 {
+ t.Fatal("expired backoff should be cleared")
+ }
+ })
+
+ t.Run("equal load rotates", func(t *testing.T) {
+ f := newTestFacade()
+ a := modelProvider{peerID: "peerA", service: "srv"}
+ b := modelProvider{peerID: "peerB", service: "srv"}
+ seen := map[string]bool{}
+ for range 10 {
+ seen[f.rankProviders([]modelProvider{a, b}, nil)[0].peerID] = true
+ }
+ if !seen["peerA"] || !seen["peerB"] {
+ t.Errorf("rotation should spread across equals, got %v", seen)
+ }
+ })
+}
+
+func TestParseRequiredRegions(t *testing.T) {
+ if got := parseRequiredRegions(""); got != nil {
+ t.Errorf("empty header: got %v, want nil", got)
+ }
+ if got := parseRequiredRegions(" eu , us ,,"); len(got) != 2 || got[0] != "eu" || got[1] != "us" {
+ t.Errorf("parse: got %v", got)
+ }
+}
+
+func TestFacade_Completions_RetriesNextProvider(t *testing.T) {
+ f := newTestFacade()
+ f.discover = func(_ context.Context) ([]*api.DiscoveredProvider, error) {
+ return []*api.DiscoveredProvider{
+ {PeerId: "peerA", SrvName: "srvA"},
+ {PeerId: "peerB", SrvName: "srvB"},
+ }, nil
}
+ f.remoteModels = func(_ context.Context, _, _ string) ([]string, error) {
+ return []string{"m1"}, nil
+ }
+ var attempts []string
+ f.forward = http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ attempts = append(attempts, r.URL.Path)
+ if len(attempts) == 1 {
+ http.Error(w, "overloaded", http.StatusServiceUnavailable)
+ return
+ }
+ w.WriteHeader(http.StatusOK)
+ _, _ = w.Write([]byte(`{"ok":true}`))
+ })
- seen := map[string]bool{}
- for range 10 {
- seen[pickProvider([]modelProvider{remoteA, remoteB}).peerID] = true
+ req := httptest.NewRequest(http.MethodPost, "/v1/chat/completions", strings.NewReader(`{"model":"m1"}`))
+ rec := httptest.NewRecorder()
+ f.handleCompletions(rec, req)
+
+ if rec.Code != http.StatusOK {
+ t.Fatalf("status: got %d, want %d (attempts: %v)", rec.Code, http.StatusOK, attempts)
+ }
+ if len(attempts) != 2 {
+ t.Fatalf("attempts: got %v, want 2", attempts)
+ }
+ // The failed provider must be in backoff now.
+ f.backoffMu.Lock()
+ n := len(f.backoff)
+ f.backoffMu.Unlock()
+ if n != 1 {
+ t.Errorf("backoff entries: got %d, want 1", n)
}
- if !seen["peerA"] || !seen["peerB"] {
- t.Errorf("round-robin should hit both remotes, got %v", seen)
+}
+
+func TestFacade_Completions_AllProvidersFail(t *testing.T) {
+ f := newTestFacade()
+ f.discover = func(_ context.Context) ([]*api.DiscoveredProvider, error) {
+ return []*api.DiscoveredProvider{{PeerId: "peerA", SrvName: "srvA"}}, nil
}
+ f.remoteModels = func(_ context.Context, _, _ string) ([]string, error) {
+ return []string{"m1"}, nil
+ }
+ f.forward = http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ http.Error(w, "down", http.StatusBadGateway)
+ })
+
+ req := httptest.NewRequest(http.MethodPost, "/v1/chat/completions", strings.NewReader(`{"model":"m1"}`))
+ rec := httptest.NewRecorder()
+ f.handleCompletions(rec, req)
+
+ if rec.Code != http.StatusServiceUnavailable {
+ t.Fatalf("status: got %d, want %d", rec.Code, http.StatusServiceUnavailable)
+ }
+ if !strings.Contains(rec.Body.String(), "no_available_provider") {
+ t.Errorf("body missing no_available_provider: %s", rec.Body.String())
+ }
+}
+
+func TestFacade_Completions_RegionRequirement(t *testing.T) {
+ f := newTestFacade()
+ f.viewProviders = func(model string) []modelProvider {
+ return []modelProvider{
+ {peerID: "peerEU", service: "srvEU", region: "eu"},
+ {peerID: "peerUS", service: "srvUS", region: "us"},
+ }
+ }
+ var gotPath, gotRegionHeader string
+ f.forward = http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ gotPath = r.URL.Path
+ gotRegionHeader = r.Header.Get(api.HeaderSamRequiredRegion)
+ })
+
+ req := httptest.NewRequest(http.MethodPost, "/v1/chat/completions", strings.NewReader(`{"model":"m1"}`))
+ req.Header.Set(api.HeaderSamRequiredRegion, "eu")
+ rec := httptest.NewRecorder()
+ f.handleCompletions(rec, req)
+
+ if want := "/sam/peerEU/inference/srvEU/v1/chat/completions"; gotPath != want {
+ t.Errorf("forwarded path: got %q, want %q", gotPath, want)
+ }
+ if gotRegionHeader != "" {
+ t.Errorf("%s must not travel to the provider, got %q", api.HeaderSamRequiredRegion, gotRegionHeader)
+ }
+
+ // No provider satisfies the requirement.
+ req = httptest.NewRequest(http.MethodPost, "/v1/chat/completions", strings.NewReader(`{"model":"m1"}`))
+ req.Header.Set(api.HeaderSamRequiredRegion, "mars")
+ rec = httptest.NewRecorder()
+ f.handleCompletions(rec, req)
+ if rec.Code != http.StatusServiceUnavailable || !strings.Contains(rec.Body.String(), "no_eligible_provider") {
+ t.Errorf("unsatisfiable region: got status %d, body %s", rec.Code, rec.Body.String())
+ }
+}
+
+func TestAttemptWriter(t *testing.T) {
+ t.Run("retryable status swallowed", func(t *testing.T) {
+ rec := httptest.NewRecorder()
+ aw := newAttemptWriter(rec)
+ aw.Header().Set("Content-Type", "application/json")
+ aw.WriteHeader(http.StatusServiceUnavailable)
+ _, _ = aw.Write([]byte("error body"))
+ if !aw.retryable {
+ t.Fatal("expected retryable")
+ }
+ if rec.Body.Len() != 0 || rec.Header().Get("Content-Type") != "" {
+ t.Errorf("nothing must reach the client on a retryable status, got body %q", rec.Body.String())
+ }
+ })
+
+ t.Run("success streams through with headers", func(t *testing.T) {
+ rec := httptest.NewRecorder()
+ aw := newAttemptWriter(rec)
+ aw.Header().Set("Content-Type", "text/event-stream")
+ aw.WriteHeader(http.StatusOK)
+ _, _ = aw.Write([]byte("data: x\n\n"))
+ aw.Flush()
+ if aw.retryable {
+ t.Fatal("must not be retryable")
+ }
+ if rec.Code != http.StatusOK || rec.Body.String() != "data: x\n\n" {
+ t.Errorf("unexpected passthrough: code %d, body %q", rec.Code, rec.Body.String())
+ }
+ if rec.Header().Get("Content-Type") != "text/event-stream" {
+ t.Errorf("headers must be copied, got %v", rec.Header())
+ }
+ if !rec.Flushed {
+ t.Error("flush must reach the destination")
+ }
+ })
+
+ t.Run("implicit 200 on first write", func(t *testing.T) {
+ rec := httptest.NewRecorder()
+ aw := newAttemptWriter(rec)
+ _, _ = aw.Write([]byte("hello"))
+ if rec.Code != http.StatusOK || rec.Body.String() != "hello" {
+ t.Errorf("implicit 200: code %d, body %q", rec.Code, rec.Body.String())
+ }
+ })
+
+ t.Run("non-retryable error passes through", func(t *testing.T) {
+ rec := httptest.NewRecorder()
+ aw := newAttemptWriter(rec)
+ aw.WriteHeader(http.StatusNotFound)
+ _, _ = aw.Write([]byte("nope"))
+ if aw.retryable {
+ t.Fatal("404 must not be retryable")
+ }
+ if rec.Code != http.StatusNotFound || rec.Body.String() != "nope" {
+ t.Errorf("passthrough: code %d, body %q", rec.Code, rec.Body.String())
+ }
+ })
}
diff --git a/internal/node/openai_scorer.go b/internal/node/openai_scorer.go
new file mode 100644
index 00000000..6dda6167
--- /dev/null
+++ b/internal/node/openai_scorer.go
@@ -0,0 +1,198 @@
+// Copyright 2026 Google LLC
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package node
+
+import (
+ "net/http"
+ "slices"
+ "sort"
+ "strings"
+ "time"
+)
+
+// providerBackoff is how long a provider is skipped after a retryable failure.
+const providerBackoff = 15 * time.Second
+
+// Rejection reasons for facade provider filtering (metric label values).
+const (
+ reasonPeerRevoked = "peer_revoked"
+ reasonProviderBackoff = "provider_backoff"
+ reasonRegionMismatch = "region_mismatch"
+ reasonNoEligible = "no_eligible_provider"
+ reasonAttemptsExceeded = "attempts_exceeded"
+)
+
+// parseRequiredRegions splits the X-Sam-Required-Region header value.
+func parseRequiredRegions(h string) []string {
+ if h == "" {
+ return nil
+ }
+ var out []string
+ for _, part := range strings.Split(h, ",") {
+ if p := strings.TrimSpace(part); p != "" {
+ out = append(out, p)
+ }
+ }
+ return out
+}
+
+// rankProviders applies hard constraints then orders the survivors: eligible
+// locals first, then remotes by ascending advertised load. Providers with no
+// load information look idle — without telemetry this degrades to rotation
+// among equals. Every exclusion is accounted per reason.
+func (f *openAIFacade) rankProviders(providers []modelProvider, requiredRegions []string) []modelProvider {
+ var locals, remotes []modelProvider
+ for _, p := range providers {
+ region := p.region
+ if p.peerID == "" && f.localRegion != nil {
+ region = f.localRegion()
+ }
+ // Region is fail-closed: a requirement never matches an unlabeled
+ // provider. Labels are routing hints until biscuit-attested.
+ if len(requiredRegions) > 0 && !slices.Contains(requiredRegions, region) {
+ recordFacadeRejection(reasonRegionMismatch)
+ continue
+ }
+ if p.peerID == "" {
+ locals = append(locals, p)
+ continue
+ }
+ if f.isRevoked != nil && f.isRevoked(p.peerID) {
+ recordFacadeRejection(reasonPeerRevoked)
+ continue
+ }
+ if f.inBackoff(p) {
+ recordFacadeRejection(reasonProviderBackoff)
+ continue
+ }
+ remotes = append(remotes, p)
+ }
+
+ sort.SliceStable(remotes, func(i, j int) bool { return remotes[i].active < remotes[j].active })
+ rotateEqualHead(remotes)
+ return append(locals, remotes...)
+}
+
+// rotateEqualHead rotates the group of equally loaded head providers so
+// repeated requests spread across them instead of hammering the first.
+func rotateEqualHead(remotes []modelProvider) {
+ if len(remotes) < 2 {
+ return
+ }
+ group := 1
+ for group < len(remotes) && remotes[group].active == remotes[0].active {
+ group++
+ }
+ if group < 2 {
+ return
+ }
+ offset := int(facadeRR.Add(1) % uint64(group))
+ head := append([]modelProvider{}, remotes[:group]...)
+ for i := range group {
+ remotes[i] = head[(i+offset)%group]
+ }
+}
+
+func backoffKey(p modelProvider) string { return p.peerID + "|" + p.service }
+
+func (f *openAIFacade) inBackoff(p modelProvider) bool {
+ f.backoffMu.Lock()
+ defer f.backoffMu.Unlock()
+ until, ok := f.backoff[backoffKey(p)]
+ if !ok {
+ return false
+ }
+ if time.Now().After(until) {
+ delete(f.backoff, backoffKey(p))
+ return false
+ }
+ return true
+}
+
+func (f *openAIFacade) recordBackoff(p modelProvider) {
+ if p.peerID == "" {
+ return // locals are not backed off; the service registry owns their lifecycle
+ }
+ f.backoffMu.Lock()
+ defer f.backoffMu.Unlock()
+ if f.backoff == nil {
+ f.backoff = map[string]time.Time{}
+ }
+ f.backoff[backoffKey(p)] = time.Now().Add(providerBackoff)
+}
+
+// retryableStatus reports provider responses that are safe to retry
+// elsewhere: the provider produced no model output.
+func retryableStatus(code int) bool {
+ switch code {
+ case http.StatusTooManyRequests, http.StatusBadGateway,
+ http.StatusServiceUnavailable, http.StatusGatewayTimeout:
+ return true
+ }
+ return false
+}
+
+// attemptWriter defers committing to the client until the provider's status
+// is known: retryable statuses are swallowed (body discarded) so the request
+// can be retried on the next-ranked provider; anything else streams through.
+type attemptWriter struct {
+ dst http.ResponseWriter
+ header http.Header
+ status int
+ retryable bool
+ started bool
+}
+
+func newAttemptWriter(dst http.ResponseWriter) *attemptWriter {
+ return &attemptWriter{dst: dst, header: http.Header{}}
+}
+
+func (a *attemptWriter) Header() http.Header { return a.header }
+
+func (a *attemptWriter) WriteHeader(code int) {
+ if a.started || a.retryable {
+ return
+ }
+ a.status = code
+ if retryableStatus(code) {
+ a.retryable = true
+ return
+ }
+ dst := a.dst.Header()
+ for k, vv := range a.header {
+ dst[k] = vv
+ }
+ a.dst.WriteHeader(code)
+ a.started = true
+}
+
+func (a *attemptWriter) Write(p []byte) (int, error) {
+ if a.retryable {
+ return len(p), nil // discard failed attempt's error body
+ }
+ if !a.started {
+ a.WriteHeader(http.StatusOK)
+ }
+ return a.dst.Write(p)
+}
+
+func (a *attemptWriter) Flush() {
+ if !a.started {
+ return
+ }
+ if fl, ok := a.dst.(http.Flusher); ok {
+ fl.Flush()
+ }
+}
diff --git a/tests/integration/catalog_test.go b/tests/integration/catalog_test.go
index ad88f67d..7c856a2c 100644
--- a/tests/integration/catalog_test.go
+++ b/tests/integration/catalog_test.go
@@ -50,7 +50,8 @@ func startBackgroundNode(t *testing.T, nodeBin string, routerAddr string, homeDi
func waitForMCPAddr(t *testing.T, logPath string) string {
t.Helper()
- deadline := time.Now().Add(5 * time.Second)
+ // Generous under CI load; polling returns as soon as the line appears.
+ deadline := time.Now().Add(10 * time.Second)
for time.Now().Before(deadline) {
data, _ := os.ReadFile(logPath)
lines := strings.Split(string(data), "\n")
@@ -64,7 +65,8 @@ func waitForMCPAddr(t *testing.T, logPath string) string {
}
time.Sleep(100 * time.Millisecond)
}
- t.Fatalf("timeout waiting for MCP addr in log: %s", logPath)
+ data, _ := os.ReadFile(logPath)
+ t.Fatalf("timeout waiting for MCP addr in log: %s\n--- log contents ---\n%s", logPath, string(data))
return ""
}