Skip to content

Agent Mesh

fix: mcp_service - don't drop the last response on backend EOF (#375)

Fernando Marino8e221f83cd64
diff --git a/internal/node/gate_test.go b/internal/node/gate_test.go
index c187bbf6..e4a30ce5 100644
--- a/internal/node/gate_test.go
+++ b/internal/node/gate_test.go
@@ -16,7 +16,9 @@ package node
import (
"context"
+ "net/http"
"net/http/httptest"
+ "strings"
"testing"
"time"
@@ -313,3 +315,138 @@ func TestHandleMCPStream_ForwarderRoutesCalls(t *testing.T) {
t.Errorf("forwarder did not pass un-namespaced name; got %q", tc.Text)
}
}
+
+// startBareQUICNode is startBareNode over the QUIC transport instead of TCP.
+//
+// The distinction matters for TestHandleStreamPassThrough_BackendEOFDoesNotDropInFlightResponse
+// below: go-libp2p's QUIC-backed network.Stream.Close cancels the read side
+// over the wire (CancelRead sends a QUIC STOP_SENDING frame to the peer),
+// while the yamux stream used over startBareNode's plain TCP transport does
+// the equivalent locally only - its CloseRead doc says plainly "Remote is
+// not notified." A stream backed by TCP+yamux structurally cannot exercise
+// the wire-level race this test is after; only a real QUIC connection can.
+func startBareQUICNode(t *testing.T, ctx context.Context) (*SamNode, func()) {
+ t.Helper()
+ dir := t.TempDir()
+ store, err := NewStore(dir)
+ if err != nil {
+ t.Fatal(err)
+ }
+
+ priv, _, err := crypto.GenerateKeyPair(crypto.Ed25519, -1)
+ if err != nil {
+ t.Fatal(err)
+ }
+ node, err := NewSamNode(Options{
+ PrivKey: priv,
+ RouterAddrs: nil,
+ Store: store,
+ MeshID: "test-mesh",
+ DiscoveryInterval: "1s",
+ ListenAddrs: []string{"/ip4/127.0.0.1/udp/0/quic-v1"},
+ EnableRelay: false,
+ NodeConfig: &NodeConfigComplete{},
+ KeyGracePeriod: 24 * time.Hour,
+ AllowLoopback: true,
+ MonitorBootstrap: 2 * time.Minute,
+ MonitorInterval: 1 * time.Minute,
+ })
+ if err != nil {
+ t.Fatal(err)
+ }
+ node.BiscuitTimeout = 500 * time.Millisecond
+ if err := node.Start(ctx); err != nil {
+ t.Fatal(err)
+ }
+
+ cleanup := func() {
+ _ = node.Teardown()
+ _ = store.Close()
+ }
+ return node, cleanup
+}
+
+// TestHandleStreamPassThrough_BackendEOFDoesNotDropInFlightResponse is a
+// regression test for google/sam#375.
+//
+// A backend that answers a single request and then closes its side of the
+// connection - a normal EOF for a one-shot HTTP-style backend, not a
+// failure - used to make HandleStreamPassThrough tear the whole
+// client-facing stream down immediately via network.Stream.Close. Close's
+// own documented contract says it "does not guarantee receipt of the data";
+// closing right behind a Write that had just reported success raced the
+// response off the wire, and the caller in the original report sometimes
+// saw EOF in its place. See the comment on the drain logic in
+// HandleStreamPassThrough for the fix.
+//
+// The race is timing-dependent, so this repeats the call many times over a
+// real QUIC connection (see startBareQUICNode) rather than asserting on a
+// single attempt.
+func TestHandleStreamPassThrough_BackendEOFDoesNotDropInFlightResponse(t *testing.T) {
+ ctx, cancel := context.WithTimeout(context.Background(), 90*time.Second)
+ defer cancel()
+
+ // A payload closer in size to the 834-byte response in the original
+ // report than a trivial "ok" would be - small messages are far more
+ // likely to always win the race regardless of the bug.
+ wantText := strings.Repeat("x", 700)
+ upstream := httptest.NewServer(mcp.NewStreamableHTTPHandler(func(*http.Request) *mcp.Server {
+ srv := mcp.NewServer(&mcp.Implementation{Name: "fake", Version: "0.0.1"}, nil)
+ srv.AddTool(&mcp.Tool{Name: "echo", Description: "echo", InputSchema: map[string]any{"type": "object"}},
+ func(context.Context, *mcp.CallToolRequest) (*mcp.CallToolResult, error) {
+ return &mcp.CallToolResult{Content: []mcp.Content{&mcp.TextContent{Text: wantText}}}, nil
+ })
+ return srv
+ }, nil))
+ defer upstream.Close()
+
+ nodeA, cleanupA := startBareQUICNode(t, ctx)
+ defer cleanupA()
+ nodeB, cleanupB := startBareQUICNode(t, ctx)
+ defer cleanupB()
+
+ svc := &MCPService{baseService: baseService{
+ info: &api.ServiceInfo{Type: api.ServiceType_SERVICE_TYPE_MCP, Name: "svc"},
+ backend: &api.RegisterServiceRequest_TargetUrl{TargetUrl: upstream.URL},
+ }}
+ if err := svc.Init(ctx); err != nil {
+ t.Fatalf("MCPService.Init: %v", err)
+ }
+ nodeA.services.insertService(svc)
+ t.Cleanup(func() { _ = svc.Teardown() })
+
+ nodeA.Host.SetStreamHandler(testMCPProtocol, func(s network.Stream) {
+ nodeA.HandleMCPStream(s, RequestContext{Target: "svc"})
+ })
+
+ if err := nodeB.Host.Connect(ctx, peer.AddrInfo{ID: nodeA.Host.ID(), Addrs: nodeA.Host.Addrs()}); err != nil {
+ t.Fatalf("connect: %v", err)
+ }
+
+ const iterations = 50
+ for i := 0; i < iterations; i++ {
+ func() {
+ s, err := nodeB.Host.NewStream(ctx, nodeA.Host.ID(), testMCPProtocol)
+ if err != nil {
+ t.Fatalf("iteration %d: NewStream: %v", i, err)
+ }
+ defer func() { _ = s.Close() }()
+
+ client := mcp.NewClient(&mcp.Implementation{Name: "tc", Version: "0.0.1"}, nil)
+ session, err := client.Connect(ctx, NewStreamTransport(s), nil)
+ if err != nil {
+ t.Fatalf("iteration %d: client.Connect: %v", i, err)
+ }
+ defer func() { _ = session.Close() }()
+
+ res, err := session.CallTool(ctx, &mcp.CallToolParams{Name: "echo", Arguments: map[string]any{}})
+ if err != nil {
+ t.Fatalf("iteration %d: CallTool: %v", i, err)
+ }
+ tc, ok := res.Content[0].(*mcp.TextContent)
+ if !ok || tc.Text != wantText {
+ t.Fatalf("iteration %d: got %v, want text of length %d", i, res.Content, len(wantText))
+ }
+ }()
+ }
+}
diff --git a/internal/node/mcp_service.go b/internal/node/mcp_service.go
index 32b11c26..2939bd34 100644
--- a/internal/node/mcp_service.go
+++ b/internal/node/mcp_service.go
@@ -140,6 +140,13 @@ var preflightMethodsUnsupportedByPassThrough = map[string]bool{
"server/discover": true,
}
+// passThroughDrainTimeout bounds how long HandleStreamPassThrough keeps a
+// client-facing stream open after the backend leg has ended, waiting for the
+// client to finish reading the last response and hang up on its own. See the
+// comment on the drain logic in HandleStreamPassThrough for why this wait
+// exists at all.
+const passThroughDrainTimeout = 5 * time.Second
+
// HandleStreamPassThrough connects to the backend and proxies JSON-RPC messages.
func (m *MCPService) HandleStreamPassThrough(s network.Stream) {
defer func() {
@@ -148,23 +155,14 @@ func (m *MCPService) HandleStreamPassThrough(s network.Stream) {
}
}()
- var backendTransport mcp.Transport
- var closeTransport func()
-
backendTransport, err := m.backendTransport()
if err != nil {
logger.Errorf("[MCPService] %s: %v", m.info.Name, err)
return
}
- closeTransport = func() {} // fresh per stream for URL; shared bridge is never closed
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
- defer func() {
- if closeTransport != nil {
- closeTransport()
- }
- }()
backendConn, err := backendTransport.Connect(ctx)
if err != nil {
@@ -180,49 +178,78 @@ func (m *MCPService) HandleStreamPassThrough(s network.Stream) {
return
}
- // Dumb pipe: Proxy JSON-RPC messages between client and backend
- errc := make(chan error, 2)
+ // Dumb pipe: proxy JSON-RPC messages between client and backend.
+ //
+ // The two legs are not symmetric on shutdown. A backend answering one
+ // request and then hanging up (a clean EOF, typical of a one-shot
+ // HTTP-style backend transport) is normal completion, not a failure of
+ // the client-facing side of the pipe - but closing s in reaction to it
+ // used to tear down both directions immediately (network.Stream.Close
+ // implies CancelRead), including the read side and, per that method's
+ // own documented contract, without waiting for the response this same
+ // goroutine had just handed to Write to actually reach the peer. Close
+ // "does not guarantee receipt of the data"; the documented safe sequence
+ // is CloseWrite, then wait for the peer to finish reading (or hang up),
+ // then Close. That race is the root cause of google/sam#375: the
+ // producer's write reports success, but the immediate teardown right
+ // behind it can still lose the response in flight, and the consumer
+ // sees EOF instead.
+ //
+ // So the backend leg ending only half-closes our write side to the
+ // client (CloseWrite: no more responses are coming, but nothing already
+ // in flight is discarded) and stops relaying backend->client; it leaves
+ // the read side alone. The client leg - the client itself finishing the
+ // read and hanging up, or a genuine transport error - is what triggers
+ // the final s.Close() at the top of this function. passThroughDrainTimeout
+ // bounds that wait so a client that never hangs up can't leak the stream
+ // forever.
+ clientErrc := make(chan error, 1)
+
+ go func() {
+ for {
+ msg, err := backendConn.Read(ctx)
+ if err != nil {
+ logger.Debugf("[MCPService] %s: backend read error: %v", m.info.Name, err)
+ if cwErr := s.CloseWrite(); cwErr != nil {
+ logger.Debugf("[MCPService] %s: failed to close write side to client: %v", m.info.Name, cwErr)
+ }
+ return
+ }
+ if err := clientConn.Write(ctx, msg); err != nil {
+ logger.Debugf("[MCPService] %s: client write error: %v", m.info.Name, err)
+ return
+ }
+ }
+ }()
go func() {
for {
msg, err := clientConn.Read(ctx)
if err != nil {
logger.Debugf("[MCPService] %s: client read error: %v", m.info.Name, err)
- errc <- err
+ clientErrc <- err
return
}
if req, ok := msg.(*jsonrpc.Request); ok && preflightMethodsUnsupportedByPassThrough[req.Method] {
resp := &jsonrpc.Response{ID: req.ID, Error: &jsonrpc.Error{Code: jsonrpc.CodeMethodNotFound, Message: req.Method + " is not supported by this pass-through proxy"}}
if werr := clientConn.Write(ctx, resp); werr != nil {
logger.Debugf("[MCPService] %s: failed to reject %s: %v", m.info.Name, req.Method, werr)
- errc <- werr
+ clientErrc <- werr
return
}
continue
}
if err := backendConn.Write(ctx, msg); err != nil {
logger.Debugf("[MCPService] %s: backend write error: %v", m.info.Name, err)
- errc <- err
+ clientErrc <- err
return
}
}
}()
- go func() {
- for {
- msg, err := backendConn.Read(ctx)
- if err != nil {
- logger.Debugf("[MCPService] %s: backend read error: %v", m.info.Name, err)
- errc <- err
- return
- }
- if err := clientConn.Write(ctx, msg); err != nil {
- logger.Debugf("[MCPService] %s: client write error: %v", m.info.Name, err)
- errc <- err
- return
- }
- }
- }()
-
- <-errc
+ select {
+ case <-clientErrc:
+ case <-time.After(passThroughDrainTimeout):
+ logger.Debugf("[MCPService] %s: client did not hang up within %v of the backend finishing; closing", m.info.Name, passThroughDrainTimeout)
+ }
}