Skip to content

Every agent's model. One place. Codex on DeepSeek, Claude Code on Kimi, from the menu bar.

gateway: a handed-off gateway waits for a connection still answering its request, not only new and idle ones (#3451)

yetonec26b2992e82a
diff --git a/internal/gateway/handover.go b/internal/gateway/handover.go
index 3a3e3c8d8..de99ffed4 100644
--- a/internal/gateway/handover.go
+++ b/internal/gateway/handover.go
@@ -206,7 +206,9 @@ func (s *Server) drainFor() time.Duration {
var unreadWait = 5 * time.Second
// unreadConns are the connections the gateway took whose next request it
-// hasn't read yet: new ones, and kept-alive ones idle between requests.
+// hasn't read yet: new ones, and kept-alive ones idle between requests or
+// still answering one (net/http notes one idle after its reply is out, so
+// an agent can have read the reply and sent its next request first).
// Shutdown closes such a connection unanswered, and a request the agent
// sent on it just then is cut ("connection reset", #3451), so a
// handover's end waits for them first: each one's next request is
@@ -215,29 +217,30 @@ var unreadWait = 5 * time.Second
type unreadConns struct {
mu sync.Mutex
conns map[net.Conn]bool
- idle map[net.Conn]bool
+ kept map[net.Conn]bool // answering a request, or idle after one
// stopped: the server takes none from here on, its Serve returned
stopped bool
}
-// note is the server's ConnState: a connection is unread from StateNew,
-// and from StateIdle, to its next state, whichever it is.
+// note is the server's ConnState: a connection is unread from StateNew
+// to its next state, and kept from StateActive until it is closed or
+// hijacked.
func (u *unreadConns) note(c net.Conn, state http.ConnState) {
u.mu.Lock()
defer u.mu.Unlock()
delete(u.conns, c)
- delete(u.idle, c)
+ delete(u.kept, c)
switch state {
case http.StateNew:
if u.conns == nil {
u.conns = map[net.Conn]bool{}
}
u.conns[c] = true
- case http.StateIdle:
- if u.idle == nil {
- u.idle = map[net.Conn]bool{}
+ case http.StateActive, http.StateIdle:
+ if u.kept == nil {
+ u.kept = map[net.Conn]bool{}
}
- u.idle[c] = true
+ u.kept[c] = true
}
}
@@ -258,7 +261,7 @@ func (u *unreadConns) stop() {
func (u *unreadConns) wait(ctx context.Context) {
for deadline := time.Now().Add(unreadWait); ; time.Sleep(10 * time.Millisecond) {
u.mu.Lock()
- done := u.stopped && len(u.conns) == 0 && len(u.idle) == 0
+ done := u.stopped && len(u.conns) == 0 && len(u.kept) == 0
u.mu.Unlock()
if done || time.Now().After(deadline) || ctx.Err() != nil {
return
diff --git a/internal/gateway/handover_test.go b/internal/gateway/handover_test.go
index d3ed04715..a8195472d 100644
--- a/internal/gateway/handover_test.go
+++ b/internal/gateway/handover_test.go
@@ -2,6 +2,7 @@ package gateway
import (
"bufio"
+ "bytes"
"context"
"fmt"
"io"
@@ -429,6 +430,126 @@ func TestHandedOffGatewayAnswersAKeptAliveConnection(t *testing.T) {
}
}
+// lagListener's connections, once told to, sit a second after the write
+// that ends the next chunked reply: as a busy machine can hold the
+// goroutine that wrote a reply, its agent reading it, before net/http
+// notes the connection idle.
+type lagListener struct {
+ net.Listener
+ lag atomic.Bool
+}
+
+func (l *lagListener) Accept() (net.Conn, error) {
+ c, err := l.Listener.Accept()
+ if err != nil {
+ return nil, err
+ }
+ return &lagConn{Conn: c, lag: &l.lag}, nil
+}
+
+type lagConn struct {
+ net.Conn
+ lag *atomic.Bool
+}
+
+func (c *lagConn) Write(b []byte) (int, error) {
+ n, err := c.Conn.Write(b)
+ if bytes.HasSuffix(b, []byte("0\r\n\r\n")) && c.lag.Swap(false) {
+ time.Sleep(time.Second)
+ }
+ return n, err
+}
+
+// A kept-alive connection whose reply its agent has read, which net/http
+// notes idle only once the end has begun (the goroutine that wrote it was
+// held), has its next request answered there too: the end waited for new
+// and idle connections only, so Shutdown closed this one as idle when
+// net/http noted it, and the agent's next request was cut (ubuntu CI,
+// 56602426d). So too for a `make dev` backend.
+func TestHandedOffGatewayAnswersAConnectionStillAnswering(t *testing.T) {
+ for _, dev := range []bool{false, true} {
+ name := "handed off"
+ if dev {
+ name = "make dev"
+ }
+ t.Run(name, func(t *testing.T) {
+ was := Handover
+ Handover = dev
+ defer func() { Handover = was }()
+ quickChat(t)
+ tcp, err := net.Listen("tcp", "127.0.0.1:0")
+ if err != nil {
+ t.Fatal(err)
+ }
+ addr := tcp.Addr().String()
+ t.Setenv("MAGPIE_ADDR", addr)
+ lagged := &lagListener{Listener: tcp}
+ Inherit(lagged, func() {})
+ old := New()
+ stopOld, oldDone := listening(t, old)
+ t.Cleanup(stopOld)
+
+ c, err := net.Dial("tcp", addr)
+ if err != nil {
+ t.Fatal(err)
+ }
+ defer c.Close()
+ rd := bufio.NewReader(c)
+ ask := func() (*http.Response, string, error) {
+ chatOn(t, c, addr)
+ c.SetReadDeadline(time.Now().Add(10 * time.Second))
+ r, err := http.ReadResponse(rd, nil)
+ if err != nil {
+ return nil, "", err
+ }
+ defer r.Body.Close()
+ b, err := io.ReadAll(r.Body)
+ return r, string(b), err
+ }
+ lagged.lag.Store(true)
+ if r, b, err := ask(); err != nil || r.StatusCode != 200 || r.Close {
+ t.Fatalf("the request before the handover: %v %q %v", r, b, err)
+ }
+ // read, while the gateway's goroutine still sits in its write
+ if !dev {
+ f, err := tcp.(*net.TCPListener).File()
+ if err != nil {
+ t.Fatal(err)
+ }
+ ln, err := net.FileListener(f) // the new one's copy, keeping the socket open
+ f.Close()
+ if err != nil {
+ t.Fatal(err)
+ }
+ defer ln.Close()
+ old.HandOff()
+ }
+ stopOld()
+ // past the write's second, net/http has noted it idle: inside
+ // the 5s the end waits (unreadWait)
+ time.Sleep(1500 * time.Millisecond)
+ r, b, err := ask()
+ if err != nil || r.StatusCode != 200 || !strings.Contains(b, "quick") {
+ t.Fatalf("a request on a connection still answering as the end began: %q %v", b, err)
+ }
+ if !r.Close {
+ t.Fatal("the reply kept the connection alive, for a request the old one's end would cut")
+ }
+ select {
+ case err := <-oldDone:
+ if err != nil {
+ t.Fatal(err)
+ }
+ case <-time.After(3 * time.Second):
+ t.Fatal("the old one never ended once its connections were answered")
+ }
+ if n := len(old.Recent()); n != 2 {
+ t.Fatalf("the old one answered %d of its 2 requests", n)
+ }
+ })
+ }
+}
+
// heldListener holds the next connection it takes, once told to, until
// let go: as a busy machine can hold the goroutine that took one, before
// net/http notes it.