Every agent's model. One place. Codex on DeepSeek, Claude Code on Kimi, from the menu bar.
gateway: a handed-off gateway stops accepting at once and answers every connection it took (#3568)
JoeyWu-tech11c427eaa61b
diff --git a/docs/subsystems/gateway-routing.md b/docs/subsystems/gateway-routing.mdindex 09bea4efa..98811c6df 100644--- a/docs/subsystems/gateway-routing.md+++ b/docs/subsystems/gateway-routing.md@@ -287,12 +287,20 @@ recorded as a prediction failure (`noPrediction`).[`handover.go`](../../internal/gateway/handover.go)). `Inherit` gives thenew process's `ListenAndServe` a socket to serve when the address is thesame (`sameAddr`), and calls back once it serves; `HandOff` has the old- one's shutdown wait up to `handedOffDrain` (15 minutes) for its requests- in flight instead of `shortDrain` (2s). Requests per minute, concurrency- and sticky routing's in-memory state are each process's own: sticky- routing is flushed to disk before the socket goes (`FlushSticks`), but- while the old one drains, a provider's limits can be passed by what each- has in flight. `TestHandedOverGatewayFinishesItsStream`,+ one stop accepting at once (`stopAccepting`: its copy of the socket goes,+ the new one's keeps it open) and its shutdown wait up to `handedOffDrain`+ (15 minutes) for its requests in flight instead of `shortDrain` (2s).+ Before that shutdown, each connection it took sends its first request+ (`unreadConns`, `unreadWait` 5s at most): `http.Server.Shutdown` closes+ one whose request it reads after it began unanswered (#3568). Requests+ per minute, concurrency and sticky routing's in-memory state are each+ process's own: sticky routing is flushed to disk before the socket goes+ (`FlushSticks`), but while the old one drains, a provider's limits can be+ passed by what each has in flight.+ `TestHandedOverGatewayFinishesItsStream`,+ `TestHandedOffGatewayTakesNoNewConnection`,+ `TestHandedOffGatewayAnswersAConnectionItTook`,+ `TestHandedOffGatewayAnswersAConnectionNotedLate`,`TestInheritedForAnotherAddressIsLetGo`, `TestToolCallOfTheMagpieBeforeGoesToIt` in[`handover_test.go`](../../internal/gateway/handover_test.go).- An agent's turn (Chat, Responses, Messages, Gemini's generateContent anddiff --git a/docs/subsystems/gui-shell.md b/docs/subsystems/gui-shell.mdindex 52867a467..e64132ee5 100644--- a/docs/subsystems/gui-shell.md+++ b/docs/subsystems/gui-shell.md@@ -506,12 +506,17 @@ then calls `beginHandover` instead of the relaunch-after-exit. The steps:serves on it and then acks.3. The old one, on the ack (`AwaitAck`), withdraws the pointer, hidesitself where it can (below), calls `gateway.Server.HandOff`, ends its- backend and waits for its requests, then quits. A handed-off server's- shutdown waits up to `handedOffDrain` (15 minutes) for requests in- flight, not the 2s an ordinary quit gives them (`shortDrain`). Its- `OnShutdown` then skips what goes with the gateway, which the new one- now does: ending Codex's failover, porting Codex rollouts and installing- the update.+ backend and waits for its requests, then quits. `HandOff` closes the+ old one's own copy of the socket at once, so every connection from+ then on waits there for the new one: one the old one took as its end+ began was closed unanswered (#3568). A handed-off server's shutdown+ waits up to `handedOffDrain` (15 minutes) for requests in flight, not+ the 2s an ordinary quit gives them (`shortDrain`), and first lets each+ connection it took send its first request (`unreadWait`, 5s at most),+ since `http.Server.Shutdown` closes one whose request it reads after+ it began. Its `OnShutdown` then skips what goes with the gateway,+ which the new one now does: ending Codex's failover, porting Codex+ rollouts and installing the update.4. Should the Offer not open, the new version not start, not say hello in60s (`helloWait`), not ack in 90s (`ackWait`), or drop the connection,the old one keeps the socket, takes its instance lock back and restartsdiff --git a/internal/gateway/gateway.go b/internal/gateway/gateway.goindex 36f989561..0971cd264 100644--- a/internal/gateway/gateway.go+++ b/internal/gateway/gateway.go@@ -442,6 +442,13 @@ type Server struct {// handedOff: another magpie serves the port now, and this one's end// waits for its requests in flight to finish (handover.go)handedOff atomic.Bool+ // notAccepting: this one let its listener go, for the magpie serving+ // the port after it to take every connection (stopAccepting); its+ // Serve ending then waits for the drain+ notAccepting atomic.Bool+ // the connections taken whose first request isn't read yet, which a+ // handover's end waits for (handover.go)+ unread unreadConns// what it serves other computers TLS with, nil without (tls.go)tls *tls.Config// the images described for models that can't see them (vision.go)@@ -621,7 +628,7 @@ func (s *Server) ListenAndServe(ctx context.Context) error {// loops' next look (#3185)go filememo.Watch(freshCtx, pluginrt.Path(), settings.Path(), provider.Path(), plugin.ListPath(), plugin.ToolsPath(), access.Path())go pluginrt.KeepFresh(freshCtx, filememo.OthersEvery, settings.Path(), provider.Path(), plugin.ListPath(), plugin.ToolsPath(), access.Path())- srv := &http.Server{Handler: s.Front(), ReadHeaderTimeout: 30 * time.Second, IdleTimeout: 5 * time.Minute}+ srv := &http.Server{Handler: s.Front(), ReadHeaderTimeout: 30 * time.Second, IdleTimeout: 5 * time.Minute, ConnState: s.unread.note}// the magpie serving the gateway, and only it, keeps the saved accounts// signed in, so two never refresh one sign-in at oncego provider.KeepLoginsAlive(ctx)@@ -683,6 +690,14 @@ func (s *Server) ListenAndServe(ctx context.Context) error {<-ctx.Done()c, cancel := context.WithTimeout(context.Background(), s.drainFor())defer cancel()+ if s.handingOver() {+ // another magpie takes every connection from here: this one+ // stops taking them, and each one it took has its first+ // request read before Shutdown, which closes one it reads+ // after it began unanswered (#3568)+ s.stopAccepting()+ s.unread.wait(c)+ }srv.Shutdown(c)s.subscription.abortAll()provider.FlushServed() // what answered last, kept for the next start and the CLI@@ -690,7 +705,9 @@ func (s *Server) ListenAndServe(ctx context.Context) error {}()for {err := srv.Serve(ln)- if err == nil || errors.Is(err, http.ErrServerClosed) {+ // or stopAccepting closed it: what this one took finishes once ctx ends+ if err == nil || errors.Is(err, http.ErrServerClosed) || s.notAccepting.Load() {+ s.unread.stop()<-drainedreturn nil}diff --git a/internal/gateway/handover.go b/internal/gateway/handover.goindex ef3b751b7..6a4339b2f 100644--- a/internal/gateway/handover.go+++ b/internal/gateway/handover.go@@ -152,19 +152,95 @@ func (s *Server) ListenerFile() (*os.File, error) {return tl.File()}-// HandOff has the gateway, once its ctx ends, wait for every request in-// flight to finish (handedOffDrain at most) rather than 2s: another-// magpie serves the port meanwhile.-func (s *Server) HandOff() { s.handedOff.Store(true) }+// HandOff has the gateway stop accepting at once: the magpie it was+// handed to takes every connection from now on (one this one took as its+// end began was closed unanswered, #3568). Once its ctx ends, it waits+// for every request in flight to finish (handedOffDrain at most) rather+// than 2s.+func (s *Server) HandOff() {+ s.handedOff.Store(true)+ s.stopAccepting()+}++// stopAccepting has the gateway take no connection from here on, for the+// magpie serving the port after it: the copy of the socket handed to that+// one keeps the socket open, so a connection waits there rather than+// being refused. (`make dev`'s takes them on a socket of its own.)+func (s *Server) stopAccepting() {+ s.lnMu.Lock()+ defer s.lnMu.Unlock()+ s.notAccepting.Store(true)+ if s.ln != nil {+ s.ln.Close()+ }+}++// handingOver is whether another magpie serves the port once this+// gateway ends: it was handed off, or `make dev` started one beside it.+func (s *Server) handingOver() bool { return Handover || s.handedOff.Load() }// drainFor is how long the gateway's end waits for the requests in flight.func (s *Server) drainFor() time.Duration {- if Handover || s.handedOff.Load() {+ if s.handingOver() {return handedOffDrain}return shortDrain}+// unreadWait is how long a handover's end waits for a connection taken to+// send its first request: as for net/http's Shutdown, one that sent+// nothing for 5s is idle.+var unreadWait = 5 * time.Second++// unreadConns are the connections the gateway took whose first request it+// hasn't read yet. Shutdown closes such a connection unanswered when its+// request is read after it began, so a handover's end waits for them+// first: the agent's request is answered here, not cut.+type unreadConns struct {+ mu sync.Mutex+ conns map[net.Conn]bool+ // stopped: the server takes none from here on, its Serve returned+ stopped bool+}++// note is the server's ConnState: a connection is unread from StateNew to+// its next state, whichever it is.+func (u *unreadConns) note(c net.Conn, state http.ConnState) {+ u.mu.Lock()+ defer u.mu.Unlock()+ if state != http.StateNew {+ delete(u.conns, c)+ return+ }+ if u.conns == nil {+ u.conns = map[net.Conn]bool{}+ }+ u.conns[c] = true+}++// stop is called once the server's Serve has returned. net/http notes a+// connection (StateNew) on the goroutine that took it, before Serve can+// return, so every one it took is noted by then: one noted after Shutdown+// began would have its request closed unanswered too.+func (u *unreadConns) stop() {+ u.mu.Lock()+ u.stopped = true+ u.mu.Unlock()+}++// wait returns once the server takes no more connections and none it took+// is unread, after unreadWait at most, or when ctx ends.+func (u *unreadConns) wait(ctx context.Context) {+ for deadline := time.Now().Add(unreadWait); ; time.Sleep(10 * time.Millisecond) {+ u.mu.Lock()+ done := u.stopped && len(u.conns) == 0+ u.mu.Unlock()+ if done || time.Now().After(deadline) || ctx.Err() != nil {+ return+ }+ }+}+// ServeCallbacks answers, on ln, the tool calls of the Claude Code runs// this process started (claudeMCPPath): their MCP helper calls the// gateway's port, which the magpie this one handed it to now serves, anddiff --git a/internal/gateway/handover_test.go b/internal/gateway/handover_test.goindex cc36db6dc..466d3bc45 100644--- a/internal/gateway/handover_test.go+++ b/internal/gateway/handover_test.go@@ -3,6 +3,7 @@ package gatewayimport ("bufio""context"+ "fmt""io""net""net/http"@@ -10,6 +11,7 @@ import ("os""path/filepath""strings"+ "sync""sync/atomic""testing""time"@@ -181,6 +183,272 @@ func TestHandedOverGatewayFinishesItsStream(t *testing.T) {}}+// quickChat is a fresh HOME whose chat is answered at once ("quick").+func quickChat(t *testing.T) {+ t.Helper()+ fresh(t)+ up := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {+ w.Header().Set("Content-Type", "application/json")+ io.WriteString(w, `{"choices":[{"message":{"role":"assistant","content":"quick"},"finish_reason":"stop"}],"usage":{"prompt_tokens":3,"completion_tokens":1}}`)+ }))+ t.Cleanup(up.Close)+ if err := provider.Save(provider.Provider{ID: "plan", Name: "Plan", Key: "upstream", Chat: up.URL + "/v1", Models: []string{"m1"}}); err != nil {+ t.Fatal(err)+ }+}++// handOffSandbox is a gateway serving on a port of its own, its chat+// answered at once (quickChat), as the magpie to hand off; stop ends it+// and done says how it ended.+func handOffSandbox(t *testing.T) (old *Server, addr string, stop context.CancelFunc, done <-chan error) {+ t.Helper()+ quickChat(t)+ t.Setenv("MAGPIE_ADDR", "127.0.0.1:0")+ old = New()+ stop, done = listening(t, old)+ t.Cleanup(stop)+ addr = old.Addr()+ t.Setenv("MAGPIE_ADDR", addr) // as settings name it, for a new one+ return old, addr, stop, done+}++// chatOn sends the chat request on c, a connection of its own to addr.+func chatOn(t *testing.T, c net.Conn, addr string) {+ t.Helper()+ if _, err := fmt.Fprintf(c, "POST /v1/chat/completions HTTP/1.1\r\nHost: %s\r\nContent-Type: application/json\r\nContent-Length: %d\r\n\r\n%s", addr, len(chatReq), chatReq); err != nil {+ t.Fatal(err)+ }+}++// answerOn is the reply read on c, or why there was none.+func answerOn(c net.Conn) (string, error) {+ c.SetReadDeadline(time.Now().Add(10 * time.Second))+ r, err := http.ReadResponse(bufio.NewReader(c), nil)+ if err != nil {+ return "", err+ }+ defer r.Body.Close()+ b, err := io.ReadAll(r.Body)+ return r.Status + " " + string(b), err+}++// A gateway handed off takes no connection from then on, its ctx not+// ended yet: one made then waits on the socket for the magpie it was+// handed to, which answers it. The old one took it before, until its end+// closed its listener, and closed it unanswered when it read the request+// after that end began (#3568).+func TestHandedOffGatewayTakesNoNewConnection(t *testing.T) {+ old, addr, stopOld, oldDone := handOffSandbox(t)+ defer func() {+ stopOld()+ <-oldDone+ }()+ f, err := old.ListenerFile()+ if err != nil {+ t.Fatal(err)+ }+ ln, err := net.FileListener(f)+ f.Close()+ if err != nil {+ t.Fatal(err)+ }+ old.HandOff()++ // made before the new one serves: only the old one could take it now+ c, err := net.Dial("tcp", addr)+ if err != nil {+ t.Fatal(err)+ }+ defer c.Close()+ chatOn(t, c, addr)+ Inherit(ln, func() {})+ next := New()+ stopNext, nextDone := listening(t, next)+ defer func() {+ stopNext()+ <-nextDone+ }()+ if got, err := answerOn(c); err != nil || !strings.HasPrefix(got, "200 ") || !strings.Contains(got, "quick") {+ t.Fatalf("a request made once the gateway was handed off: %q %v", got, err)+ }+ if n := len(old.Recent()); n != 0 {+ t.Fatalf("the old one answered %d request(s) made once it had handed the gateway off", n)+ }+ if n := len(next.Recent()); n != 1 {+ t.Fatalf("the new one answered %d of the 1 request made once the gateway was handed off", n)+ }+}++// A connection the gateway took before it handed off, whose request comes+// only once its end has begun, is answered there: the end waits for every+// connection taken to send its first request before Shutdown, which closes+// one it reads after it began unanswered (#3568). So too for a `make dev`+// backend ending beside the one started after it (Handover).+func TestHandedOffGatewayAnswersAConnectionItTook(t *testing.T) {+ for _, dev := range []bool{false, true} {+ name := "handed off"+ if dev {+ name = "make dev"+ }+ t.Run(name, func(t *testing.T) {+ was := Handover+ Handover = dev+ defer func() { Handover = was }()+ old, addr, stopOld, oldDone := handOffSandbox(t)++ c, err := net.Dial("tcp", addr)+ if err != nil {+ t.Fatal(err)+ }+ defer c.Close()+ // one made after it, answered: the old one took c first, as it+ // takes connections in the order they came+ one := &http.Client{Transport: &http.Transport{DisableKeepAlives: true}}+ r, err := one.Post("http://"+addr+"/v1/chat/completions", "application/json", strings.NewReader(chatReq))+ if err != nil {+ t.Fatal(err)+ }+ r.Body.Close()+ if r.StatusCode != 200 {+ t.Fatalf("the request after it: %d", r.StatusCode)+ }++ if !dev {+ f, err := old.ListenerFile()+ if err != nil {+ t.Fatal(err)+ }+ ln, err := net.FileListener(f) // the new one's copy, keeping the socket open+ f.Close()+ if err != nil {+ t.Fatal(err)+ }+ defer ln.Close()+ old.HandOff()+ }+ stopOld()+ // its agent sends the request late, once the end has begun:+ // well inside the 5s the end waits for it (unreadWait)+ time.Sleep(100 * time.Millisecond)+ chatOn(t, c, addr)+ if got, err := answerOn(c); err != nil || !strings.HasPrefix(got, "200 ") || !strings.Contains(got, "quick") {+ t.Fatalf("a request on a connection the old one took, sent once its end began: %q %v", got, err)+ }+ // its end follows the answer, not the 5s the wait gives up after+ // (unreadWait)+ select {+ case err := <-oldDone:+ if err != nil {+ t.Fatal(err)+ }+ case <-time.After(3 * time.Second):+ t.Fatal("the old one never ended once its connections were answered")+ }+ if n := len(old.Recent()); n != 2 {+ t.Fatalf("the old one answered %d of its 2 requests", n)+ }+ })+ }+}++// heldListener holds the next connection it takes, once told to, until+// let go: as a busy machine can hold the goroutine that took one, before+// net/http notes it.+type heldListener struct {+ net.Listener+ hold atomic.Bool+ took chan struct{} // closed once it holds one+ let chan struct{} // closed to hand it on+}++func (l *heldListener) Accept() (net.Conn, error) {+ c, err := l.Listener.Accept()+ if err == nil && l.hold.Swap(false) {+ close(l.took)+ <-l.let+ }+ return c, err+}++// A connection the gateway took just before its end began, which net/http+// notes only after that (the goroutine that took it was held), is answered+// there too: the end waits for Serve to return, by when every connection+// it took is noted. Shutdown marks the server shutting down before it waits+// for Serve, so a request on one noted after that is closed unanswered.+// So too for a `make dev` backend ending beside the one started after it.+func TestHandedOffGatewayAnswersAConnectionNotedLate(t *testing.T) {+ for _, dev := range []bool{false, true} {+ name := "handed off"+ if dev {+ name = "make dev"+ }+ t.Run(name, func(t *testing.T) {+ was := Handover+ Handover = dev+ defer func() { Handover = was }()+ quickChat(t)+ tcp, err := net.Listen("tcp", "127.0.0.1:0")+ if err != nil {+ t.Fatal(err)+ }+ addr := tcp.Addr().String()+ t.Setenv("MAGPIE_ADDR", addr)+ held := &heldListener{Listener: tcp, took: make(chan struct{}), let: make(chan struct{})}+ Inherit(held, func() {})+ old := New()+ stopOld, oldDone := listening(t, old)+ t.Cleanup(stopOld)+ letGo := sync.OnceFunc(func() { close(held.let) })+ t.Cleanup(letGo)++ held.hold.Store(true)+ c, err := net.Dial("tcp", addr)+ if err != nil {+ t.Fatal(err)+ }+ defer c.Close()+ select {+ case <-held.took:+ case <-time.After(10 * time.Second):+ t.Fatal("the gateway never took the connection")+ }+ if !dev {+ f, err := tcp.(*net.TCPListener).File()+ if err != nil {+ t.Fatal(err)+ }+ ln, err := net.FileListener(f) // the new one's copy, keeping the socket open+ f.Close()+ if err != nil {+ t.Fatal(err)+ }+ defer ln.Close()+ old.HandOff()+ }+ stopOld()+ // the end has begun, and would be in Shutdown by now, before+ // net/http notes the connection and reads its request+ time.Sleep(100 * time.Millisecond)+ chatOn(t, c, addr)+ letGo()+ if got, err := answerOn(c); err != nil || !strings.HasPrefix(got, "200 ") || !strings.Contains(got, "quick") {+ t.Fatalf("a request on a connection the old one took, noted once its end began: %q %v", got, err)+ }+ select {+ case err := <-oldDone:+ if err != nil {+ t.Fatal(err)+ }+ case <-time.After(3 * time.Second):+ t.Fatal("the old one never ended once its connection was answered")+ }+ if n := len(old.Recent()); n != 1 {+ t.Fatalf("the old one answered %d of its 1 request", n)+ }+ })+ }+}+// A socket handed over for another address is closed, and the gateway// listens where settings say.func TestInheritedForAnotherAddressIsLetGo(t *testing.T) {