Every agent's model. One place. Codex on DeepSeek, Claude Code on Kimi, from the menu bar.
gateway: a streamed reply's text is gathered in a builder, not copied whole on every delta
yetone24f89cdb1465
diff --git a/internal/gateway/collector_text_test.go b/internal/gateway/collector_text_test.gonew file mode 100644index 000000000..5a5efc67d--- /dev/null+++ b/internal/gateway/collector_text_test.go@@ -0,0 +1,88 @@+package gateway++import (+ "runtime"+ "strings"+ "testing"+)++// collectStream is a streamed reply as the collector takes it: n deltas+// of thinking, then n of text, then a tool call and more text.+func collectStream(c *collector, n int) {+ c.add(Event{Kind: KStart, MsgID: "m", Model: "x"})+ for i := range n {+ c.add(Event{Kind: KThink, Text: "think " + string(rune('a'+i%26)) + " "})+ }+ c.add(Event{Kind: KSig, Text: "sig"})+ for i := range n {+ c.add(Event{Kind: KText, Text: "word " + string(rune('a'+i%26)) + " of the reply, "})+ }+ c.add(Event{Kind: KToolStart, ID: "t1", Name: "Read"})+ c.add(Event{Kind: KToolArgs, Text: `{"path":"/a"}`})+ c.add(Event{Kind: KText, Text: "after "})+ c.add(Event{Kind: KText, Text: "the call"})+ c.add(Event{Kind: KStop, Stop: "stop"})+}++// The collector's reply is the stream's, delta for delta: each part's+// text whole, in its own part, a part begun after a call its own.+func TestCollectorKeepsEveryDelta(t *testing.T) {+ var c collector+ collectStream(&c, 500)+ res := c.finish()+ var think, text strings.Builder+ for i := range 500 {+ think.WriteString("think " + string(rune('a'+i%26)) + " ")+ text.WriteString("word " + string(rune('a'+i%26)) + " of the reply, ")+ }+ if len(res.Parts) != 4 {+ t.Fatalf("parts %+v", res.Parts)+ }+ if p := res.Parts[0]; p.Kind != Thinking || p.Text != think.String() || p.Signature != "sig" {+ t.Errorf("thinking %q sig %q", p.Text[:min(40, len(p.Text))], p.Signature)+ }+ if p := res.Parts[1]; p.Kind != Text || p.Text != text.String() {+ t.Errorf("text %d bytes, want %d", len(p.Text), text.Len())+ }+ if p := res.Parts[2]; p.Kind != ToolCall || string(p.Args) != `{"path":"/a"}` {+ t.Errorf("call %+v", p)+ }+ if p := res.Parts[3]; p.Kind != Text || p.Text != "after the call" {+ t.Errorf("last text %q", p.Text)+ }+}++// A streamed reply's text is gathered in time linear in its length: the+// collector used to add each delta to the part's string, copying the whole+// reply so far every time, about 57 KB a delta on a 4000-delta reply (a+// Chat upstream to an Anthropic client copied 229 MB to stream 88 KB).+func TestCollectorTextLinear(t *testing.T) {+ const n = 4000+ var ms runtime.MemStats+ runtime.GC()+ runtime.ReadMemStats(&ms)+ before := ms.TotalAlloc+ var c collector+ collectStream(&c, n)+ res := c.finish()+ runtime.ReadMemStats(&ms)+ got := ms.TotalAlloc - before+ size := 0+ for _, p := range res.Parts {+ size += len(p.Text)+ }+ // the strings themselves, the builder's doublings, the parts: a few+ // times the reply. The quadratic copy was ~150× it here.+ if got > uint64(16*size) {+ t.Fatalf("collecting a %d-byte reply allocated %d bytes (%.0f×)", size, got, float64(got)/float64(size))+ }+}++func BenchmarkCollectorText(b *testing.B) {+ b.ReportAllocs()+ for b.Loop() {+ var c collector+ collectStream(&c, 4000)+ c.finish()+ }+}diff --git a/internal/gateway/ir.go b/internal/gateway/ir.goindex ff2dfd78e..348523f1d 100644--- a/internal/gateway/ir.go+++ b/internal/gateway/ir.go@@ -460,6 +460,11 @@ type Result struct {type collector struct {res Resultargs strings.Builder // arguments of the open tool call+ // text is what the part at textAt-1 has had added to its text since,+ // written into the part by flush: adding each delta to the part's+ // string copied the whole reply so far every time+ text strings.Builder+ textAt int// fresh: a KThinkStart said the next reasoning is a block of its ownfresh boolerr string@@ -488,6 +493,25 @@ func (c *collector) open() *Part {return nil}+// grow adds s to the text of the part at i, the last one.+func (c *collector) grow(i int, s string) {+ if c.textAt != i+1 {+ c.flush()+ c.text.WriteString(c.res.Parts[i].Text)+ c.textAt = i + 1+ }+ c.text.WriteString(s)+}++// flush writes the text grow gathered into its part.+func (c *collector) flush() {+ if c.textAt > 0 {+ c.res.Parts[c.textAt-1].Text = c.text.String()+ c.text = strings.Builder{}+ c.textAt = 0+ }+}+func (c *collector) closeTool() {if p := c.last(ToolCall); p != nil && p.Args == nil {s := strings.TrimSpace(c.args.String())@@ -512,7 +536,7 @@ func (c *collector) add(ev Event) {c.res.Usage.add(ev.Usage)case KText:if p := c.last(Text); p != nil {- p.Text += ev.Text+ c.grow(len(c.res.Parts)-1, ev.Text)} else {c.closeTool()c.res.Parts = append(c.res.Parts, Part{Kind: Text, Text: ev.Text})@@ -522,7 +546,7 @@ func (c *collector) add(ev Event) {returncase KThink:if p := c.open(); p != nil {- p.Text += ev.Text+ c.grow(len(c.res.Parts)-1, ev.Text)} else {c.closeTool()c.res.Parts = append(c.res.Parts, Part{Kind: Thinking, Text: ev.Text})@@ -581,6 +605,7 @@ func (c *collector) add(ev Event) {}func (c *collector) finish() Result {+ c.flush()c.closeTool()// a reply of tool calls ended "stop" (a relay's end_turn beside its// tool_use blocks, 蓝猫 on Discord) is the client's to run the calls