Skip to content

Every agent's model. One place. Codex on DeepSeek, Claude Code on Kimi, from the menu bar.

gateway: a streamed reply's text is gathered in a builder, not copied whole on every delta

yetone24f89cdb1465
diff --git a/internal/gateway/collector_text_test.go b/internal/gateway/collector_text_test.go
new file mode 100644
index 000000000..5a5efc67d
--- /dev/null
+++ b/internal/gateway/collector_text_test.go
@@ -0,0 +1,88 @@
+package gateway
+
+import (
+ "runtime"
+ "strings"
+ "testing"
+)
+
+// collectStream is a streamed reply as the collector takes it: n deltas
+// of thinking, then n of text, then a tool call and more text.
+func collectStream(c *collector, n int) {
+ c.add(Event{Kind: KStart, MsgID: "m", Model: "x"})
+ for i := range n {
+ c.add(Event{Kind: KThink, Text: "think " + string(rune('a'+i%26)) + " "})
+ }
+ c.add(Event{Kind: KSig, Text: "sig"})
+ for i := range n {
+ c.add(Event{Kind: KText, Text: "word " + string(rune('a'+i%26)) + " of the reply, "})
+ }
+ c.add(Event{Kind: KToolStart, ID: "t1", Name: "Read"})
+ c.add(Event{Kind: KToolArgs, Text: `{"path":"/a"}`})
+ c.add(Event{Kind: KText, Text: "after "})
+ c.add(Event{Kind: KText, Text: "the call"})
+ c.add(Event{Kind: KStop, Stop: "stop"})
+}
+
+// The collector's reply is the stream's, delta for delta: each part's
+// text whole, in its own part, a part begun after a call its own.
+func TestCollectorKeepsEveryDelta(t *testing.T) {
+ var c collector
+ collectStream(&c, 500)
+ res := c.finish()
+ var think, text strings.Builder
+ for i := range 500 {
+ think.WriteString("think " + string(rune('a'+i%26)) + " ")
+ text.WriteString("word " + string(rune('a'+i%26)) + " of the reply, ")
+ }
+ if len(res.Parts) != 4 {
+ t.Fatalf("parts %+v", res.Parts)
+ }
+ if p := res.Parts[0]; p.Kind != Thinking || p.Text != think.String() || p.Signature != "sig" {
+ t.Errorf("thinking %q sig %q", p.Text[:min(40, len(p.Text))], p.Signature)
+ }
+ if p := res.Parts[1]; p.Kind != Text || p.Text != text.String() {
+ t.Errorf("text %d bytes, want %d", len(p.Text), text.Len())
+ }
+ if p := res.Parts[2]; p.Kind != ToolCall || string(p.Args) != `{"path":"/a"}` {
+ t.Errorf("call %+v", p)
+ }
+ if p := res.Parts[3]; p.Kind != Text || p.Text != "after the call" {
+ t.Errorf("last text %q", p.Text)
+ }
+}
+
+// A streamed reply's text is gathered in time linear in its length: the
+// collector used to add each delta to the part's string, copying the whole
+// reply so far every time, about 57 KB a delta on a 4000-delta reply (a
+// Chat upstream to an Anthropic client copied 229 MB to stream 88 KB).
+func TestCollectorTextLinear(t *testing.T) {
+ const n = 4000
+ var ms runtime.MemStats
+ runtime.GC()
+ runtime.ReadMemStats(&ms)
+ before := ms.TotalAlloc
+ var c collector
+ collectStream(&c, n)
+ res := c.finish()
+ runtime.ReadMemStats(&ms)
+ got := ms.TotalAlloc - before
+ size := 0
+ for _, p := range res.Parts {
+ size += len(p.Text)
+ }
+ // the strings themselves, the builder's doublings, the parts: a few
+ // times the reply. The quadratic copy was ~150× it here.
+ if got > uint64(16*size) {
+ t.Fatalf("collecting a %d-byte reply allocated %d bytes (%.0f×)", size, got, float64(got)/float64(size))
+ }
+}
+
+func BenchmarkCollectorText(b *testing.B) {
+ b.ReportAllocs()
+ for b.Loop() {
+ var c collector
+ collectStream(&c, 4000)
+ c.finish()
+ }
+}
diff --git a/internal/gateway/ir.go b/internal/gateway/ir.go
index ff2dfd78e..348523f1d 100644
--- a/internal/gateway/ir.go
+++ b/internal/gateway/ir.go
@@ -460,6 +460,11 @@ type Result struct {
type collector struct {
res Result
args strings.Builder // arguments of the open tool call
+ // text is what the part at textAt-1 has had added to its text since,
+ // written into the part by flush: adding each delta to the part's
+ // string copied the whole reply so far every time
+ text strings.Builder
+ textAt int
// fresh: a KThinkStart said the next reasoning is a block of its own
fresh bool
err string
@@ -488,6 +493,25 @@ func (c *collector) open() *Part {
return nil
}
+// grow adds s to the text of the part at i, the last one.
+func (c *collector) grow(i int, s string) {
+ if c.textAt != i+1 {
+ c.flush()
+ c.text.WriteString(c.res.Parts[i].Text)
+ c.textAt = i + 1
+ }
+ c.text.WriteString(s)
+}
+
+// flush writes the text grow gathered into its part.
+func (c *collector) flush() {
+ if c.textAt > 0 {
+ c.res.Parts[c.textAt-1].Text = c.text.String()
+ c.text = strings.Builder{}
+ c.textAt = 0
+ }
+}
+
func (c *collector) closeTool() {
if p := c.last(ToolCall); p != nil && p.Args == nil {
s := strings.TrimSpace(c.args.String())
@@ -512,7 +536,7 @@ func (c *collector) add(ev Event) {
c.res.Usage.add(ev.Usage)
case KText:
if p := c.last(Text); p != nil {
- p.Text += ev.Text
+ c.grow(len(c.res.Parts)-1, ev.Text)
} else {
c.closeTool()
c.res.Parts = append(c.res.Parts, Part{Kind: Text, Text: ev.Text})
@@ -522,7 +546,7 @@ func (c *collector) add(ev Event) {
return
case KThink:
if p := c.open(); p != nil {
- p.Text += ev.Text
+ c.grow(len(c.res.Parts)-1, ev.Text)
} else {
c.closeTool()
c.res.Parts = append(c.res.Parts, Part{Kind: Thinking, Text: ev.Text})
@@ -581,6 +605,7 @@ func (c *collector) add(ev Event) {
}
func (c *collector) finish() Result {
+ c.flush()
c.closeTool()
// a reply of tool calls ended "stop" (a relay's end_turn beside its
// tool_use blocks, 蓝猫 on Discord) is the client's to run the calls