Skip to content

Every agent's model. One place. Codex on DeepSeek, Claude Code on Kimi, from the menu bar.

gateway: a relayed Responses stream's tidies pass its events in place (#3383)

yetonedcfda5f166fd
diff --git a/internal/gateway/itemids.go b/internal/gateway/itemids.go
index 126a1bba8..6c0c7f914 100644
--- a/internal/gateway/itemids.go
+++ b/internal/gateway/itemids.go
@@ -23,15 +23,9 @@ type idOf struct{ id, typ string }
func (t *idTidy) write(b []byte) []byte {
t.buf = append(t.buf, b...)
- var out []byte
- for {
- n := eventEnd(t.buf)
- if n < 0 {
- return out
- }
- out = append(out, t.event(t.buf[:n])...)
- t.buf = t.buf[n:]
- }
+ out, rest := tidyEvents(t.buf, t.event)
+ t.buf = rest
+ return out
}
func (t *idTidy) flush() []byte {
@@ -43,7 +37,7 @@ func (t *idTidy) flush() []byte {
func (t *idTidy) event(ev []byte) []byte {
data := eventData(ev)
if len(data) == 0 || !gjson.ValidBytes(data) {
- return append([]byte(nil), ev...)
+ return ev
}
e := gjson.ParseBytes(data)
at := e.Get("output_index")
@@ -55,23 +49,23 @@ func (t *idTidy) event(ev []byte) []byte {
}
t.ids[at.Int()] = idOf{id, e.Get("item.type").String()}
}
- return append([]byte(nil), ev...)
+ return ev
case typ == "response.completed" || typ == "response.incomplete" || typ == "response.failed":
return t.ended(ev, data, e)
case !at.Exists():
- return append([]byte(nil), ev...)
+ return ev
}
was, ok := t.ids[at.Int()]
if !ok {
- return append([]byte(nil), ev...)
+ return ev
}
itemID, itemIn := e.Get("item_id"), e.Get("item.id")
if (!itemID.Exists() || itemID.String() == was.id) && (!itemIn.Exists() || itemIn.String() == was.id) {
- return append([]byte(nil), ev...)
+ return ev
}
var m map[string]any
if decodeNumbers(data, &m) != nil {
- return append([]byte(nil), ev...)
+ return ev
}
if itemID.Exists() {
m["item_id"] = was.id
@@ -81,7 +75,7 @@ func (t *idTidy) event(ev []byte) []byte {
}
nb, err := marshalPlain(m)
if err != nil {
- return append([]byte(nil), ev...)
+ return ev
}
return withData(ev, nb)
}
@@ -97,11 +91,11 @@ func (t *idTidy) ended(ev, data []byte, e gjson.Result) []byte {
}
}
if !differs {
- return append([]byte(nil), ev...)
+ return ev
}
var m map[string]any
if decodeNumbers(data, &m) != nil {
- return append([]byte(nil), ev...)
+ return ev
}
res, _ := m["response"].(map[string]any)
out, _ := res["output"].([]any)
@@ -117,7 +111,7 @@ func (t *idTidy) ended(ev, data []byte, e gjson.Result) []byte {
}
nb, err := marshalPlain(m)
if err != nil {
- return append([]byte(nil), ev...)
+ return ev
}
return withData(ev, nb)
}
diff --git a/internal/gateway/phase.go b/internal/gateway/phase.go
index 58301fde0..ff34f0305 100644
--- a/internal/gateway/phase.go
+++ b/internal/gateway/phase.go
@@ -42,6 +42,28 @@ func unphased(it map[string]any) bool {
return p == ""
}
+// anyUnphased reports whether a reply's output has an item unphased
+// would give a phase, read in place: a reply whose messages all have one
+// (OpenAI's own) goes as it came without being decoded whole.
+func anyUnphased(output gjson.Result) bool {
+ if !output.IsArray() {
+ return false
+ }
+ found := false
+ output.ForEach(func(_, it gjson.Result) bool {
+ typ, role, phase := it.Get("type"), it.Get("role"), it.Get("phase")
+ if typ.Type != gjson.String || typ.Str != "message" {
+ return true
+ }
+ if role.Exists() && (role.Type != gjson.String || role.Str != "assistant") {
+ return true
+ }
+ found = phase.Type != gjson.String || phase.Str == ""
+ return !found
+ })
+ return found
+}
+
// phaseOutput gives each assistant message in a reply's output its phase,
// and reports whether it gave any. done is whether the reply ended
// completed: only then is the last message the final answer.
@@ -87,15 +109,29 @@ func (t *phaseTidy) write(b []byte) []byte {
if !t.sse {
return nil
}
- var out []byte
+ out, rest := tidyEvents(t.buf, t.event)
+ t.buf = rest
+ return out
+}
+
+// tidyEvents is each whole event in buf through event, and buf's part of
+// an event that hasn't ended, moved to buf's front so the next write
+// appends into the same bytes. What event gives may be its ev, a slice of
+// buf, so it is copied out before buf is written again.
+func tidyEvents(buf []byte, event func(ev []byte) []byte) (out, rest []byte) {
+ at := 0
for {
- n := eventEnd(t.buf)
+ n := eventEnd(buf[at:])
if n < 0 {
- return out
+ break
+ }
+ if out == nil {
+ out = make([]byte, 0, len(buf))
}
- out = append(out, t.event(t.buf[:n])...)
- t.buf = t.buf[n:]
+ out = append(out, event(buf[at:at+n])...)
+ at += n
}
+ return out, append(buf[:0], buf[at:]...)
}
func (t *phaseTidy) flush() []byte {
@@ -113,19 +149,32 @@ func (t *phaseTidy) flush() []byte {
return append(out, b...)
}
-// eventData is an event's data, joined as SSE joins it.
+// eventData is an event's data, joined as SSE joins it. An event of one
+// data line, as nearly every upstream's is, gets ev's own bytes, so what
+// it gives is only read.
func eventData(ev []byte) []byte {
- var data []byte
+ var data, first []byte
n := 0
- for _, line := range bytes.Split(ev, []byte("\n")) {
- line = bytes.TrimRight(line, "\r")
- if d, ok := bytes.CutPrefix(line, []byte("data:")); ok {
- if n > 0 {
- data = append(data, '\n')
- }
- data = append(data, bytes.TrimPrefix(d, []byte(" "))...)
- n++
+ for rest := ev; len(rest) > 0; {
+ var line []byte
+ line, rest, _ = bytes.Cut(rest, []byte("\n"))
+ d, ok := bytes.CutPrefix(bytes.TrimRight(line, "\r"), []byte("data:"))
+ if !ok {
+ continue
}
+ d = bytes.TrimPrefix(d, []byte(" "))
+ switch n {
+ case 0:
+ first = d
+ case 1:
+ data = append(append(append([]byte(nil), first...), '\n'), d...)
+ default:
+ data = append(append(data, '\n'), d...)
+ }
+ n++
+ }
+ if n == 1 {
+ return first
}
return data
}
@@ -177,7 +226,7 @@ func (t *phaseTidy) event(ev []byte) []byte {
case "response.incomplete":
return t.ended(ev, data, false)
}
- return append([]byte(nil), ev...)
+ return ev
}
func (t *phaseTidy) hold(ev []byte) []byte {
@@ -207,18 +256,21 @@ func (t *phaseTidy) release(phase string) []byte {
// ended is the reply's last event, its output's messages given their phase
// as the stream gave them.
func (t *phaseTidy) ended(ev, data []byte, done bool) []byte {
+ if !anyUnphased(gjson.GetBytes(data, "response.output")) {
+ return ev
+ }
var e map[string]any
if decodeNumbers(data, &e) != nil {
- return append([]byte(nil), ev...)
+ return ev
}
res, _ := e["response"].(map[string]any)
out, _ := res["output"].([]any)
if !phaseOutput(out, done) {
- return append([]byte(nil), ev...)
+ return ev
}
nb, err := marshalPlain(e)
if err != nil {
- return append([]byte(nil), ev...)
+ return ev
}
return withData(ev, nb)
}
diff --git a/internal/gateway/tidy_inplace_test.go b/internal/gateway/tidy_inplace_test.go
new file mode 100644
index 000000000..7d1cd2b54
--- /dev/null
+++ b/internal/gateway/tidy_inplace_test.go
@@ -0,0 +1,81 @@
+package gateway
+
+import (
+ "bytes"
+ "encoding/json"
+ "testing"
+
+ "github.com/tidwall/gjson"
+)
+
+// anyUnphased, read in place, finds the items unphased finds in a decoded
+// reply, so a reply it lets through as it came is one phaseOutput would
+// have left alone.
+func TestAnyUnphasedAgreesWithUnphased(t *testing.T) {
+ items := []string{
+ `{"type":"message","role":"assistant","phase":"commentary"}`,
+ `{"type":"message","role":"assistant"}`,
+ `{"type":"message"}`,
+ `{"type":"message","role":"user"}`,
+ `{"type":"message","role":null}`,
+ `{"type":"message","role":"assistant","phase":""}`,
+ `{"type":"message","role":"assistant","phase":null}`,
+ `{"type":"message","role":"assistant","phase":1}`,
+ `{"type":"reasoning"}`,
+ `{"type":"function_call","name":"shell"}`,
+ `{"type":["message"]}`,
+ `"message"`,
+ }
+ for _, it := range items {
+ var m map[string]any
+ _ = json.Unmarshal([]byte(it), &m)
+ want := unphased(m)
+ if got := anyUnphased(gjson.Parse("[" + it + "]")); got != want {
+ t.Errorf("%s: anyUnphased %v, unphased %v", it, got, want)
+ }
+ two := `[{"type":"message","role":"assistant","phase":"final_answer"},` + it + `]`
+ if got := anyUnphased(gjson.Parse(two)); got != want {
+ t.Errorf("after a phased message, %s: anyUnphased %v, unphased %v", it, got, want)
+ }
+ }
+ if anyUnphased(gjson.Parse(`{"type":"message"}`)) {
+ t.Error("an output that isn't a list has nothing to phase")
+ }
+}
+
+func TestEventDataJoinsAsSSE(t *testing.T) {
+ for ev, want := range map[string]string{
+ "event: x\ndata: {\"a\":1}\n\n": `{"a":1}`,
+ "data:{\"a\":1}\r\n\r\n": `{"a":1}`,
+ "data: a\ndata: b\ndata: c\n\n": "a\n b\nc",
+ ": ping\n\n": "",
+ "id: 1\ndata: one\r\nretry: 5\n\n": "one",
+ } {
+ if got := string(eventData([]byte(ev))); got != want {
+ t.Errorf("%q: %q, want %q", ev, got, want)
+ }
+ }
+}
+
+// A stream split anywhere comes out of both tidies as it went in when
+// nothing in it needs tidying, however its writes fall.
+func TestTidiesPassAStreamSplitAnywhere(t *testing.T) {
+ stream := "event: response.output_item.added\ndata: {\"type\":\"response.output_item.added\",\"output_index\":0,\"item\":{\"id\":\"m1\",\"type\":\"message\"}}\n\n" +
+ "data: {\"type\":\"response.output_text.delta\",\"output_index\":0,\"item_id\":\"m1\",\"delta\":\"hi\"}\r\n\r\n" +
+ "event: response.output_item.done\ndata: {\"type\":\"response.output_item.done\",\"output_index\":0,\"item\":{\"id\":\"m1\",\"type\":\"message\",\"role\":\"assistant\",\"phase\":\"final_answer\"}}\n\n" +
+ "data: {\"type\":\"response.completed\",\"response\":{\"output\":[{\"id\":\"m1\",\"type\":\"message\",\"role\":\"assistant\",\"phase\":\"final_answer\"}]}}\n\n"
+ for _, step := range []int{1, 7, 64, len(stream)} {
+ ids, phases := &idTidy{}, &phaseTidy{sse: true}
+ var got []byte
+ for i := 0; i < len(stream); i += step {
+ b := []byte(stream[i:min(i+step, len(stream))])
+ got = append(got, phases.write(ids.write(b))...)
+ clear(b) // the caller's buffer is read again for its next write
+ }
+ got = append(got, phases.write(ids.flush())...)
+ got = append(got, phases.flush()...)
+ if !bytes.Equal(got, []byte(stream)) {
+ t.Fatalf("written %d bytes at a time:\n%s\nwant\n%s", step, got, stream)
+ }
+ }
+}