Every agent's model. One place. Codex on DeepSeek, Claude Code on Kimi, from the menu bar.
gateway: a relayed Responses stream's tidies pass its events in place (#3383)
yetonedcfda5f166fd
diff --git a/internal/gateway/itemids.go b/internal/gateway/itemids.goindex 126a1bba8..6c0c7f914 100644--- a/internal/gateway/itemids.go+++ b/internal/gateway/itemids.go@@ -23,15 +23,9 @@ type idOf struct{ id, typ string }func (t *idTidy) write(b []byte) []byte {t.buf = append(t.buf, b...)- var out []byte- for {- n := eventEnd(t.buf)- if n < 0 {- return out- }- out = append(out, t.event(t.buf[:n])...)- t.buf = t.buf[n:]- }+ out, rest := tidyEvents(t.buf, t.event)+ t.buf = rest+ return out}func (t *idTidy) flush() []byte {@@ -43,7 +37,7 @@ func (t *idTidy) flush() []byte {func (t *idTidy) event(ev []byte) []byte {data := eventData(ev)if len(data) == 0 || !gjson.ValidBytes(data) {- return append([]byte(nil), ev...)+ return ev}e := gjson.ParseBytes(data)at := e.Get("output_index")@@ -55,23 +49,23 @@ func (t *idTidy) event(ev []byte) []byte {}t.ids[at.Int()] = idOf{id, e.Get("item.type").String()}}- return append([]byte(nil), ev...)+ return evcase typ == "response.completed" || typ == "response.incomplete" || typ == "response.failed":return t.ended(ev, data, e)case !at.Exists():- return append([]byte(nil), ev...)+ return ev}was, ok := t.ids[at.Int()]if !ok {- return append([]byte(nil), ev...)+ return ev}itemID, itemIn := e.Get("item_id"), e.Get("item.id")if (!itemID.Exists() || itemID.String() == was.id) && (!itemIn.Exists() || itemIn.String() == was.id) {- return append([]byte(nil), ev...)+ return ev}var m map[string]anyif decodeNumbers(data, &m) != nil {- return append([]byte(nil), ev...)+ return ev}if itemID.Exists() {m["item_id"] = was.id@@ -81,7 +75,7 @@ func (t *idTidy) event(ev []byte) []byte {}nb, err := marshalPlain(m)if err != nil {- return append([]byte(nil), ev...)+ return ev}return withData(ev, nb)}@@ -97,11 +91,11 @@ func (t *idTidy) ended(ev, data []byte, e gjson.Result) []byte {}}if !differs {- return append([]byte(nil), ev...)+ return ev}var m map[string]anyif decodeNumbers(data, &m) != nil {- return append([]byte(nil), ev...)+ return ev}res, _ := m["response"].(map[string]any)out, _ := res["output"].([]any)@@ -117,7 +111,7 @@ func (t *idTidy) ended(ev, data []byte, e gjson.Result) []byte {}nb, err := marshalPlain(m)if err != nil {- return append([]byte(nil), ev...)+ return ev}return withData(ev, nb)}diff --git a/internal/gateway/phase.go b/internal/gateway/phase.goindex 58301fde0..ff34f0305 100644--- a/internal/gateway/phase.go+++ b/internal/gateway/phase.go@@ -42,6 +42,28 @@ func unphased(it map[string]any) bool {return p == ""}+// anyUnphased reports whether a reply's output has an item unphased+// would give a phase, read in place: a reply whose messages all have one+// (OpenAI's own) goes as it came without being decoded whole.+func anyUnphased(output gjson.Result) bool {+ if !output.IsArray() {+ return false+ }+ found := false+ output.ForEach(func(_, it gjson.Result) bool {+ typ, role, phase := it.Get("type"), it.Get("role"), it.Get("phase")+ if typ.Type != gjson.String || typ.Str != "message" {+ return true+ }+ if role.Exists() && (role.Type != gjson.String || role.Str != "assistant") {+ return true+ }+ found = phase.Type != gjson.String || phase.Str == ""+ return !found+ })+ return found+}+// phaseOutput gives each assistant message in a reply's output its phase,// and reports whether it gave any. done is whether the reply ended// completed: only then is the last message the final answer.@@ -87,15 +109,29 @@ func (t *phaseTidy) write(b []byte) []byte {if !t.sse {return nil}- var out []byte+ out, rest := tidyEvents(t.buf, t.event)+ t.buf = rest+ return out+}++// tidyEvents is each whole event in buf through event, and buf's part of+// an event that hasn't ended, moved to buf's front so the next write+// appends into the same bytes. What event gives may be its ev, a slice of+// buf, so it is copied out before buf is written again.+func tidyEvents(buf []byte, event func(ev []byte) []byte) (out, rest []byte) {+ at := 0for {- n := eventEnd(t.buf)+ n := eventEnd(buf[at:])if n < 0 {- return out+ break+ }+ if out == nil {+ out = make([]byte, 0, len(buf))}- out = append(out, t.event(t.buf[:n])...)- t.buf = t.buf[n:]+ out = append(out, event(buf[at:at+n])...)+ at += n}+ return out, append(buf[:0], buf[at:]...)}func (t *phaseTidy) flush() []byte {@@ -113,19 +149,32 @@ func (t *phaseTidy) flush() []byte {return append(out, b...)}-// eventData is an event's data, joined as SSE joins it.+// eventData is an event's data, joined as SSE joins it. An event of one+// data line, as nearly every upstream's is, gets ev's own bytes, so what+// it gives is only read.func eventData(ev []byte) []byte {- var data []byte+ var data, first []byten := 0- for _, line := range bytes.Split(ev, []byte("\n")) {- line = bytes.TrimRight(line, "\r")- if d, ok := bytes.CutPrefix(line, []byte("data:")); ok {- if n > 0 {- data = append(data, '\n')- }- data = append(data, bytes.TrimPrefix(d, []byte(" "))...)- n+++ for rest := ev; len(rest) > 0; {+ var line []byte+ line, rest, _ = bytes.Cut(rest, []byte("\n"))+ d, ok := bytes.CutPrefix(bytes.TrimRight(line, "\r"), []byte("data:"))+ if !ok {+ continue}+ d = bytes.TrimPrefix(d, []byte(" "))+ switch n {+ case 0:+ first = d+ case 1:+ data = append(append(append([]byte(nil), first...), '\n'), d...)+ default:+ data = append(append(data, '\n'), d...)+ }+ n+++ }+ if n == 1 {+ return first}return data}@@ -177,7 +226,7 @@ func (t *phaseTidy) event(ev []byte) []byte {case "response.incomplete":return t.ended(ev, data, false)}- return append([]byte(nil), ev...)+ return ev}func (t *phaseTidy) hold(ev []byte) []byte {@@ -207,18 +256,21 @@ func (t *phaseTidy) release(phase string) []byte {// ended is the reply's last event, its output's messages given their phase// as the stream gave them.func (t *phaseTidy) ended(ev, data []byte, done bool) []byte {+ if !anyUnphased(gjson.GetBytes(data, "response.output")) {+ return ev+ }var e map[string]anyif decodeNumbers(data, &e) != nil {- return append([]byte(nil), ev...)+ return ev}res, _ := e["response"].(map[string]any)out, _ := res["output"].([]any)if !phaseOutput(out, done) {- return append([]byte(nil), ev...)+ return ev}nb, err := marshalPlain(e)if err != nil {- return append([]byte(nil), ev...)+ return ev}return withData(ev, nb)}diff --git a/internal/gateway/tidy_inplace_test.go b/internal/gateway/tidy_inplace_test.gonew file mode 100644index 000000000..7d1cd2b54--- /dev/null+++ b/internal/gateway/tidy_inplace_test.go@@ -0,0 +1,81 @@+package gateway++import (+ "bytes"+ "encoding/json"+ "testing"++ "github.com/tidwall/gjson"+)++// anyUnphased, read in place, finds the items unphased finds in a decoded+// reply, so a reply it lets through as it came is one phaseOutput would+// have left alone.+func TestAnyUnphasedAgreesWithUnphased(t *testing.T) {+ items := []string{+ `{"type":"message","role":"assistant","phase":"commentary"}`,+ `{"type":"message","role":"assistant"}`,+ `{"type":"message"}`,+ `{"type":"message","role":"user"}`,+ `{"type":"message","role":null}`,+ `{"type":"message","role":"assistant","phase":""}`,+ `{"type":"message","role":"assistant","phase":null}`,+ `{"type":"message","role":"assistant","phase":1}`,+ `{"type":"reasoning"}`,+ `{"type":"function_call","name":"shell"}`,+ `{"type":["message"]}`,+ `"message"`,+ }+ for _, it := range items {+ var m map[string]any+ _ = json.Unmarshal([]byte(it), &m)+ want := unphased(m)+ if got := anyUnphased(gjson.Parse("[" + it + "]")); got != want {+ t.Errorf("%s: anyUnphased %v, unphased %v", it, got, want)+ }+ two := `[{"type":"message","role":"assistant","phase":"final_answer"},` + it + `]`+ if got := anyUnphased(gjson.Parse(two)); got != want {+ t.Errorf("after a phased message, %s: anyUnphased %v, unphased %v", it, got, want)+ }+ }+ if anyUnphased(gjson.Parse(`{"type":"message"}`)) {+ t.Error("an output that isn't a list has nothing to phase")+ }+}++func TestEventDataJoinsAsSSE(t *testing.T) {+ for ev, want := range map[string]string{+ "event: x\ndata: {\"a\":1}\n\n": `{"a":1}`,+ "data:{\"a\":1}\r\n\r\n": `{"a":1}`,+ "data: a\ndata: b\ndata: c\n\n": "a\n b\nc",+ ": ping\n\n": "",+ "id: 1\ndata: one\r\nretry: 5\n\n": "one",+ } {+ if got := string(eventData([]byte(ev))); got != want {+ t.Errorf("%q: %q, want %q", ev, got, want)+ }+ }+}++// A stream split anywhere comes out of both tidies as it went in when+// nothing in it needs tidying, however its writes fall.+func TestTidiesPassAStreamSplitAnywhere(t *testing.T) {+ stream := "event: response.output_item.added\ndata: {\"type\":\"response.output_item.added\",\"output_index\":0,\"item\":{\"id\":\"m1\",\"type\":\"message\"}}\n\n" ++ "data: {\"type\":\"response.output_text.delta\",\"output_index\":0,\"item_id\":\"m1\",\"delta\":\"hi\"}\r\n\r\n" ++ "event: response.output_item.done\ndata: {\"type\":\"response.output_item.done\",\"output_index\":0,\"item\":{\"id\":\"m1\",\"type\":\"message\",\"role\":\"assistant\",\"phase\":\"final_answer\"}}\n\n" ++ "data: {\"type\":\"response.completed\",\"response\":{\"output\":[{\"id\":\"m1\",\"type\":\"message\",\"role\":\"assistant\",\"phase\":\"final_answer\"}]}}\n\n"+ for _, step := range []int{1, 7, 64, len(stream)} {+ ids, phases := &idTidy{}, &phaseTidy{sse: true}+ var got []byte+ for i := 0; i < len(stream); i += step {+ b := []byte(stream[i:min(i+step, len(stream))])+ got = append(got, phases.write(ids.write(b))...)+ clear(b) // the caller's buffer is read again for its next write+ }+ got = append(got, phases.write(ids.flush())...)+ got = append(got, phases.flush()...)+ if !bytes.Equal(got, []byte(stream)) {+ t.Fatalf("written %d bytes at a time:\n%s\nwant\n%s", step, got, stream)+ }+ }+}