Skip to content

The cross-harness, self-improving memory layer for AI agents. Join our community: https://discord.com/invite/zdNChS2fBu

perf(gateway): walk callout payloads once, splitting the messages as the adapter passes them (#1271)

kyle-elliott-asymptote538f0395fee3
diff --git a/gateway/gateway.go b/gateway/gateway.go
index 8985e8b8..8689b47c 100644
--- a/gateway/gateway.go
+++ b/gateway/gateway.go
@@ -77,6 +77,11 @@ type Call struct {
// it exactly as it named the request's, so the reply joins its request's session without
// waiting for the request's analysis or asking the store.
RequestBody []byte
+ // BodyMessages, when set, are the elements of the message array in a request's Body, in order,
+ // as slices of Body, split by an adapter that walked them while reading its own payload. The
+ // pipeline decodes Body without walking the array again; the split is checked against Body,
+ // not trusted, and one that does not match is ignored (codec.SkimSplitRequest).
+ BodyMessages [][]byte
// Release, when set, is called by the Handler once, when it will read nothing of Body or
// RequestBody again, which may be after Handle returns. An adapter that reads bodies into
// pooled buffers returns them here (ReadBody); nothing derived from a body refers to it.
diff --git a/gateway/internal/jsonscan/jsonscan.go b/gateway/internal/jsonscan/jsonscan.go
index bf4c4c0d..21a1fb9e 100644
--- a/gateway/internal/jsonscan/jsonscan.go
+++ b/gateway/internal/jsonscan/jsonscan.go
@@ -32,13 +32,75 @@ func Object(b []byte, yield func(name string, value []byte) error) error {
// the whole value and walking it again reads the bytes once: a gateway envelope's request object
// holds the whole conversation. enter may be nil.
func Members(b []byte, enter func(name string) bool, yield func(parent, name string, value []byte) error) error {
- end, err := object(b, ws(b, 0), "", enter, yield)
+ return Split(b, enter, nil, func(parent, name string, value []byte, _ [][]byte) error { return yield(parent, name, value) })
+}
+
+// Split is Members, splitting an array it passes into its elements in the same pass. split is
+// asked about each array or object value on the way, by kind and by its path of member names from
+// the top (unescaped): an array it accepts is split, into elements that alias b (bracket-matched,
+// as Elements finds them); an object it accepts is walked through, still yielded whole, so that an
+// array inside it can be split. A member is yielded with the elements of the last array split
+// within its value, if any. Finding where an array ends walks each of its elements already, so
+// splitting it costs nothing more, and spares a reader of the elements, such as the codec, walking
+// the array again. split may be nil.
+func Split(b []byte, enter func(name string) bool, split func(array bool, path ...string) bool, yield func(parent, name string, value []byte, elems [][]byte) error) error {
+ end, err := object(b, ws(b, 0), func(name string, start int) (int, error) {
+ if enter != nil && start < len(b) && b[start] == '{' && enter(name) {
+ return object(b, start, func(member string, start int) (int, error) {
+ return yieldValue(b, start, split, yield, name, member)
+ })
+ }
+ return yieldValue(b, start, split, yield, "", name)
+ })
if err != nil {
return err
}
return trailing(b, end)
}
+// yieldValue finds the end of the value at b[start] of member name (under parent), splitting
+// what split asks for, and yields it.
+func yieldValue(b []byte, start int, split func(array bool, path ...string) bool, yield func(parent, name string, value []byte, elems [][]byte) error, parent, name string) (int, error) {
+ var (
+ end int
+ elems [][]byte
+ err error
+ )
+ switch {
+ case split == nil:
+ end, err = skipValue(b, start)
+ case parent == "":
+ end, elems, err = value(b, start, []string{name}, split)
+ default:
+ end, elems, err = value(b, start, []string{parent, name}, split)
+ }
+ if err != nil {
+ return 0, err
+ }
+ return end, yield(parent, name, b[start:end], elems)
+}
+
+// value returns the index just past the value at b[i], at path, and the elements of the last
+// array within it that split asks to split.
+func value(b []byte, i int, path []string, split func(array bool, path ...string) bool) (end int, elems [][]byte, err error) {
+ if i >= len(b) || (b[i] != '[' && b[i] != '{') || !split(b[i] == '[', path...) {
+ end, err = skipValue(b, i)
+ return end, nil, err
+ }
+ if b[i] == '[' {
+ end, err = elements(b, i, func(e []byte) bool { elems = append(elems, e); return true })
+ return end, elems, err
+ }
+ end, err = object(b, i, func(name string, start int) (int, error) {
+ end, e, err := value(b, start, append(path[:len(path):len(path)], name), split)
+ if e != nil {
+ elems = e
+ }
+ return end, err
+ })
+ return end, elems, err
+}
+
// errFound stops a walk once FirstString has its member.
var errFound = errors.New("found")
@@ -111,32 +173,38 @@ func MemberStart(b []byte, name string, before func(name, value []byte)) (int, b
// yield returns false. It reads only as far as the element yield stops at; elements are bracket-
// matched, not validated.
func Elements(b []byte, i int, yield func(elem []byte) bool) error {
+ _, err := elements(b, i, yield)
+ return err
+}
+
+// elements is Elements, returning the index just past the array when yield never stops it.
+func elements(b []byte, i int, yield func(elem []byte) bool) (int, error) {
if i >= len(b) || b[i] != '[' {
- return errSyntax
+ return 0, errSyntax
}
i = ws(b, i+1)
if i < len(b) && b[i] == ']' {
- return nil
+ return i + 1, nil
}
for {
end, err := skipValue(b, i)
if err != nil {
- return err
+ return 0, err
}
if !yield(b[i:end]) {
- return nil
+ return end, nil
}
i = ws(b, end)
if i >= len(b) {
- return errSyntax
+ return 0, errSyntax
}
switch b[i] {
case ',':
i = ws(b, i+1)
case ']':
- return nil
+ return i + 1, nil
default:
- return errSyntax
+ return 0, errSyntax
}
}
}
@@ -145,9 +213,10 @@ func Elements(b []byte, i int, yield func(elem []byte) bool) error {
// matched brackets, or a literal. It does not validate what is inside.
func ValueEnd(b []byte, i int) (int, error) { return skipValue(b, i) }
-// object walks the object starting at b[i] == '{', yielding its members under parent, and returns
-// the index just past it. Only the top level (parent "") enters members.
-func object(b []byte, i int, parent string, enter func(string) bool, yield func(parent, name string, value []byte) error) (int, error) {
+// object walks the object starting at b[i] == '{', calling member with each member's name,
+// unescaped, and the offset of its value, which member returns the end of; it returns the index
+// just past the object.
+func object(b []byte, i int, member func(name string, start int) (int, error)) (int, error) {
if i >= len(b) || b[i] != '{' {
return 0, errSyntax
}
@@ -171,13 +240,7 @@ func object(b []byte, i int, parent string, enter func(string) bool, yield func(
if i >= len(b) || b[i] != ':' {
return 0, errSyntax
}
- start := ws(b, i+1)
- if parent == "" && enter != nil && start < len(b) && b[start] == '{' && enter(string(name)) {
- end, err = object(b, start, string(name), nil, yield)
- } else if end, err = skipValue(b, start); err == nil {
- err = yield(parent, string(name), b[start:end])
- }
- if err != nil {
+ if end, err = member(string(name), ws(b, i+1)); err != nil {
return 0, err
}
i = ws(b, end)
diff --git a/gateway/internal/jsonscan/jsonscan_test.go b/gateway/internal/jsonscan/jsonscan_test.go
index 101f4926..0070c254 100644
--- a/gateway/internal/jsonscan/jsonscan_test.go
+++ b/gateway/internal/jsonscan/jsonscan_test.go
@@ -207,3 +207,57 @@ func TestMemberStartBefore(t *testing.T) {
t.Errorf("MemberStart = %d, %v, before saw %q, want %q", at, ok, got, want)
}
}
+
+// Split yields every member as Members does, and splits exactly the arrays split asks for, by kind
+// and path: walking through an object to an array inside it, with the elements of the last array
+// split within a value, and nothing for an array split did not ask for.
+func TestSplit(t *testing.T) {
+ body := `{"request":{"json":{"model":"m","messages":[{"a":"]"}, "x",[1]],"tools":[2]},"n":1},` +
+ `"messages":[3,4],"messages":[5],"json":{"messages":[6]},"other":[7]}`
+ split := func(array bool, path ...string) bool {
+ switch {
+ case !array && len(path) == 2 && path[0] == "request" && path[1] == "json":
+ return true
+ case array && len(path) == 3 && path[2] == "messages":
+ return true
+ }
+ return array && len(path) == 1 && path[0] == "messages"
+ }
+ type yielded struct {
+ parent, name, value string
+ elems []string
+ }
+ var got []yielded
+ err := Split([]byte(body), func(name string) bool { return name == "request" }, split, func(parent, name string, value []byte, elems [][]byte) error {
+ y := yielded{parent: parent, name: name, value: string(value)}
+ for _, e := range elems {
+ y.elems = append(y.elems, string(e))
+ }
+ got = append(got, y)
+ return nil
+ })
+ want := []yielded{
+ {"request", "json", `{"model":"m","messages":[{"a":"]"}, "x",[1]],"tools":[2]}`, []string{`{"a":"]"}`, `"x"`, `[1]`}},
+ {"request", "n", `1`, nil},
+ {"", "messages", `[3,4]`, []string{`3`, `4`}},
+ {"", "messages", `[5]`, []string{`5`}},
+ {"", "json", `{"messages":[6]}`, nil}, // an object split did not ask to walk through
+ {"", "other", `[7]`, nil},
+ }
+ if err != nil || !slices.EqualFunc(got, want, func(a, b yielded) bool {
+ return a.parent == b.parent && a.name == b.name && a.value == b.value && slices.Equal(a.elems, b.elems)
+ }) {
+ t.Errorf("Split = %v\n%+v\nwant %+v", err, got, want)
+ }
+ // An array where split walks through objects is not split, and a value that is not an
+ // array or object is skipped as usual.
+ err = Split([]byte(`{"request":{"json":[1,2]}}`), func(string) bool { return true }, split, func(_, _ string, _ []byte, elems [][]byte) error {
+ if elems != nil {
+ t.Errorf("an array split asked to walk through as an object was split: %q", elems)
+ }
+ return nil
+ })
+ if err != nil {
+ t.Error(err)
+ }
+}
diff --git a/gateway/litellm/bench_test.go b/gateway/litellm/bench_test.go
new file mode 100644
index 00000000..2c8a55cf
--- /dev/null
+++ b/gateway/litellm/bench_test.go
@@ -0,0 +1,101 @@
+package litellm
+
+import (
+ "bytes"
+ "context"
+ "fmt"
+ "net/http"
+ "net/http/httptest"
+ "testing"
+
+ "github.com/asymptote-labs/agent-beacon/gateway/events"
+ "github.com/asymptote-labs/agent-beacon/gateway/internal/synth"
+ "github.com/asymptote-labs/agent-beacon/gateway/llm"
+ "github.com/asymptote-labs/agent-beacon/gateway/llm/codec"
+ "github.com/asymptote-labs/agent-beacon/gateway/pipeline"
+)
+
+// benchPayload is a request-phase guardrail payload at turn k of a long session, laid out as
+// LiteLLM sends it: the messages, the tools, then texts, a copy of every message's text.
+func benchPayload(k int) []byte {
+ s := synth.NewScript(1, 0, k, 3000)
+ pc := [][]byte{[]byte(`{"input_type":"request","litellm_call_id":"c1","litellm_trace_id":"t1","model":"` + synth.Model + `","structured_messages":[`)}
+ pc = s.ChatMessages(pc, k, synth.Benign)
+ pc = append(pc, []byte(`],"tools":`), s.ChatTools(), []byte(`,"texts":[`))
+ pc = s.Texts(pc, k, synth.Benign)
+ pc = append(pc, []byte(`],"request_headers":{"x-claude-code-session-id":"s1","user-agent":"claude-cli/2.1.0"},"request_data":{"user_api_key_user_id":"u"}}`))
+ return synth.AppendJoin(nil, pc)
+}
+
+type inlineAnalyzer struct{}
+
+func (inlineAnalyzer) Name() string { return "inline" }
+func (inlineAnalyzer) Inline() bool { return true }
+func (inlineAnalyzer) Analyze(context.Context, *pipeline.Turn) (pipeline.Report, error) {
+ return pipeline.Report{}, nil
+}
+
+// BenchmarkRequestPhase is a request-phase call's work in this process, by stage, in the steady
+// state of a long session (the codec's memo warm with its history): walking the payload, building
+// the Chat body over it, skimming that body, and the whole call through a pipeline that analyzes
+// before it answers.
+func BenchmarkRequestPhase(b *testing.B) {
+ for _, k := range []int{100, 300} {
+ payload := benchPayload(k)
+ name := fmt.Sprintf("turns=%d/%dKB", k, len(payload)>>10)
+ buf := make([]byte, len(payload))
+ b.Run(name+"/walk", func(b *testing.B) {
+ b.SetBytes(int64(len(payload)))
+ for b.Loop() {
+ var req request
+ if err := decodeRequest(payload, &req); err != nil {
+ b.Fatal(err)
+ }
+ }
+ })
+ b.Run(name+"/build", func(b *testing.B) {
+ b.SetBytes(int64(len(payload)))
+ for b.Loop() {
+ copy(buf, payload)
+ var req request
+ _ = decodeRequest(buf, &req)
+ if _, _, err := requestBody(req, buf); err != nil {
+ b.Fatal(err)
+ }
+ }
+ })
+ copy(buf, payload)
+ var req request
+ _ = decodeRequest(buf, &req)
+ body, split, _ := requestBody(req, buf)
+ if _, err := codec.SkimSplitRequest(llm.OpenAIChat, body, split); err != nil {
+ b.Fatal(err)
+ }
+ b.Run(name+"/skim", func(b *testing.B) {
+ b.SetBytes(int64(len(payload)))
+ for b.Loop() {
+ if _, err := codec.SkimSplitRequest(llm.OpenAIChat, body, split); err != nil {
+ b.Fatal(err)
+ }
+ }
+ })
+ b.Run(name+"/call", func(b *testing.B) {
+ ev, err := events.New(events.Config{})
+ if err != nil {
+ b.Fatal(err)
+ }
+ p := pipeline.New(pipeline.Config{Analyzers: []pipeline.Analyzer{ev, inlineAnalyzer{}}})
+ defer p.Close()
+ a, _ := New(Config{}, nil)
+ ad := a.(*adapter)
+ b.SetBytes(int64(len(payload)))
+ for b.Loop() {
+ rec := httptest.NewRecorder()
+ ad.guardrail(rec, httptest.NewRequest(http.MethodPost, Path, bytes.NewReader(payload)), p)
+ if rec.Code != http.StatusOK {
+ b.Fatal(rec.Body.String())
+ }
+ }
+ })
+ }
+}
diff --git a/gateway/litellm/litellm.go b/gateway/litellm/litellm.go
index 25d22ce8..3e79ab64 100644
--- a/gateway/litellm/litellm.go
+++ b/gateway/litellm/litellm.go
@@ -66,16 +66,18 @@ func (a *adapter) Serve(ctx context.Context, h gateway.Handler) error {
// request is LiteLLM's guardrail payload; only the fields the adapter reads are declared. The
// conversation fields stay raw JSON: they are spliced into the body the pipeline decodes.
type request struct {
- InputType string `json:"input_type"`
- CallID string `json:"litellm_call_id"`
- TraceID string `json:"litellm_trace_id"`
- Model string `json:"model"`
- StructuredMessages jsontext.Value `json:"structured_messages"`
- Texts jsontext.Value `json:"texts"`
- Tools jsontext.Value `json:"tools"`
- ToolCalls jsontext.Value `json:"tool_calls"`
- RequestHeaders map[string]string `json:"request_headers"`
- RequestData struct {
+ InputType string `json:"input_type"`
+ CallID string `json:"litellm_call_id"`
+ TraceID string `json:"litellm_trace_id"`
+ Model string `json:"model"`
+ StructuredMessages jsontext.Value `json:"structured_messages"`
+ Texts jsontext.Value `json:"texts"`
+ Tools jsontext.Value `json:"tools"`
+ ToolCalls jsontext.Value `json:"tool_calls"`
+ // messages are StructuredMessages' elements, as the walk of the payload split them.
+ messages [][]byte
+ RequestHeaders map[string]string `json:"request_headers"`
+ RequestData struct {
KeyHash string `json:"user_api_key_hash"`
KeyAlias string `json:"user_api_key_alias"`
UserID string `json:"user_api_key_user_id"`
@@ -128,7 +130,7 @@ func (a *adapter) guardrail(w http.ResponseWriter, r *http.Request, h gateway.Ha
// A request built over its payload keeps the payload until the pipeline has read it. Any other
// body (a reply, a texts-only request, one the payload had no room for) is a copy and refers
// to none of the payload, which goes back to the pool at once.
- if len(body) > 0 && len(c.Body) > 0 && &c.Body[0] == &body[0] {
+ if _, ok := sliceOf(body, c.Body); ok {
c.Release, c.BodyBytes = free, int64(cap(body))
} else {
free()
@@ -144,13 +146,16 @@ func (a *adapter) guardrail(w http.ResponseWriter, r *http.Request, h gateway.Ha
// The other members the adapter reads are decoded as encoding/json (v1) decodes them, names
// matched case-insensitively and the last of duplicates winning. Decoding the whole payload
// validated and copied a long session's history, and a second copy of its text (texts), on every
-// call, only for the codec to read the history again.
+// call, only for the codec to read the history again. The walk splits the messages into their
+// elements as it passes them, which the codec takes rather than walking them again.
func decodeRequest(body []byte, req *request) error {
small := append(make([]byte, 0, 1024), '{')
- err := jsonscan.Object(body, func(name string, value []byte) error {
+ isMessages := func(name string) bool { return strings.EqualFold(name, "structured_messages") }
+ split := func(array bool, path ...string) bool { return array && len(path) == 1 && isMessages(path[0]) }
+ err := jsonscan.Split(body, nil, split, func(_, name string, value []byte, elems [][]byte) error {
switch {
- case strings.EqualFold(name, "structured_messages"):
- req.StructuredMessages = value
+ case isMessages(name):
+ req.StructuredMessages, req.messages = value, elems
case strings.EqualFold(name, "texts"):
req.Texts = value
case strings.EqualFold(name, "tools"):
@@ -214,7 +219,7 @@ func (a *adapter) call(req request, payload []byte) (*gateway.Call, error) {
switch req.InputType {
case "request":
c.Phase = gateway.PhaseRequest
- c.Body, err = requestBody(req, payload)
+ c.Body, c.BodyMessages, err = requestBody(req, payload)
case "response":
c.Phase = gateway.PhaseResponse
c.Body, err = responseBody(req)
@@ -223,7 +228,7 @@ func (a *adapter) call(req request, payload []byte) (*gateway.Call, error) {
// the request's messages hands them on only for a reply without a trace id, rather than
// copying the whole conversation into a body nothing reads.
if err == nil && req.TraceID == "" && nullable(req.StructuredMessages) != nil {
- c.RequestBody, err = requestBody(req, nil)
+ c.RequestBody, _, err = requestBody(req, nil)
}
default:
return nil, errors.New("unknown input_type " + req.InputType)
@@ -239,19 +244,20 @@ func (a *adapter) call(req request, payload []byte) (*gateway.Call, error) {
//
// payload, when not nil, is the buffer req's members were read from, which nothing reads once the
// request is built: the request is built over it (inPlace) rather than copied into a second buffer
-// the size of the conversation.
-func requestBody(req request, payload []byte) ([]byte, error) {
+// the size of the conversation, and the messages' elements, which do not move, are returned as
+// the walk split them, for the codec. A copied request returns none.
+func requestBody(req request, payload []byte) (body []byte, messages [][]byte, err error) {
msgs := nullable(req.StructuredMessages)
tools := nullable(req.Tools)
if msgs != nil && payload != nil {
if b, ok := inPlace(payload, req.Model, msgs, tools); ok {
- return b, nil
+ return b, req.messages, nil
}
}
if msgs == nil {
texts, err := decodeTexts(req.Texts)
if err != nil {
- return nil, err
+ return nil, nil, err
}
msgs = append(msgs, '[')
for i, t := range texts {
@@ -277,74 +283,69 @@ func requestBody(req request, payload []byte) ([]byte, error) {
b = append(b, `,"tools":`...)
b = append(b, tools...)
}
- return append(b, '}'), nil
+ return append(b, '}'), nil, nil
}
+// messagesKey opens a request built over its payload, written over the end of the longer name the
+// payload gives the messages ("structured_messages", in any spelling) just before them.
+const messagesKey = `{"messages":`
+
// inPlace builds requestBody's request over payload, the buffer msgs and tools (nil when absent)
-// are slices of: {"messages":…,"tools":…,"model":…}, with messages and tools in the order they
-// appear in payload. Each then starts no later in the result than it did in payload, where the
-// messages are named "structured_messages" and every member is separated from the one before at
-// least as widely, so moving them left in that order never overwrites one not yet moved; the model
-// is written last, from its decoded copy. Every position is checked before anything is written:
-// when one does not hold, or the result would not fit, it reports false with payload untouched.
+// are slices of, without moving the messages, a long session's history:
+// {"messages":…,"tools":…,"model":…}, opened over the end of the messages' name and closed after
+// them, where the tools are moved from wherever they were and the model written from its decoded
+// copy. Every position is checked before anything is written: the opening must not reach the
+// tools, tools after the messages must start no earlier than where they go, and the request must
+// fit the payload. When one does not hold it reports false with payload untouched.
func inPlace(payload []byte, model string, msgs, tools []byte) ([]byte, bool) {
- type member struct {
- name string
- value []byte
- at int
+ ms, ok := sliceOf(payload, msgs)
+ if !ok || ms < len(messagesKey) {
+ return nil, false
}
- var members [2]member
- n := 0
- for _, m := range []member{{`"messages":`, msgs, 0}, {`"tools":`, tools, 0}} {
- if m.value == nil {
- continue
- }
- at := cap(payload) - cap(m.value)
- if at < 0 || at+len(m.value) > len(payload) || &payload[at] != &m.value[0] {
+ start, me := ms-len(messagesKey), ms+len(msgs)
+ end := me
+ var ts int
+ if tools != nil {
+ if ts, ok = sliceOf(payload, tools); !ok {
return nil, false
}
- m.at = at
- members[n] = m
- n++
- }
- ms := members[:n]
- if n == 2 && ms[1].at < ms[0].at {
- ms[0], ms[1] = ms[1], ms[0]
+ if ts < ms && ts+len(tools) > start {
+ return nil, false // the opening would overwrite tools before the messages
+ }
+ if end += len(`,"tools":`); ts > ms && end > ts {
+ return nil, false // tools after the messages would be overwritten before they move
+ }
+ end += len(tools)
}
var quoted []byte
if model != "" {
quoted, _ = jsontext.AppendQuote(nil, model)
- }
- end := 1
- for i, m := range ms {
- if i > 0 {
- end++
- }
- if end += len(m.name); end > m.at {
- return nil, false
- }
- end += len(m.value)
- }
- if quoted != nil {
end += len(`,"model":`) + len(quoted)
}
if end++; end > len(payload) {
return nil, false
}
- w := copy(payload, "{")
- for i, m := range ms {
- if i > 0 {
- w += copy(payload[w:], ",")
- }
- w += copy(payload[w:], m.name)
- w += copy(payload[w:], payload[m.at:m.at+len(m.value)]) // copy moves overlapping bytes correctly
+ copy(payload[start:], messagesKey)
+ w := me
+ if tools != nil {
+ w += copy(payload[w:], `,"tools":`)
+ w += copy(payload[w:], payload[ts:ts+len(tools)]) // copy moves overlapping bytes correctly
}
if quoted != nil {
w += copy(payload[w:], `,"model":`)
w += copy(payload[w:], quoted)
}
w += copy(payload[w:], "}")
- return payload[:w], true
+ return payload[start:w], true
+}
+
+// sliceOf returns where s starts in b when s is a non-empty slice of b's bytes.
+func sliceOf(b, s []byte) (int, bool) {
+ at := cap(b) - cap(s)
+ if len(s) == 0 || at < 0 || at+len(s) > len(b) || &b[at] != &s[0] {
+ return 0, false
+ }
+ return at, true
}
// responseBody reassembles a Chat Completions reply: LiteLLM sends the reply's text and its tool
diff --git a/gateway/litellm/payload_test.go b/gateway/litellm/payload_test.go
index a579f8dc..3b421211 100644
--- a/gateway/litellm/payload_test.go
+++ b/gateway/litellm/payload_test.go
@@ -4,6 +4,7 @@ import (
"bytes"
json "encoding/json/v2"
"reflect"
+ "strings"
"testing"
"github.com/asymptote-labs/agent-beacon/gateway/llm"
@@ -28,6 +29,7 @@ func TestDecodeRequestMatchesFullDecode(t *testing.T) {
if err := json.Unmarshal([]byte(body), &want, decodeOptions); err != nil {
t.Fatalf("%s: full decode: %v", body, err)
}
+ got.messages = nil // the walk's split of the messages, which no decode makes
if !reflect.DeepEqual(got, want) {
t.Errorf("%s:\n got %+v\nwant %+v", body, got, want)
}
@@ -35,59 +37,78 @@ func TestDecodeRequestMatchesFullDecode(t *testing.T) {
}
// A request body built over its payload must decode exactly as the one built in a buffer of its
-// own, whatever order, spelling and spacing the payload gives its members.
+// own, whatever order, spelling and spacing the payload gives its members, and with the walk's
+// split of its messages as without; a payload without room after the messages for tools placed
+// before them is copied instead.
func TestRequestBodyInPlaceDecodesAsCopied(t *testing.T) {
msgs := `[{"role":"system","content":"SYS"},{"role":"user","content":"hello"},{"role":"assistant","content":null,"tool_calls":[{"id":"tu1","type":"function","function":{"name":"Bash","arguments":"{\"command\":\"ls\"}"}}]},{"role":"tool","content":"a\nb","tool_call_id":"tu1"}]`
tools := `[{"type":"function","function":{"name":"Bash","description":"run","parameters":{"type":"object"}}}]`
- for _, payload := range []string{
- `{"input_type":"request","model":"gpt-4o","structured_messages":` + msgs + `,"texts":["SYS","hello"],"tools":` + tools + `}`,
- `{"tools":` + tools + `,"structured_messages":` + msgs + `,"model":"gpt-4o","input_type":"request"}`,
- `{"structured_messages":` + msgs + `,"tools":` + tools + `}`,
- `{"structured_messages":` + msgs + `}`,
- `{"Tools" : ` + tools + ` , "Structured_Messages" :` + msgs + `,"MODEL":"m"}`,
- `{"structured\u005fmessages":` + msgs + `,"tool\u0073":` + tools + `,"model":"caf\u00e9 \"q\""}`,
- `{"structured_messages":[1],"structured_messages":` + msgs + `,"tools":null,"model":"m"}`,
+ long := `"texts":["` + strings.Repeat("x", 200) + `"]`
+ for _, tc := range []struct {
+ payload string
+ inPlace bool
+ }{
+ {`{"input_type":"request","model":"gpt-4o","structured_messages":` + msgs + `,"texts":["SYS","hello"],"tools":` + tools + `}`, true},
+ {`{"tools":` + tools + `,"structured_messages":` + msgs + `,"model":"gpt-4o","input_type":"request"}`, false},
+ {`{"tools":` + tools + `,"structured_messages":` + msgs + `,"model":"gpt-4o",` + long + `}`, true},
+ {`{"structured_messages":` + msgs + `,"tools":` + tools + `}`, true},
+ {`{"structured_messages":` + msgs + `}`, true},
+ {`{"Tools" : ` + tools + ` , "Structured_Messages" :` + msgs + `,"MODEL":"m"}`, false},
+ {`{"structured\u005fmessages":` + msgs + `,"tool\u0073":` + tools + `,"model":"caf\u00e9 \"q\""}`, true},
+ {`{"structured_messages":[1],"structured_messages":` + msgs + `,"tools":null,"model":"m"}`, true},
+ {`{"structured_messages" :` + "\n\t " + msgs + " ,\n" + `"tools":` + tools + "\n}", true},
} {
var req request
- if err := decodeRequest([]byte(payload), &req); err != nil {
- t.Fatalf("%s: %v", payload, err)
+ if err := decodeRequest([]byte(tc.payload), &req); err != nil {
+ t.Fatalf("%s: %v", tc.payload, err)
}
- copied, err := requestBody(req, nil)
+ copied, _, err := requestBody(req, nil)
if err != nil {
t.Fatal(err)
}
- buf := []byte(payload)
+ buf := []byte(tc.payload)
if err := decodeRequest(buf, &req); err != nil {
t.Fatal(err)
}
- built, err := requestBody(req, buf)
+ built, split, err := requestBody(req, buf)
if err != nil {
t.Fatal(err)
}
- if &built[0] != &buf[0] {
- t.Errorf("%s: body was not built in place", payload)
+ if _, in := sliceOf(buf, built); in != tc.inPlace {
+ t.Errorf("%s: built in place %v, want %v", tc.payload, in, tc.inPlace)
+ }
+ if tc.inPlace && len(split) != 4 {
+ t.Errorf("%s: %d messages split, want 4", tc.payload, len(split))
}
want, err := codec.DecodeRequest(llm.OpenAIChat, copied)
if err != nil {
- t.Fatalf("%s: %s: %v", payload, copied, err)
+ t.Fatalf("%s: %s: %v", tc.payload, copied, err)
}
got, err := codec.DecodeRequest(llm.OpenAIChat, built)
if err != nil {
- t.Fatalf("%s: %s: %v", payload, built, err)
+ t.Fatalf("%s: %s: %v", tc.payload, built, err)
}
if !reflect.DeepEqual(got, want) {
- t.Errorf("%s:\n got %s\n %+v\nwant %s\n %+v", payload, built, got, copied, want)
+ t.Errorf("%s:\n got %s\n %+v\nwant %s\n %+v", tc.payload, built, got, copied, want)
+ }
+ plain, perr := codec.SkimRequest(llm.OpenAIChat, built)
+ splitSkim, serr := codec.SkimSplitRequest(llm.OpenAIChat, built, split)
+ if perr != nil || serr != nil || !reflect.DeepEqual(splitSkim, plain) {
+ t.Errorf("%s: skimmed with the walk's split %+v (%v), without %+v (%v)", tc.payload, splitSkim, serr, plain, perr)
}
}
}
-// A member that starts too early to be moved left leaves the payload as it was.
+// Tools before the messages with no room after them for the tools leave the payload as it was.
func TestInPlaceRefusesWithoutRoom(t *testing.T) {
- payload := []byte(`{"x":[1],"tools":[2]}`)
+ payload := []byte(`{"tools":[1,2,3,4,5,6,7,8,9],"structured_messages":[1]}`)
orig := bytes.Clone(payload)
- msgs := payload[5:8] // "messages": needs 11 bytes before it
- if _, ok := inPlace(payload, "m", msgs, payload[17:20]); ok {
- t.Fatal("built a body whose messages start too early")
+ tools, msgs := payload[9:28], payload[51:54]
+ if string(tools) != "[1,2,3,4,5,6,7,8,9]" || string(msgs) != "[1]" {
+ t.Fatalf("test slices %s %s", tools, msgs)
+ }
+ if _, ok := inPlace(payload, "m", msgs, tools); ok {
+ t.Fatal("built a body the payload has no room for")
}
if !bytes.Equal(payload, orig) {
t.Fatalf("payload modified: %s", payload)
diff --git a/gateway/llm/codec/skim.go b/gateway/llm/codec/skim.go
index 55324156..22b7e2b0 100644
--- a/gateway/llm/codec/skim.go
+++ b/gateway/llm/codec/skim.go
@@ -39,6 +39,20 @@ import (
// call that a tool result in those references, so that binding a tail tool result to its call
// finds what a full decode finds. Every other field is as DecodeRequest decodes it.
func SkimRequest(p llm.Protocol, body []byte) (*llm.Request, error) {
+ return skimRequest(p, body, nil)
+}
+
+// SkimSplitRequest is SkimRequest for a body whose message array the caller already split while
+// walking it: elems are the array's elements, in order, as slices of body (nil: none, as
+// SkimRequest). The skim takes them for the array rather than walking it again. They are checked,
+// not trusted: they must be exactly the array's elements, separated by its own syntax, and an
+// element the cache does not hold is validated alone as any element cut from the body is; a split
+// that does not match is ignored.
+func SkimSplitRequest(p llm.Protocol, body []byte, elems [][]byte) (*llm.Request, error) {
+ return skimRequest(p, body, elems)
+}
+
+func skimRequest(p llm.Protocol, body []byte, split [][]byte) (*llm.Request, error) {
c, ok := For(p)
if !ok {
return nil, fmt.Errorf("codec: unknown protocol %q", p)
@@ -49,7 +63,7 @@ func SkimRequest(p llm.Protocol, body []byte) (*llm.Request, error) {
// body and taken from the cache (see planSkim). Anything that goes wrong on that path, an
// invalid body included, is decided by the plain skim of the body itself, which also
// reports the error.
- if plan := planSkim(body, p, seeds); plan != nil {
+ if plan := planSkim(body, p, seeds, split); plan != nil {
in := &skimInput{plan: plan, seeds: seeds}
if req, err := rc.request(plan.shell, in); err == nil && plan.visited(in) {
return req, nil
@@ -270,7 +284,7 @@ func (in *skimInput) messages(dec *jsontext.Decoder, codec skimCodec) ([]llm.Mes
if err != nil {
return eofUnexpected(err)
}
- return sk.add(raw, true)
+ return sk.add(raw, fromDecoder)
})
} else {
// The decoder must be at the placeholder: the shell's top-level member, reached once.
@@ -278,8 +292,12 @@ func (in *skimInput) messages(dec *jsontext.Decoder, codec skimCodec) ([]llm.Mes
if rerr != nil || string(v) != "[]" || dec.InputOffset() != int64(in.plan.messagesEnd) || in.visits != 1 {
return nil, nil, nil, errPlan
}
+ from := fromPlan
+ if in.plan.split {
+ from = fromSplit
+ }
for _, raw := range in.plan.elems {
- if err = sk.add(raw, false); err != nil {
+ if err = sk.add(raw, from); err != nil {
break
}
}
diff --git a/gateway/llm/codec/skim_test.go b/gateway/llm/codec/skim_test.go
index 6a2e7eba..a1220fcd 100644
--- a/gateway/llm/codec/skim_test.go
+++ b/gateway/llm/codec/skim_test.go
@@ -552,7 +552,7 @@ func TestSkimMemoSession(t *testing.T) {
for _, bad := range []synth.Bad{synth.Benign, synth.Injection} {
body := synthRequest(s, p, k, bad)
if k > 2 {
- plan := planSkim(body, p, memoSeeds[p])
+ plan := planSkim(body, p, memoSeeds[p], nil)
if plan == nil || !plan.cutMessages || len(plan.cuts) == 0 {
t.Fatalf("%s turn %d: history or tools did not hit the memo: %+v", p, k, plan)
}
@@ -602,7 +602,7 @@ func TestSkimMemoInvalid(t *testing.T) {
} {
resetSkimMemo()
compareSkim(t, p, []byte(valid))
- if plan := planSkim([]byte(valid), p, memoSeeds[p]); plan == nil || !plan.cutMessages {
+ if plan := planSkim([]byte(valid), p, memoSeeds[p], nil); plan == nil || !plan.cutMessages {
t.Fatalf("%s: the warmed body is not planned", p)
}
body := []byte(skimBody(p, inner))
@@ -720,7 +720,7 @@ func TestSkimMemoResent(t *testing.T) {
// Warm, the tools and system prompt come from the memo, a system message's parts too.
resetSkimMemo()
compareSkim(t, p, prev)
- plan := planSkim(body, p, memoSeeds[p])
+ plan := planSkim(body, p, memoSeeds[p], nil)
if plan == nil || !plan.cutMessages || len(plan.cuts) != len(memoMembers[p]) {
t.Fatalf("%s: plan %+v", p, plan)
}
diff --git a/gateway/llm/codec/skimmemo.go b/gateway/llm/codec/skimmemo.go
index 66c6c123..c5dd35e3 100644
--- a/gateway/llm/codec/skimmemo.go
+++ b/gateway/llm/codec/skimmemo.go
@@ -56,6 +56,14 @@ import (
// - Nesting: in the body, members sit one level deep and elements two. The scan gives up on
// any value nested deeper than maxPlanDepth, far inside jsontext's limit, so validating a
// value alone never accepts what the body's depth would reject.
+// - A split the caller made (SkimSplitRequest) stands in for the boundary scan of the array
+// only where it is exactly that array's elements: each a slice of the body, the first just
+// after the bracket and whitespace, each next one after whitespace, a comma and whitespace,
+// the last followed by whitespace and the closing bracket, which is the array's syntax as the
+// scan checks it. Its elements are then extents like the scan's, with one check the scan
+// made and the caller need not have: on a miss each must be one value nesting no deeper than
+// maxPlanDepth (scanValue), before it is validated. A split that does not match is ignored
+// and the array scanned as usual.
// Whenever any of this fails, or the planned decode reports any error, SkimRequest skims the
// original body plainly, which also reports the error a plain skim reports.
//
@@ -207,9 +215,19 @@ func resetSkimMemo() {
var errInvalidElement = errors.New("skim: invalid message")
-// add skims one array element. validated reports that the decoder already validated it in
-// place; an element cut out by a plan is validated here, on a miss, before anything else.
-func (sk *skimmer) add(raw []byte, validated bool) error {
+// elemSource is how an element of the message array reached the skim.
+type elemSource uint8
+
+const (
+ fromDecoder elemSource = iota // read by the decoder, which validated it in place
+ fromPlan // cut out by planSkim's boundary scan
+ fromSplit // split by the caller (SkimSplitRequest)
+)
+
+// add skims one array element. An element the decoder did not read is validated here, on a miss,
+// before anything else; one the caller split is first checked as the boundary scan checks
+// what it cuts.
+func (sk *skimmer) add(raw []byte, from elemSource) error {
if len(raw) > math.MaxUint32 {
return sk.skim(raw, 0, false)
}
@@ -217,7 +235,12 @@ func (sk *skimmer) add(raw []byte, validated bool) error {
if e, ok := memoGet(key, len(raw)); ok {
return sk.replay(&e, raw)
}
- if !validated && !jsontext.Value(raw).IsValid(decodeOptions) {
+ if from == fromSplit {
+ if end, depth := scanValue(raw, 0); end != len(raw) || depth > maxPlanDepth {
+ return errInvalidElement
+ }
+ }
+ if from != fromDecoder && !jsontext.Value(raw).IsValid(decodeOptions) {
return errInvalidElement
}
return sk.skim(raw, key, true)
@@ -344,8 +367,10 @@ func (sk *skimmer) replay(e *memoEntry, raw []byte) error {
type skimPlan struct {
shell []byte // the body with what was cut replaced by placeholders
// cutMessages reports that the message array was cut: its elements are elems, extents of the
- // body, and its placeholder "[]" ends at messagesEnd in the shell.
+ // body, and its placeholder "[]" ends at messagesEnd in the shell. split reports that elems
+ // are the caller's split rather than the boundary scan's.
cutMessages bool
+ split bool
messagesEnd int
elems [][]byte
cuts []skimCut
@@ -384,8 +409,9 @@ type span struct {
// planSkim cuts body for a memoized skim of protocol p, or returns nil when there is nothing
// worth cutting or the body does not look like an object the scan follows exactly. It never
-// decides validity; see the soundness notes above.
-func planSkim(body []byte, p llm.Protocol, seeds *memoSeedSet) *skimPlan {
+// decides validity; see the soundness notes above. split, when not nil, is the caller's split of
+// the message array, taken for it when it matches (checkSplit).
+func planSkim(body []byte, p llm.Protocol, seeds *memoSeedSet, split [][]byte) *skimPlan {
members, array := memoMembers[p], arrayMember(p)
var (
spans []span
@@ -421,8 +447,9 @@ func planSkim(body []byte, p llm.Protocol, seeds *memoSeedSet) *skimPlan {
arrayI = start
// The array is cut whether or not its messages are cached: the shell then holds
// only the rest of the body, and the elements are validated one by one instead.
- plan.elems, i = scanArray(body, start)
- if i < 0 {
+ if end := checkSplit(body, start, split); end >= 0 {
+ plan.elems, i, plan.split = split, end, true
+ } else if plan.elems, i = scanArray(body, start); i < 0 {
return nil
}
plan.cutMessages = true
@@ -530,6 +557,36 @@ func scanArray(body []byte, start int) (elems [][]byte, end int) {
}
}
+// checkSplit returns the offset just past the array at body[start] when elems are exactly its
+// elements: slices of body, in order, separated by the array's own syntax. It returns -1 when they
+// are not, or there are none.
+func checkSplit(body []byte, start int, elems [][]byte) int {
+ if len(elems) == 0 {
+ return -1
+ }
+ i := skipWS(body, start+1)
+ for k, e := range elems {
+ at := cap(body) - cap(e)
+ if len(e) == 0 || at != i || at+len(e) > len(body) || &body[at] != &e[0] {
+ return -1
+ }
+ if i = skipWS(body, at+len(e)); i >= len(body) {
+ return -1
+ }
+ if k == len(elems)-1 {
+ if body[i] != ']' {
+ return -1
+ }
+ return i + 1
+ }
+ if body[i] != ',' {
+ return -1
+ }
+ i = skipWS(body, i+1)
+ }
+ return -1
+}
+
// errPlan reports a planned skim whose placeholders are not where the plan put them.
var errPlan = errors.New("skim: the request is not where the plan cut it")
diff --git a/gateway/llm/codec/skimsplit_test.go b/gateway/llm/codec/skimsplit_test.go
new file mode 100644
index 00000000..97b9d47a
--- /dev/null
+++ b/gateway/llm/codec/skimsplit_test.go
@@ -0,0 +1,117 @@
+package codec
+
+import (
+ "bytes"
+ "reflect"
+ "strings"
+ "testing"
+
+ "github.com/asymptote-labs/agent-beacon/gateway/internal/jsonscan"
+ "github.com/asymptote-labs/agent-beacon/gateway/llm"
+)
+
+// sameSkim asserts that skimming body with the caller's split decodes exactly as skimming it
+// alone does, or fails as it does.
+func sameSkim(t *testing.T, p llm.Protocol, body []byte, split [][]byte) {
+ t.Helper()
+ want, werr := SkimRequest(p, body)
+ got, gerr := SkimSplitRequest(p, body, split)
+ if (werr == nil) != (gerr == nil) {
+ t.Fatalf("%s: split skim error %v, plain skim error %v\nbody: %s", p, gerr, werr, body)
+ }
+ if !reflect.DeepEqual(got, want) {
+ t.Fatalf("%s: split skim differs:\n got %+v\nwant %+v\nbody: %s", p, got, want, body)
+ }
+}
+
+// split is the adapter's split of body's message array: jsonscan's elements, nil when it finds no
+// array there.
+func split(t *testing.T, p llm.Protocol, body []byte) [][]byte {
+ t.Helper()
+ var elems [][]byte
+ _ = jsonscan.Split(body, nil, func(array bool, path ...string) bool { return array && len(path) == 1 && path[0] == arrayMember(p) }, func(_, name string, _ []byte, e [][]byte) error {
+ if name == arrayMember(p) {
+ elems = e
+ }
+ return nil
+ })
+ return elems
+}
+
+// A body skimmed with the caller's split of its message array decodes as it does alone, and the
+// split stands in for the array's scan; a split that is not exactly that array's elements is
+// ignored.
+func TestSkimSplit(t *testing.T) {
+ for _, p := range skimProtocols {
+ resetSkimMemo()
+ elems := append(append([]string{}, skimElements[p]...), skimTurn[p])
+ body := []byte(skimBody(p, elems...))
+ good := split(t, p, body)
+ start, _ := jsonscan.MemberStart(body, arrayMember(p), nil)
+ if scanned, _ := scanArray(body, start); !reflect.DeepEqual(good, scanned) {
+ t.Fatalf("%s: split %d elements, the boundary scan %d", p, len(good), len(scanned))
+ }
+ for _, warm := range []bool{false, true} { // a miss validates each element; a hit replays it
+ if plan := planSkim(body, p, memoSeeds[p], good); plan == nil || !plan.split {
+ t.Fatalf("%s: the split of the array was not taken (warm %v)", p, warm)
+ }
+ sameSkim(t, p, body, good)
+ }
+ copies := make([][]byte, len(good))
+ for i, e := range good {
+ copies[i] = bytes.Clone(e)
+ }
+ slices := func(f func([][]byte) [][]byte) [][]byte { return f(append([][]byte{}, good...)) }
+ for name, bad := range map[string][][]byte{
+ "copies": copies,
+ "missing": good[:len(good)-1],
+ "skipped": append(append([][]byte{}, good[:1]...), good[2:]...),
+ "reversed": slices(func(s [][]byte) [][]byte { s[0], s[1] = s[1], s[0]; return s }),
+ "shrunk": slices(func(s [][]byte) [][]byte { s[0] = s[0][:len(s[0])-1]; return s }),
+ "grown": slices(func(s [][]byte) [][]byte { s[len(s)-1] = s[len(s)-1][:len(s[len(s)-1])+1]; return s }),
+ "another": split(t, p, []byte(skimBody(p, elems...))),
+ } {
+ if plan := planSkim(body, p, memoSeeds[p], bad); plan == nil || plan.split {
+ t.Fatalf("%s: split %s was taken", p, name)
+ }
+ sameSkim(t, p, body, bad)
+ }
+ // Two elements given as one are separated as the array separates them, so the split is
+ // taken, but the extent is not one value and the skim decodes the body without it.
+ merged := slices(func(s [][]byte) [][]byte { s[0] = s[0][:len(s[0])+1+len(s[1])]; return append(s[:1], s[2:]...) })
+ resetSkimMemo()
+ sameSkim(t, p, body, merged)
+ }
+}
+
+// A body the plain skim rejects is rejected with the adapter's split too, wherever the split finds
+// elements a scan of the array would not, and an element nested beyond the plan's depth is not
+// taken from a split.
+func TestSkimSplitInvalid(t *testing.T) {
+ for _, p := range skimProtocols {
+ elems := strings.Join(skimElements[p], ",")
+ turn := skimTurn[p]
+ bad := skimInvalid[p][0]
+ // A message valid alone (9,999 levels) and too deep in the body (10,001; jsontext allows 10,000).
+ deep := `{"role":"user","content":"x","extra":` + strings.Repeat("[", 9998) + strings.Repeat("]", 9998) + `}`
+ for _, inner := range []string{
+ elems + "," + bad + "," + turn,
+ elems + ",tru," + turn,
+ elems + ",1 2," + turn,
+ elems + `,"a""b",` + turn,
+ elems + `,{"role":"user","content":"x"}},` + turn,
+ elems + `,[{"role":"user"]},` + turn,
+ elems + "," + deep + "," + turn,
+ elems + `,{"role":"user","content":"` + strings.Repeat(`\\`, 3) + `"},` + turn,
+ } {
+ for _, warm := range []bool{false, true} {
+ resetSkimMemo()
+ if warm {
+ SkimRequest(p, []byte(skimBody(p, elems, turn)))
+ }
+ body := []byte(skimBody(p, inner))
+ sameSkim(t, p, body, split(t, p, body))
+ }
+ }
+ }
+}
diff --git a/gateway/pipeline/pipeline.go b/gateway/pipeline/pipeline.go
index 73f76d28..cc3ddef8 100644
--- a/gateway/pipeline/pipeline.go
+++ b/gateway/pipeline/pipeline.go
@@ -309,7 +309,7 @@ func decode(c *gateway.Call) error {
if c.Body == nil {
return errors.New("request call carries no content")
}
- r, err := codec.SkimRequest(c.Protocol, c.Body)
+ r, err := codec.SkimSplitRequest(c.Protocol, c.Body, c.BodyMessages)
if err != nil {
return fmt.Errorf("decode request: %w", err)
}
diff --git a/gateway/portkey/envelope_test.go b/gateway/portkey/envelope_test.go
index f41daf47..b0788d5e 100644
--- a/gateway/portkey/envelope_test.go
+++ b/gateway/portkey/envelope_test.go
@@ -25,6 +25,7 @@ func TestDecodeEnvelopeMatchesFullDecode(t *testing.T) {
if err := json.Unmarshal([]byte(body), &want, decodeOptions); err != nil {
t.Fatalf("%s: full decode: %v", body, err)
}
+ got.requestMessages = nil // the walk's split of the messages, which no decode makes
if !reflect.DeepEqual(got, want) {
t.Errorf("%s:\n got %+v\nwant %+v", body, got, want)
}
diff --git a/gateway/portkey/portkey.go b/gateway/portkey/portkey.go
index c6f99f27..ed434975 100644
--- a/gateway/portkey/portkey.go
+++ b/gateway/portkey/portkey.go
@@ -117,6 +117,8 @@ type envelope struct {
RequestType string `json:"requestType"`
Metadata map[string]jsontext.Value `json:"metadata"`
EventType string `json:"eventType"`
+ // requestMessages are request.json's message array's elements, as the walk split them.
+ requestMessages [][]byte
}
// decodeOptions decode the envelope with encoding/json's (v1) semantics in one pass: v1's decoder
@@ -184,7 +186,9 @@ func (a *adapter) handler(h gateway.Handler) http.Handler {
// are taken as bytes of the payload, neither decoded nor validated here (the codec validates what
// it reads); the other members the adapter reads are decoded as encoding/json (v1) decodes them,
// names matched case-insensitively and the last of duplicates winning. Decoding the whole payload
-// validated and copied the conversation, and the reply, only for the codec to read them again.
+// validated and copied the conversation, and the reply, only for the codec to read them again. The
+// walk splits request.json's messages into their elements as it passes them, which the codec
+// takes rather than walking them again.
func decodeEnvelope(body []byte, env *envelope) error {
raws := [2]*jsontext.Value{&env.Request.JSON, &env.Response.JSON}
phase := func(name string) int {
@@ -205,10 +209,21 @@ func decodeEnvelope(body []byte, env *envelope) error {
}
small := append(make([]byte, 0, 1024), '{')
var phases [2][]byte // the request and response objects' other members
- err := jsonscan.Members(body, func(name string) bool { return phase(name) >= 0 }, func(parent, name string, value []byte) error {
+ enter := func(name string) bool { return phase(name) >= 0 }
+ // request.json is walked through, and its messages array split.
+ split := func(array bool, path ...string) bool {
+ if len(path) < 2 || phase(path[0]) != 0 || !strings.EqualFold(path[1], "json") {
+ return false
+ }
+ return !array && len(path) == 2 || array && len(path) == 3 && path[2] == "messages"
+ }
+ err := jsonscan.Split(body, enter, split, func(parent, name string, value []byte, elems [][]byte) error {
if p := phase(parent); p >= 0 {
if strings.EqualFold(name, "json") {
*raws[p] = value
+ if p == 0 {
+ env.requestMessages = elems
+ }
} else if strings.EqualFold(name, "isStreamingRequest") {
if phases[p] == nil {
phases[p] = []byte{'{'}
@@ -278,7 +293,7 @@ func (a *adapter) call(env *envelope) (*gateway.Call, bool) {
switch env.EventType {
case "beforeRequestHook":
c.Phase = gateway.PhaseRequest
- c.Body = env.Request.JSON
+ c.Body, c.BodyMessages = env.Request.JSON, env.requestMessages
case "afterRequestHook":
if isNull(env.Response.JSON) {
return nil, false
diff --git a/gateway/portkey/portkey_test.go b/gateway/portkey/portkey_test.go
index 977980b3..b84cedb3 100644
--- a/gateway/portkey/portkey_test.go
+++ b/gateway/portkey/portkey_test.go
@@ -6,12 +6,14 @@ import (
"log/slog"
"net/http"
"net/http/httptest"
+ "reflect"
"strings"
"testing"
"github.com/asymptote-labs/agent-beacon/gateway"
"github.com/asymptote-labs/agent-beacon/gateway/internal/gatewaytest"
"github.com/asymptote-labs/agent-beacon/gateway/llm"
+ "github.com/asymptote-labs/agent-beacon/gateway/llm/codec"
)
type fakeHandler struct {
@@ -124,6 +126,31 @@ func TestWebhookCalls(t *testing.T) {
}
}
+// A request-phase call hands on the messages of request.json as the walk of the payload split
+// them, for the codec to take rather than walk again: decoding with them is decoding without.
+func TestRequestCarriesSplitMessages(t *testing.T) {
+ req := `{"model":"gpt-4o","messages":[{"role":"system","content":"s"}, {"role":"user","content":"list files"} ],"stream":false}`
+ h := &fakeHandler{}
+ post(t, Config{}, h, "", payload("beforeRequestHook", "chatComplete", req, `null`, false))
+ if len(h.calls) != 1 {
+ t.Fatalf("%d calls", len(h.calls))
+ }
+ c := h.calls[0]
+ if len(c.BodyMessages) != 2 || string(c.BodyMessages[1]) != `{"role":"user","content":"list files"}` {
+ t.Fatalf("split %q", c.BodyMessages)
+ }
+ for i, e := range c.BodyMessages {
+ if at := cap(c.Body) - cap(e); at < 0 || at+len(e) > len(c.Body) || &c.Body[at] != &e[0] {
+ t.Fatalf("message %d is not a slice of the body", i)
+ }
+ }
+ want, werr := codec.SkimRequest(c.Protocol, c.Body)
+ got, gerr := codec.SkimSplitRequest(c.Protocol, c.Body, c.BodyMessages)
+ if werr != nil || gerr != nil || !reflect.DeepEqual(got, want) {
+ t.Errorf("skimmed with the split %+v (%v), without %+v (%v)", got, gerr, want, werr)
+ }
+}
+
// Through a real pipeline, a decider that blocks a marker denies a request or reply that carries
// it, with its reason and rule in the data Portkey reports to the client.
func TestBlockRenderedThroughPipeline(t *testing.T) {