The cross-harness, self-improving memory layer for AI agents. Join our community: https://discord.com/invite/zdNChS2fBu
Go agent-ai-cliagent-harnessesagent-securitybrowser-extensionciclaude-codecloudcodexcursordetection-engineeringjevknowledge-basememorymobile-device-managementobservabilitysecuritysecurity-information-and-event-managementsecurity-toolstelemetrytraces
perf(gateway): walk callout payloads once, splitting the messages as the adapter passes them (#1271)
kyle-elliott-asymptote538f0395fee3
diff --git a/gateway/gateway.go b/gateway/gateway.goindex 8985e8b8..8689b47c 100644--- a/gateway/gateway.go+++ b/gateway/gateway.go@@ -77,6 +77,11 @@ type Call struct {// it exactly as it named the request's, so the reply joins its request's session without// waiting for the request's analysis or asking the store.RequestBody []byte+ // BodyMessages, when set, are the elements of the message array in a request's Body, in order,+ // as slices of Body, split by an adapter that walked them while reading its own payload. The+ // pipeline decodes Body without walking the array again; the split is checked against Body,+ // not trusted, and one that does not match is ignored (codec.SkimSplitRequest).+ BodyMessages [][]byte// Release, when set, is called by the Handler once, when it will read nothing of Body or// RequestBody again, which may be after Handle returns. An adapter that reads bodies into// pooled buffers returns them here (ReadBody); nothing derived from a body refers to it.diff --git a/gateway/internal/jsonscan/jsonscan.go b/gateway/internal/jsonscan/jsonscan.goindex bf4c4c0d..21a1fb9e 100644--- a/gateway/internal/jsonscan/jsonscan.go+++ b/gateway/internal/jsonscan/jsonscan.go@@ -32,13 +32,75 @@ func Object(b []byte, yield func(name string, value []byte) error) error {// the whole value and walking it again reads the bytes once: a gateway envelope's request object// holds the whole conversation. enter may be nil.func Members(b []byte, enter func(name string) bool, yield func(parent, name string, value []byte) error) error {- end, err := object(b, ws(b, 0), "", enter, yield)+ return Split(b, enter, nil, func(parent, name string, value []byte, _ [][]byte) error { return yield(parent, name, value) })+}++// Split is Members, splitting an array it passes into its elements in the same pass. split is+// asked about each array or object value on the way, by kind and by its path of member names from+// the top (unescaped): an array it accepts is split, into elements that alias b (bracket-matched,+// as Elements finds them); an object it accepts is walked through, still yielded whole, so that an+// array inside it can be split. A member is yielded with the elements of the last array split+// within its value, if any. Finding where an array ends walks each of its elements already, so+// splitting it costs nothing more, and spares a reader of the elements, such as the codec, walking+// the array again. split may be nil.+func Split(b []byte, enter func(name string) bool, split func(array bool, path ...string) bool, yield func(parent, name string, value []byte, elems [][]byte) error) error {+ end, err := object(b, ws(b, 0), func(name string, start int) (int, error) {+ if enter != nil && start < len(b) && b[start] == '{' && enter(name) {+ return object(b, start, func(member string, start int) (int, error) {+ return yieldValue(b, start, split, yield, name, member)+ })+ }+ return yieldValue(b, start, split, yield, "", name)+ })if err != nil {return err}return trailing(b, end)}+// yieldValue finds the end of the value at b[start] of member name (under parent), splitting+// what split asks for, and yields it.+func yieldValue(b []byte, start int, split func(array bool, path ...string) bool, yield func(parent, name string, value []byte, elems [][]byte) error, parent, name string) (int, error) {+ var (+ end int+ elems [][]byte+ err error+ )+ switch {+ case split == nil:+ end, err = skipValue(b, start)+ case parent == "":+ end, elems, err = value(b, start, []string{name}, split)+ default:+ end, elems, err = value(b, start, []string{parent, name}, split)+ }+ if err != nil {+ return 0, err+ }+ return end, yield(parent, name, b[start:end], elems)+}++// value returns the index just past the value at b[i], at path, and the elements of the last+// array within it that split asks to split.+func value(b []byte, i int, path []string, split func(array bool, path ...string) bool) (end int, elems [][]byte, err error) {+ if i >= len(b) || (b[i] != '[' && b[i] != '{') || !split(b[i] == '[', path...) {+ end, err = skipValue(b, i)+ return end, nil, err+ }+ if b[i] == '[' {+ end, err = elements(b, i, func(e []byte) bool { elems = append(elems, e); return true })+ return end, elems, err+ }+ end, err = object(b, i, func(name string, start int) (int, error) {+ end, e, err := value(b, start, append(path[:len(path):len(path)], name), split)+ if e != nil {+ elems = e+ }+ return end, err+ })+ return end, elems, err+}+// errFound stops a walk once FirstString has its member.var errFound = errors.New("found")@@ -111,32 +173,38 @@ func MemberStart(b []byte, name string, before func(name, value []byte)) (int, b// yield returns false. It reads only as far as the element yield stops at; elements are bracket-// matched, not validated.func Elements(b []byte, i int, yield func(elem []byte) bool) error {+ _, err := elements(b, i, yield)+ return err+}++// elements is Elements, returning the index just past the array when yield never stops it.+func elements(b []byte, i int, yield func(elem []byte) bool) (int, error) {if i >= len(b) || b[i] != '[' {- return errSyntax+ return 0, errSyntax}i = ws(b, i+1)if i < len(b) && b[i] == ']' {- return nil+ return i + 1, nil}for {end, err := skipValue(b, i)if err != nil {- return err+ return 0, err}if !yield(b[i:end]) {- return nil+ return end, nil}i = ws(b, end)if i >= len(b) {- return errSyntax+ return 0, errSyntax}switch b[i] {case ',':i = ws(b, i+1)case ']':- return nil+ return i + 1, nildefault:- return errSyntax+ return 0, errSyntax}}}@@ -145,9 +213,10 @@ func Elements(b []byte, i int, yield func(elem []byte) bool) error {// matched brackets, or a literal. It does not validate what is inside.func ValueEnd(b []byte, i int) (int, error) { return skipValue(b, i) }-// object walks the object starting at b[i] == '{', yielding its members under parent, and returns-// the index just past it. Only the top level (parent "") enters members.-func object(b []byte, i int, parent string, enter func(string) bool, yield func(parent, name string, value []byte) error) (int, error) {+// object walks the object starting at b[i] == '{', calling member with each member's name,+// unescaped, and the offset of its value, which member returns the end of; it returns the index+// just past the object.+func object(b []byte, i int, member func(name string, start int) (int, error)) (int, error) {if i >= len(b) || b[i] != '{' {return 0, errSyntax}@@ -171,13 +240,7 @@ func object(b []byte, i int, parent string, enter func(string) bool, yield func(if i >= len(b) || b[i] != ':' {return 0, errSyntax}- start := ws(b, i+1)- if parent == "" && enter != nil && start < len(b) && b[start] == '{' && enter(string(name)) {- end, err = object(b, start, string(name), nil, yield)- } else if end, err = skipValue(b, start); err == nil {- err = yield(parent, string(name), b[start:end])- }- if err != nil {+ if end, err = member(string(name), ws(b, i+1)); err != nil {return 0, err}i = ws(b, end)diff --git a/gateway/internal/jsonscan/jsonscan_test.go b/gateway/internal/jsonscan/jsonscan_test.goindex 101f4926..0070c254 100644--- a/gateway/internal/jsonscan/jsonscan_test.go+++ b/gateway/internal/jsonscan/jsonscan_test.go@@ -207,3 +207,57 @@ func TestMemberStartBefore(t *testing.T) {t.Errorf("MemberStart = %d, %v, before saw %q, want %q", at, ok, got, want)}}++// Split yields every member as Members does, and splits exactly the arrays split asks for, by kind+// and path: walking through an object to an array inside it, with the elements of the last array+// split within a value, and nothing for an array split did not ask for.+func TestSplit(t *testing.T) {+ body := `{"request":{"json":{"model":"m","messages":[{"a":"]"}, "x",[1]],"tools":[2]},"n":1},` ++ `"messages":[3,4],"messages":[5],"json":{"messages":[6]},"other":[7]}`+ split := func(array bool, path ...string) bool {+ switch {+ case !array && len(path) == 2 && path[0] == "request" && path[1] == "json":+ return true+ case array && len(path) == 3 && path[2] == "messages":+ return true+ }+ return array && len(path) == 1 && path[0] == "messages"+ }+ type yielded struct {+ parent, name, value string+ elems []string+ }+ var got []yielded+ err := Split([]byte(body), func(name string) bool { return name == "request" }, split, func(parent, name string, value []byte, elems [][]byte) error {+ y := yielded{parent: parent, name: name, value: string(value)}+ for _, e := range elems {+ y.elems = append(y.elems, string(e))+ }+ got = append(got, y)+ return nil+ })+ want := []yielded{+ {"request", "json", `{"model":"m","messages":[{"a":"]"}, "x",[1]],"tools":[2]}`, []string{`{"a":"]"}`, `"x"`, `[1]`}},+ {"request", "n", `1`, nil},+ {"", "messages", `[3,4]`, []string{`3`, `4`}},+ {"", "messages", `[5]`, []string{`5`}},+ {"", "json", `{"messages":[6]}`, nil}, // an object split did not ask to walk through+ {"", "other", `[7]`, nil},+ }+ if err != nil || !slices.EqualFunc(got, want, func(a, b yielded) bool {+ return a.parent == b.parent && a.name == b.name && a.value == b.value && slices.Equal(a.elems, b.elems)+ }) {+ t.Errorf("Split = %v\n%+v\nwant %+v", err, got, want)+ }+ // An array where split walks through objects is not split, and a value that is not an+ // array or object is skipped as usual.+ err = Split([]byte(`{"request":{"json":[1,2]}}`), func(string) bool { return true }, split, func(_, _ string, _ []byte, elems [][]byte) error {+ if elems != nil {+ t.Errorf("an array split asked to walk through as an object was split: %q", elems)+ }+ return nil+ })+ if err != nil {+ t.Error(err)+ }+}diff --git a/gateway/litellm/bench_test.go b/gateway/litellm/bench_test.gonew file mode 100644index 00000000..2c8a55cf--- /dev/null+++ b/gateway/litellm/bench_test.go@@ -0,0 +1,101 @@+package litellm++import (+ "bytes"+ "context"+ "fmt"+ "net/http"+ "net/http/httptest"+ "testing"++ "github.com/asymptote-labs/agent-beacon/gateway/events"+ "github.com/asymptote-labs/agent-beacon/gateway/internal/synth"+ "github.com/asymptote-labs/agent-beacon/gateway/llm"+ "github.com/asymptote-labs/agent-beacon/gateway/llm/codec"+ "github.com/asymptote-labs/agent-beacon/gateway/pipeline"+)++// benchPayload is a request-phase guardrail payload at turn k of a long session, laid out as+// LiteLLM sends it: the messages, the tools, then texts, a copy of every message's text.+func benchPayload(k int) []byte {+ s := synth.NewScript(1, 0, k, 3000)+ pc := [][]byte{[]byte(`{"input_type":"request","litellm_call_id":"c1","litellm_trace_id":"t1","model":"` + synth.Model + `","structured_messages":[`)}+ pc = s.ChatMessages(pc, k, synth.Benign)+ pc = append(pc, []byte(`],"tools":`), s.ChatTools(), []byte(`,"texts":[`))+ pc = s.Texts(pc, k, synth.Benign)+ pc = append(pc, []byte(`],"request_headers":{"x-claude-code-session-id":"s1","user-agent":"claude-cli/2.1.0"},"request_data":{"user_api_key_user_id":"u"}}`))+ return synth.AppendJoin(nil, pc)+}++type inlineAnalyzer struct{}++func (inlineAnalyzer) Name() string { return "inline" }+func (inlineAnalyzer) Inline() bool { return true }+func (inlineAnalyzer) Analyze(context.Context, *pipeline.Turn) (pipeline.Report, error) {+ return pipeline.Report{}, nil+}++// BenchmarkRequestPhase is a request-phase call's work in this process, by stage, in the steady+// state of a long session (the codec's memo warm with its history): walking the payload, building+// the Chat body over it, skimming that body, and the whole call through a pipeline that analyzes+// before it answers.+func BenchmarkRequestPhase(b *testing.B) {+ for _, k := range []int{100, 300} {+ payload := benchPayload(k)+ name := fmt.Sprintf("turns=%d/%dKB", k, len(payload)>>10)+ buf := make([]byte, len(payload))+ b.Run(name+"/walk", func(b *testing.B) {+ b.SetBytes(int64(len(payload)))+ for b.Loop() {+ var req request+ if err := decodeRequest(payload, &req); err != nil {+ b.Fatal(err)+ }+ }+ })+ b.Run(name+"/build", func(b *testing.B) {+ b.SetBytes(int64(len(payload)))+ for b.Loop() {+ copy(buf, payload)+ var req request+ _ = decodeRequest(buf, &req)+ if _, _, err := requestBody(req, buf); err != nil {+ b.Fatal(err)+ }+ }+ })+ copy(buf, payload)+ var req request+ _ = decodeRequest(buf, &req)+ body, split, _ := requestBody(req, buf)+ if _, err := codec.SkimSplitRequest(llm.OpenAIChat, body, split); err != nil {+ b.Fatal(err)+ }+ b.Run(name+"/skim", func(b *testing.B) {+ b.SetBytes(int64(len(payload)))+ for b.Loop() {+ if _, err := codec.SkimSplitRequest(llm.OpenAIChat, body, split); err != nil {+ b.Fatal(err)+ }+ }+ })+ b.Run(name+"/call", func(b *testing.B) {+ ev, err := events.New(events.Config{})+ if err != nil {+ b.Fatal(err)+ }+ p := pipeline.New(pipeline.Config{Analyzers: []pipeline.Analyzer{ev, inlineAnalyzer{}}})+ defer p.Close()+ a, _ := New(Config{}, nil)+ ad := a.(*adapter)+ b.SetBytes(int64(len(payload)))+ for b.Loop() {+ rec := httptest.NewRecorder()+ ad.guardrail(rec, httptest.NewRequest(http.MethodPost, Path, bytes.NewReader(payload)), p)+ if rec.Code != http.StatusOK {+ b.Fatal(rec.Body.String())+ }+ }+ })+ }+}diff --git a/gateway/litellm/litellm.go b/gateway/litellm/litellm.goindex 25d22ce8..3e79ab64 100644--- a/gateway/litellm/litellm.go+++ b/gateway/litellm/litellm.go@@ -66,16 +66,18 @@ func (a *adapter) Serve(ctx context.Context, h gateway.Handler) error {// request is LiteLLM's guardrail payload; only the fields the adapter reads are declared. The// conversation fields stay raw JSON: they are spliced into the body the pipeline decodes.type request struct {- InputType string `json:"input_type"`- CallID string `json:"litellm_call_id"`- TraceID string `json:"litellm_trace_id"`- Model string `json:"model"`- StructuredMessages jsontext.Value `json:"structured_messages"`- Texts jsontext.Value `json:"texts"`- Tools jsontext.Value `json:"tools"`- ToolCalls jsontext.Value `json:"tool_calls"`- RequestHeaders map[string]string `json:"request_headers"`- RequestData struct {+ InputType string `json:"input_type"`+ CallID string `json:"litellm_call_id"`+ TraceID string `json:"litellm_trace_id"`+ Model string `json:"model"`+ StructuredMessages jsontext.Value `json:"structured_messages"`+ Texts jsontext.Value `json:"texts"`+ Tools jsontext.Value `json:"tools"`+ ToolCalls jsontext.Value `json:"tool_calls"`+ // messages are StructuredMessages' elements, as the walk of the payload split them.+ messages [][]byte+ RequestHeaders map[string]string `json:"request_headers"`+ RequestData struct {KeyHash string `json:"user_api_key_hash"`KeyAlias string `json:"user_api_key_alias"`UserID string `json:"user_api_key_user_id"`@@ -128,7 +130,7 @@ func (a *adapter) guardrail(w http.ResponseWriter, r *http.Request, h gateway.Ha// A request built over its payload keeps the payload until the pipeline has read it. Any other// body (a reply, a texts-only request, one the payload had no room for) is a copy and refers// to none of the payload, which goes back to the pool at once.- if len(body) > 0 && len(c.Body) > 0 && &c.Body[0] == &body[0] {+ if _, ok := sliceOf(body, c.Body); ok {c.Release, c.BodyBytes = free, int64(cap(body))} else {free()@@ -144,13 +146,16 @@ func (a *adapter) guardrail(w http.ResponseWriter, r *http.Request, h gateway.Ha// The other members the adapter reads are decoded as encoding/json (v1) decodes them, names// matched case-insensitively and the last of duplicates winning. Decoding the whole payload// validated and copied a long session's history, and a second copy of its text (texts), on every-// call, only for the codec to read the history again.+// call, only for the codec to read the history again. The walk splits the messages into their+// elements as it passes them, which the codec takes rather than walking them again.func decodeRequest(body []byte, req *request) error {small := append(make([]byte, 0, 1024), '{')- err := jsonscan.Object(body, func(name string, value []byte) error {+ isMessages := func(name string) bool { return strings.EqualFold(name, "structured_messages") }+ split := func(array bool, path ...string) bool { return array && len(path) == 1 && isMessages(path[0]) }+ err := jsonscan.Split(body, nil, split, func(_, name string, value []byte, elems [][]byte) error {switch {- case strings.EqualFold(name, "structured_messages"):- req.StructuredMessages = value+ case isMessages(name):+ req.StructuredMessages, req.messages = value, elemscase strings.EqualFold(name, "texts"):req.Texts = valuecase strings.EqualFold(name, "tools"):@@ -214,7 +219,7 @@ func (a *adapter) call(req request, payload []byte) (*gateway.Call, error) {switch req.InputType {case "request":c.Phase = gateway.PhaseRequest- c.Body, err = requestBody(req, payload)+ c.Body, c.BodyMessages, err = requestBody(req, payload)case "response":c.Phase = gateway.PhaseResponsec.Body, err = responseBody(req)@@ -223,7 +228,7 @@ func (a *adapter) call(req request, payload []byte) (*gateway.Call, error) {// the request's messages hands them on only for a reply without a trace id, rather than// copying the whole conversation into a body nothing reads.if err == nil && req.TraceID == "" && nullable(req.StructuredMessages) != nil {- c.RequestBody, err = requestBody(req, nil)+ c.RequestBody, _, err = requestBody(req, nil)}default:return nil, errors.New("unknown input_type " + req.InputType)@@ -239,19 +244,20 @@ func (a *adapter) call(req request, payload []byte) (*gateway.Call, error) {//// payload, when not nil, is the buffer req's members were read from, which nothing reads once the// request is built: the request is built over it (inPlace) rather than copied into a second buffer-// the size of the conversation.-func requestBody(req request, payload []byte) ([]byte, error) {+// the size of the conversation, and the messages' elements, which do not move, are returned as+// the walk split them, for the codec. A copied request returns none.+func requestBody(req request, payload []byte) (body []byte, messages [][]byte, err error) {msgs := nullable(req.StructuredMessages)tools := nullable(req.Tools)if msgs != nil && payload != nil {if b, ok := inPlace(payload, req.Model, msgs, tools); ok {- return b, nil+ return b, req.messages, nil}}if msgs == nil {texts, err := decodeTexts(req.Texts)if err != nil {- return nil, err+ return nil, nil, err}msgs = append(msgs, '[')for i, t := range texts {@@ -277,74 +283,69 @@ func requestBody(req request, payload []byte) ([]byte, error) {b = append(b, `,"tools":`...)b = append(b, tools...)}- return append(b, '}'), nil+ return append(b, '}'), nil, nil}+// messagesKey opens a request built over its payload, written over the end of the longer name the+// payload gives the messages ("structured_messages", in any spelling) just before them.+const messagesKey = `{"messages":`+// inPlace builds requestBody's request over payload, the buffer msgs and tools (nil when absent)-// are slices of: {"messages":…,"tools":…,"model":…}, with messages and tools in the order they-// appear in payload. Each then starts no later in the result than it did in payload, where the-// messages are named "structured_messages" and every member is separated from the one before at-// least as widely, so moving them left in that order never overwrites one not yet moved; the model-// is written last, from its decoded copy. Every position is checked before anything is written:-// when one does not hold, or the result would not fit, it reports false with payload untouched.+// are slices of, without moving the messages, a long session's history:+// {"messages":…,"tools":…,"model":…}, opened over the end of the messages' name and closed after+// them, where the tools are moved from wherever they were and the model written from its decoded+// copy. Every position is checked before anything is written: the opening must not reach the+// tools, tools after the messages must start no earlier than where they go, and the request must+// fit the payload. When one does not hold it reports false with payload untouched.func inPlace(payload []byte, model string, msgs, tools []byte) ([]byte, bool) {- type member struct {- name string- value []byte- at int+ ms, ok := sliceOf(payload, msgs)+ if !ok || ms < len(messagesKey) {+ return nil, false}- var members [2]member- n := 0- for _, m := range []member{{`"messages":`, msgs, 0}, {`"tools":`, tools, 0}} {- if m.value == nil {- continue- }- at := cap(payload) - cap(m.value)- if at < 0 || at+len(m.value) > len(payload) || &payload[at] != &m.value[0] {+ start, me := ms-len(messagesKey), ms+len(msgs)+ end := me+ var ts int+ if tools != nil {+ if ts, ok = sliceOf(payload, tools); !ok {return nil, false}- m.at = at- members[n] = m- n++- }- ms := members[:n]- if n == 2 && ms[1].at < ms[0].at {- ms[0], ms[1] = ms[1], ms[0]+ if ts < ms && ts+len(tools) > start {+ return nil, false // the opening would overwrite tools before the messages+ }+ if end += len(`,"tools":`); ts > ms && end > ts {+ return nil, false // tools after the messages would be overwritten before they move+ }+ end += len(tools)}var quoted []byteif model != "" {quoted, _ = jsontext.AppendQuote(nil, model)- }- end := 1- for i, m := range ms {- if i > 0 {- end++- }- if end += len(m.name); end > m.at {- return nil, false- }- end += len(m.value)- }- if quoted != nil {end += len(`,"model":`) + len(quoted)}if end++; end > len(payload) {return nil, false}- w := copy(payload, "{")- for i, m := range ms {- if i > 0 {- w += copy(payload[w:], ",")- }- w += copy(payload[w:], m.name)- w += copy(payload[w:], payload[m.at:m.at+len(m.value)]) // copy moves overlapping bytes correctly+ copy(payload[start:], messagesKey)+ w := me+ if tools != nil {+ w += copy(payload[w:], `,"tools":`)+ w += copy(payload[w:], payload[ts:ts+len(tools)]) // copy moves overlapping bytes correctly}if quoted != nil {w += copy(payload[w:], `,"model":`)w += copy(payload[w:], quoted)}w += copy(payload[w:], "}")- return payload[:w], true+ return payload[start:w], true+}++// sliceOf returns where s starts in b when s is a non-empty slice of b's bytes.+func sliceOf(b, s []byte) (int, bool) {+ at := cap(b) - cap(s)+ if len(s) == 0 || at < 0 || at+len(s) > len(b) || &b[at] != &s[0] {+ return 0, false+ }+ return at, true}// responseBody reassembles a Chat Completions reply: LiteLLM sends the reply's text and its tooldiff --git a/gateway/litellm/payload_test.go b/gateway/litellm/payload_test.goindex a579f8dc..3b421211 100644--- a/gateway/litellm/payload_test.go+++ b/gateway/litellm/payload_test.go@@ -4,6 +4,7 @@ import ("bytes"json "encoding/json/v2""reflect"+ "strings""testing""github.com/asymptote-labs/agent-beacon/gateway/llm"@@ -28,6 +29,7 @@ func TestDecodeRequestMatchesFullDecode(t *testing.T) {if err := json.Unmarshal([]byte(body), &want, decodeOptions); err != nil {t.Fatalf("%s: full decode: %v", body, err)}+ got.messages = nil // the walk's split of the messages, which no decode makesif !reflect.DeepEqual(got, want) {t.Errorf("%s:\n got %+v\nwant %+v", body, got, want)}@@ -35,59 +37,78 @@ func TestDecodeRequestMatchesFullDecode(t *testing.T) {}// A request body built over its payload must decode exactly as the one built in a buffer of its-// own, whatever order, spelling and spacing the payload gives its members.+// own, whatever order, spelling and spacing the payload gives its members, and with the walk's+// split of its messages as without; a payload without room after the messages for tools placed+// before them is copied instead.func TestRequestBodyInPlaceDecodesAsCopied(t *testing.T) {msgs := `[{"role":"system","content":"SYS"},{"role":"user","content":"hello"},{"role":"assistant","content":null,"tool_calls":[{"id":"tu1","type":"function","function":{"name":"Bash","arguments":"{\"command\":\"ls\"}"}}]},{"role":"tool","content":"a\nb","tool_call_id":"tu1"}]`tools := `[{"type":"function","function":{"name":"Bash","description":"run","parameters":{"type":"object"}}}]`- for _, payload := range []string{- `{"input_type":"request","model":"gpt-4o","structured_messages":` + msgs + `,"texts":["SYS","hello"],"tools":` + tools + `}`,- `{"tools":` + tools + `,"structured_messages":` + msgs + `,"model":"gpt-4o","input_type":"request"}`,- `{"structured_messages":` + msgs + `,"tools":` + tools + `}`,- `{"structured_messages":` + msgs + `}`,- `{"Tools" : ` + tools + ` , "Structured_Messages" :` + msgs + `,"MODEL":"m"}`,- `{"structured\u005fmessages":` + msgs + `,"tool\u0073":` + tools + `,"model":"caf\u00e9 \"q\""}`,- `{"structured_messages":[1],"structured_messages":` + msgs + `,"tools":null,"model":"m"}`,+ long := `"texts":["` + strings.Repeat("x", 200) + `"]`+ for _, tc := range []struct {+ payload string+ inPlace bool+ }{+ {`{"input_type":"request","model":"gpt-4o","structured_messages":` + msgs + `,"texts":["SYS","hello"],"tools":` + tools + `}`, true},+ {`{"tools":` + tools + `,"structured_messages":` + msgs + `,"model":"gpt-4o","input_type":"request"}`, false},+ {`{"tools":` + tools + `,"structured_messages":` + msgs + `,"model":"gpt-4o",` + long + `}`, true},+ {`{"structured_messages":` + msgs + `,"tools":` + tools + `}`, true},+ {`{"structured_messages":` + msgs + `}`, true},+ {`{"Tools" : ` + tools + ` , "Structured_Messages" :` + msgs + `,"MODEL":"m"}`, false},+ {`{"structured\u005fmessages":` + msgs + `,"tool\u0073":` + tools + `,"model":"caf\u00e9 \"q\""}`, true},+ {`{"structured_messages":[1],"structured_messages":` + msgs + `,"tools":null,"model":"m"}`, true},+ {`{"structured_messages" :` + "\n\t " + msgs + " ,\n" + `"tools":` + tools + "\n}", true},} {var req request- if err := decodeRequest([]byte(payload), &req); err != nil {- t.Fatalf("%s: %v", payload, err)+ if err := decodeRequest([]byte(tc.payload), &req); err != nil {+ t.Fatalf("%s: %v", tc.payload, err)}- copied, err := requestBody(req, nil)+ copied, _, err := requestBody(req, nil)if err != nil {t.Fatal(err)}- buf := []byte(payload)+ buf := []byte(tc.payload)if err := decodeRequest(buf, &req); err != nil {t.Fatal(err)}- built, err := requestBody(req, buf)+ built, split, err := requestBody(req, buf)if err != nil {t.Fatal(err)}- if &built[0] != &buf[0] {- t.Errorf("%s: body was not built in place", payload)+ if _, in := sliceOf(buf, built); in != tc.inPlace {+ t.Errorf("%s: built in place %v, want %v", tc.payload, in, tc.inPlace)+ }+ if tc.inPlace && len(split) != 4 {+ t.Errorf("%s: %d messages split, want 4", tc.payload, len(split))}want, err := codec.DecodeRequest(llm.OpenAIChat, copied)if err != nil {- t.Fatalf("%s: %s: %v", payload, copied, err)+ t.Fatalf("%s: %s: %v", tc.payload, copied, err)}got, err := codec.DecodeRequest(llm.OpenAIChat, built)if err != nil {- t.Fatalf("%s: %s: %v", payload, built, err)+ t.Fatalf("%s: %s: %v", tc.payload, built, err)}if !reflect.DeepEqual(got, want) {- t.Errorf("%s:\n got %s\n %+v\nwant %s\n %+v", payload, built, got, copied, want)+ t.Errorf("%s:\n got %s\n %+v\nwant %s\n %+v", tc.payload, built, got, copied, want)+ }+ plain, perr := codec.SkimRequest(llm.OpenAIChat, built)+ splitSkim, serr := codec.SkimSplitRequest(llm.OpenAIChat, built, split)+ if perr != nil || serr != nil || !reflect.DeepEqual(splitSkim, plain) {+ t.Errorf("%s: skimmed with the walk's split %+v (%v), without %+v (%v)", tc.payload, splitSkim, serr, plain, perr)}}}-// A member that starts too early to be moved left leaves the payload as it was.+// Tools before the messages with no room after them for the tools leave the payload as it was.func TestInPlaceRefusesWithoutRoom(t *testing.T) {- payload := []byte(`{"x":[1],"tools":[2]}`)+ payload := []byte(`{"tools":[1,2,3,4,5,6,7,8,9],"structured_messages":[1]}`)orig := bytes.Clone(payload)- msgs := payload[5:8] // "messages": needs 11 bytes before it- if _, ok := inPlace(payload, "m", msgs, payload[17:20]); ok {- t.Fatal("built a body whose messages start too early")+ tools, msgs := payload[9:28], payload[51:54]+ if string(tools) != "[1,2,3,4,5,6,7,8,9]" || string(msgs) != "[1]" {+ t.Fatalf("test slices %s %s", tools, msgs)+ }+ if _, ok := inPlace(payload, "m", msgs, tools); ok {+ t.Fatal("built a body the payload has no room for")}if !bytes.Equal(payload, orig) {t.Fatalf("payload modified: %s", payload)diff --git a/gateway/llm/codec/skim.go b/gateway/llm/codec/skim.goindex 55324156..22b7e2b0 100644--- a/gateway/llm/codec/skim.go+++ b/gateway/llm/codec/skim.go@@ -39,6 +39,20 @@ import (// call that a tool result in those references, so that binding a tail tool result to its call// finds what a full decode finds. Every other field is as DecodeRequest decodes it.func SkimRequest(p llm.Protocol, body []byte) (*llm.Request, error) {+ return skimRequest(p, body, nil)+}++// SkimSplitRequest is SkimRequest for a body whose message array the caller already split while+// walking it: elems are the array's elements, in order, as slices of body (nil: none, as+// SkimRequest). The skim takes them for the array rather than walking it again. They are checked,+// not trusted: they must be exactly the array's elements, separated by its own syntax, and an+// element the cache does not hold is validated alone as any element cut from the body is; a split+// that does not match is ignored.+func SkimSplitRequest(p llm.Protocol, body []byte, elems [][]byte) (*llm.Request, error) {+ return skimRequest(p, body, elems)+}++func skimRequest(p llm.Protocol, body []byte, split [][]byte) (*llm.Request, error) {c, ok := For(p)if !ok {return nil, fmt.Errorf("codec: unknown protocol %q", p)@@ -49,7 +63,7 @@ func SkimRequest(p llm.Protocol, body []byte) (*llm.Request, error) {// body and taken from the cache (see planSkim). Anything that goes wrong on that path, an// invalid body included, is decided by the plain skim of the body itself, which also// reports the error.- if plan := planSkim(body, p, seeds); plan != nil {+ if plan := planSkim(body, p, seeds, split); plan != nil {in := &skimInput{plan: plan, seeds: seeds}if req, err := rc.request(plan.shell, in); err == nil && plan.visited(in) {return req, nil@@ -270,7 +284,7 @@ func (in *skimInput) messages(dec *jsontext.Decoder, codec skimCodec) ([]llm.Mesif err != nil {return eofUnexpected(err)}- return sk.add(raw, true)+ return sk.add(raw, fromDecoder)})} else {// The decoder must be at the placeholder: the shell's top-level member, reached once.@@ -278,8 +292,12 @@ func (in *skimInput) messages(dec *jsontext.Decoder, codec skimCodec) ([]llm.Mesif rerr != nil || string(v) != "[]" || dec.InputOffset() != int64(in.plan.messagesEnd) || in.visits != 1 {return nil, nil, nil, errPlan}+ from := fromPlan+ if in.plan.split {+ from = fromSplit+ }for _, raw := range in.plan.elems {- if err = sk.add(raw, false); err != nil {+ if err = sk.add(raw, from); err != nil {break}}diff --git a/gateway/llm/codec/skim_test.go b/gateway/llm/codec/skim_test.goindex 6a2e7eba..a1220fcd 100644--- a/gateway/llm/codec/skim_test.go+++ b/gateway/llm/codec/skim_test.go@@ -552,7 +552,7 @@ func TestSkimMemoSession(t *testing.T) {for _, bad := range []synth.Bad{synth.Benign, synth.Injection} {body := synthRequest(s, p, k, bad)if k > 2 {- plan := planSkim(body, p, memoSeeds[p])+ plan := planSkim(body, p, memoSeeds[p], nil)if plan == nil || !plan.cutMessages || len(plan.cuts) == 0 {t.Fatalf("%s turn %d: history or tools did not hit the memo: %+v", p, k, plan)}@@ -602,7 +602,7 @@ func TestSkimMemoInvalid(t *testing.T) {} {resetSkimMemo()compareSkim(t, p, []byte(valid))- if plan := planSkim([]byte(valid), p, memoSeeds[p]); plan == nil || !plan.cutMessages {+ if plan := planSkim([]byte(valid), p, memoSeeds[p], nil); plan == nil || !plan.cutMessages {t.Fatalf("%s: the warmed body is not planned", p)}body := []byte(skimBody(p, inner))@@ -720,7 +720,7 @@ func TestSkimMemoResent(t *testing.T) {// Warm, the tools and system prompt come from the memo, a system message's parts too.resetSkimMemo()compareSkim(t, p, prev)- plan := planSkim(body, p, memoSeeds[p])+ plan := planSkim(body, p, memoSeeds[p], nil)if plan == nil || !plan.cutMessages || len(plan.cuts) != len(memoMembers[p]) {t.Fatalf("%s: plan %+v", p, plan)}diff --git a/gateway/llm/codec/skimmemo.go b/gateway/llm/codec/skimmemo.goindex 66c6c123..c5dd35e3 100644--- a/gateway/llm/codec/skimmemo.go+++ b/gateway/llm/codec/skimmemo.go@@ -56,6 +56,14 @@ import (// - Nesting: in the body, members sit one level deep and elements two. The scan gives up on// any value nested deeper than maxPlanDepth, far inside jsontext's limit, so validating a// value alone never accepts what the body's depth would reject.+// - A split the caller made (SkimSplitRequest) stands in for the boundary scan of the array+// only where it is exactly that array's elements: each a slice of the body, the first just+// after the bracket and whitespace, each next one after whitespace, a comma and whitespace,+// the last followed by whitespace and the closing bracket, which is the array's syntax as the+// scan checks it. Its elements are then extents like the scan's, with one check the scan+// made and the caller need not have: on a miss each must be one value nesting no deeper than+// maxPlanDepth (scanValue), before it is validated. A split that does not match is ignored+// and the array scanned as usual.// Whenever any of this fails, or the planned decode reports any error, SkimRequest skims the// original body plainly, which also reports the error a plain skim reports.//@@ -207,9 +215,19 @@ func resetSkimMemo() {var errInvalidElement = errors.New("skim: invalid message")-// add skims one array element. validated reports that the decoder already validated it in-// place; an element cut out by a plan is validated here, on a miss, before anything else.-func (sk *skimmer) add(raw []byte, validated bool) error {+// elemSource is how an element of the message array reached the skim.+type elemSource uint8++const (+ fromDecoder elemSource = iota // read by the decoder, which validated it in place+ fromPlan // cut out by planSkim's boundary scan+ fromSplit // split by the caller (SkimSplitRequest)+)++// add skims one array element. An element the decoder did not read is validated here, on a miss,+// before anything else; one the caller split is first checked as the boundary scan checks+// what it cuts.+func (sk *skimmer) add(raw []byte, from elemSource) error {if len(raw) > math.MaxUint32 {return sk.skim(raw, 0, false)}@@ -217,7 +235,12 @@ func (sk *skimmer) add(raw []byte, validated bool) error {if e, ok := memoGet(key, len(raw)); ok {return sk.replay(&e, raw)}- if !validated && !jsontext.Value(raw).IsValid(decodeOptions) {+ if from == fromSplit {+ if end, depth := scanValue(raw, 0); end != len(raw) || depth > maxPlanDepth {+ return errInvalidElement+ }+ }+ if from != fromDecoder && !jsontext.Value(raw).IsValid(decodeOptions) {return errInvalidElement}return sk.skim(raw, key, true)@@ -344,8 +367,10 @@ func (sk *skimmer) replay(e *memoEntry, raw []byte) error {type skimPlan struct {shell []byte // the body with what was cut replaced by placeholders// cutMessages reports that the message array was cut: its elements are elems, extents of the- // body, and its placeholder "[]" ends at messagesEnd in the shell.+ // body, and its placeholder "[]" ends at messagesEnd in the shell. split reports that elems+ // are the caller's split rather than the boundary scan's.cutMessages bool+ split boolmessagesEnd intelems [][]bytecuts []skimCut@@ -384,8 +409,9 @@ type span struct {// planSkim cuts body for a memoized skim of protocol p, or returns nil when there is nothing// worth cutting or the body does not look like an object the scan follows exactly. It never-// decides validity; see the soundness notes above.-func planSkim(body []byte, p llm.Protocol, seeds *memoSeedSet) *skimPlan {+// decides validity; see the soundness notes above. split, when not nil, is the caller's split of+// the message array, taken for it when it matches (checkSplit).+func planSkim(body []byte, p llm.Protocol, seeds *memoSeedSet, split [][]byte) *skimPlan {members, array := memoMembers[p], arrayMember(p)var (spans []span@@ -421,8 +447,9 @@ func planSkim(body []byte, p llm.Protocol, seeds *memoSeedSet) *skimPlan {arrayI = start// The array is cut whether or not its messages are cached: the shell then holds// only the rest of the body, and the elements are validated one by one instead.- plan.elems, i = scanArray(body, start)- if i < 0 {+ if end := checkSplit(body, start, split); end >= 0 {+ plan.elems, i, plan.split = split, end, true+ } else if plan.elems, i = scanArray(body, start); i < 0 {return nil}plan.cutMessages = true@@ -530,6 +557,36 @@ func scanArray(body []byte, start int) (elems [][]byte, end int) {}}+// checkSplit returns the offset just past the array at body[start] when elems are exactly its+// elements: slices of body, in order, separated by the array's own syntax. It returns -1 when they+// are not, or there are none.+func checkSplit(body []byte, start int, elems [][]byte) int {+ if len(elems) == 0 {+ return -1+ }+ i := skipWS(body, start+1)+ for k, e := range elems {+ at := cap(body) - cap(e)+ if len(e) == 0 || at != i || at+len(e) > len(body) || &body[at] != &e[0] {+ return -1+ }+ if i = skipWS(body, at+len(e)); i >= len(body) {+ return -1+ }+ if k == len(elems)-1 {+ if body[i] != ']' {+ return -1+ }+ return i + 1+ }+ if body[i] != ',' {+ return -1+ }+ i = skipWS(body, i+1)+ }+ return -1+}+// errPlan reports a planned skim whose placeholders are not where the plan put them.var errPlan = errors.New("skim: the request is not where the plan cut it")diff --git a/gateway/llm/codec/skimsplit_test.go b/gateway/llm/codec/skimsplit_test.gonew file mode 100644index 00000000..97b9d47a--- /dev/null+++ b/gateway/llm/codec/skimsplit_test.go@@ -0,0 +1,117 @@+package codec++import (+ "bytes"+ "reflect"+ "strings"+ "testing"++ "github.com/asymptote-labs/agent-beacon/gateway/internal/jsonscan"+ "github.com/asymptote-labs/agent-beacon/gateway/llm"+)++// sameSkim asserts that skimming body with the caller's split decodes exactly as skimming it+// alone does, or fails as it does.+func sameSkim(t *testing.T, p llm.Protocol, body []byte, split [][]byte) {+ t.Helper()+ want, werr := SkimRequest(p, body)+ got, gerr := SkimSplitRequest(p, body, split)+ if (werr == nil) != (gerr == nil) {+ t.Fatalf("%s: split skim error %v, plain skim error %v\nbody: %s", p, gerr, werr, body)+ }+ if !reflect.DeepEqual(got, want) {+ t.Fatalf("%s: split skim differs:\n got %+v\nwant %+v\nbody: %s", p, got, want, body)+ }+}++// split is the adapter's split of body's message array: jsonscan's elements, nil when it finds no+// array there.+func split(t *testing.T, p llm.Protocol, body []byte) [][]byte {+ t.Helper()+ var elems [][]byte+ _ = jsonscan.Split(body, nil, func(array bool, path ...string) bool { return array && len(path) == 1 && path[0] == arrayMember(p) }, func(_, name string, _ []byte, e [][]byte) error {+ if name == arrayMember(p) {+ elems = e+ }+ return nil+ })+ return elems+}++// A body skimmed with the caller's split of its message array decodes as it does alone, and the+// split stands in for the array's scan; a split that is not exactly that array's elements is+// ignored.+func TestSkimSplit(t *testing.T) {+ for _, p := range skimProtocols {+ resetSkimMemo()+ elems := append(append([]string{}, skimElements[p]...), skimTurn[p])+ body := []byte(skimBody(p, elems...))+ good := split(t, p, body)+ start, _ := jsonscan.MemberStart(body, arrayMember(p), nil)+ if scanned, _ := scanArray(body, start); !reflect.DeepEqual(good, scanned) {+ t.Fatalf("%s: split %d elements, the boundary scan %d", p, len(good), len(scanned))+ }+ for _, warm := range []bool{false, true} { // a miss validates each element; a hit replays it+ if plan := planSkim(body, p, memoSeeds[p], good); plan == nil || !plan.split {+ t.Fatalf("%s: the split of the array was not taken (warm %v)", p, warm)+ }+ sameSkim(t, p, body, good)+ }+ copies := make([][]byte, len(good))+ for i, e := range good {+ copies[i] = bytes.Clone(e)+ }+ slices := func(f func([][]byte) [][]byte) [][]byte { return f(append([][]byte{}, good...)) }+ for name, bad := range map[string][][]byte{+ "copies": copies,+ "missing": good[:len(good)-1],+ "skipped": append(append([][]byte{}, good[:1]...), good[2:]...),+ "reversed": slices(func(s [][]byte) [][]byte { s[0], s[1] = s[1], s[0]; return s }),+ "shrunk": slices(func(s [][]byte) [][]byte { s[0] = s[0][:len(s[0])-1]; return s }),+ "grown": slices(func(s [][]byte) [][]byte { s[len(s)-1] = s[len(s)-1][:len(s[len(s)-1])+1]; return s }),+ "another": split(t, p, []byte(skimBody(p, elems...))),+ } {+ if plan := planSkim(body, p, memoSeeds[p], bad); plan == nil || plan.split {+ t.Fatalf("%s: split %s was taken", p, name)+ }+ sameSkim(t, p, body, bad)+ }+ // Two elements given as one are separated as the array separates them, so the split is+ // taken, but the extent is not one value and the skim decodes the body without it.+ merged := slices(func(s [][]byte) [][]byte { s[0] = s[0][:len(s[0])+1+len(s[1])]; return append(s[:1], s[2:]...) })+ resetSkimMemo()+ sameSkim(t, p, body, merged)+ }+}++// A body the plain skim rejects is rejected with the adapter's split too, wherever the split finds+// elements a scan of the array would not, and an element nested beyond the plan's depth is not+// taken from a split.+func TestSkimSplitInvalid(t *testing.T) {+ for _, p := range skimProtocols {+ elems := strings.Join(skimElements[p], ",")+ turn := skimTurn[p]+ bad := skimInvalid[p][0]+ // A message valid alone (9,999 levels) and too deep in the body (10,001; jsontext allows 10,000).+ deep := `{"role":"user","content":"x","extra":` + strings.Repeat("[", 9998) + strings.Repeat("]", 9998) + `}`+ for _, inner := range []string{+ elems + "," + bad + "," + turn,+ elems + ",tru," + turn,+ elems + ",1 2," + turn,+ elems + `,"a""b",` + turn,+ elems + `,{"role":"user","content":"x"}},` + turn,+ elems + `,[{"role":"user"]},` + turn,+ elems + "," + deep + "," + turn,+ elems + `,{"role":"user","content":"` + strings.Repeat(`\\`, 3) + `"},` + turn,+ } {+ for _, warm := range []bool{false, true} {+ resetSkimMemo()+ if warm {+ SkimRequest(p, []byte(skimBody(p, elems, turn)))+ }+ body := []byte(skimBody(p, inner))+ sameSkim(t, p, body, split(t, p, body))+ }+ }+ }+}diff --git a/gateway/pipeline/pipeline.go b/gateway/pipeline/pipeline.goindex 73f76d28..cc3ddef8 100644--- a/gateway/pipeline/pipeline.go+++ b/gateway/pipeline/pipeline.go@@ -309,7 +309,7 @@ func decode(c *gateway.Call) error {if c.Body == nil {return errors.New("request call carries no content")}- r, err := codec.SkimRequest(c.Protocol, c.Body)+ r, err := codec.SkimSplitRequest(c.Protocol, c.Body, c.BodyMessages)if err != nil {return fmt.Errorf("decode request: %w", err)}diff --git a/gateway/portkey/envelope_test.go b/gateway/portkey/envelope_test.goindex f41daf47..b0788d5e 100644--- a/gateway/portkey/envelope_test.go+++ b/gateway/portkey/envelope_test.go@@ -25,6 +25,7 @@ func TestDecodeEnvelopeMatchesFullDecode(t *testing.T) {if err := json.Unmarshal([]byte(body), &want, decodeOptions); err != nil {t.Fatalf("%s: full decode: %v", body, err)}+ got.requestMessages = nil // the walk's split of the messages, which no decode makesif !reflect.DeepEqual(got, want) {t.Errorf("%s:\n got %+v\nwant %+v", body, got, want)}diff --git a/gateway/portkey/portkey.go b/gateway/portkey/portkey.goindex c6f99f27..ed434975 100644--- a/gateway/portkey/portkey.go+++ b/gateway/portkey/portkey.go@@ -117,6 +117,8 @@ type envelope struct {RequestType string `json:"requestType"`Metadata map[string]jsontext.Value `json:"metadata"`EventType string `json:"eventType"`+ // requestMessages are request.json's message array's elements, as the walk split them.+ requestMessages [][]byte}// decodeOptions decode the envelope with encoding/json's (v1) semantics in one pass: v1's decoder@@ -184,7 +186,9 @@ func (a *adapter) handler(h gateway.Handler) http.Handler {// are taken as bytes of the payload, neither decoded nor validated here (the codec validates what// it reads); the other members the adapter reads are decoded as encoding/json (v1) decodes them,// names matched case-insensitively and the last of duplicates winning. Decoding the whole payload-// validated and copied the conversation, and the reply, only for the codec to read them again.+// validated and copied the conversation, and the reply, only for the codec to read them again. The+// walk splits request.json's messages into their elements as it passes them, which the codec+// takes rather than walking them again.func decodeEnvelope(body []byte, env *envelope) error {raws := [2]*jsontext.Value{&env.Request.JSON, &env.Response.JSON}phase := func(name string) int {@@ -205,10 +209,21 @@ func decodeEnvelope(body []byte, env *envelope) error {}small := append(make([]byte, 0, 1024), '{')var phases [2][]byte // the request and response objects' other members- err := jsonscan.Members(body, func(name string) bool { return phase(name) >= 0 }, func(parent, name string, value []byte) error {+ enter := func(name string) bool { return phase(name) >= 0 }+ // request.json is walked through, and its messages array split.+ split := func(array bool, path ...string) bool {+ if len(path) < 2 || phase(path[0]) != 0 || !strings.EqualFold(path[1], "json") {+ return false+ }+ return !array && len(path) == 2 || array && len(path) == 3 && path[2] == "messages"+ }+ err := jsonscan.Split(body, enter, split, func(parent, name string, value []byte, elems [][]byte) error {if p := phase(parent); p >= 0 {if strings.EqualFold(name, "json") {*raws[p] = value+ if p == 0 {+ env.requestMessages = elems+ }} else if strings.EqualFold(name, "isStreamingRequest") {if phases[p] == nil {phases[p] = []byte{'{'}@@ -278,7 +293,7 @@ func (a *adapter) call(env *envelope) (*gateway.Call, bool) {switch env.EventType {case "beforeRequestHook":c.Phase = gateway.PhaseRequest- c.Body = env.Request.JSON+ c.Body, c.BodyMessages = env.Request.JSON, env.requestMessagescase "afterRequestHook":if isNull(env.Response.JSON) {return nil, falsediff --git a/gateway/portkey/portkey_test.go b/gateway/portkey/portkey_test.goindex 977980b3..b84cedb3 100644--- a/gateway/portkey/portkey_test.go+++ b/gateway/portkey/portkey_test.go@@ -6,12 +6,14 @@ import ("log/slog""net/http""net/http/httptest"+ "reflect""strings""testing""github.com/asymptote-labs/agent-beacon/gateway""github.com/asymptote-labs/agent-beacon/gateway/internal/gatewaytest""github.com/asymptote-labs/agent-beacon/gateway/llm"+ "github.com/asymptote-labs/agent-beacon/gateway/llm/codec")type fakeHandler struct {@@ -124,6 +126,31 @@ func TestWebhookCalls(t *testing.T) {}}+// A request-phase call hands on the messages of request.json as the walk of the payload split+// them, for the codec to take rather than walk again: decoding with them is decoding without.+func TestRequestCarriesSplitMessages(t *testing.T) {+ req := `{"model":"gpt-4o","messages":[{"role":"system","content":"s"}, {"role":"user","content":"list files"} ],"stream":false}`+ h := &fakeHandler{}+ post(t, Config{}, h, "", payload("beforeRequestHook", "chatComplete", req, `null`, false))+ if len(h.calls) != 1 {+ t.Fatalf("%d calls", len(h.calls))+ }+ c := h.calls[0]+ if len(c.BodyMessages) != 2 || string(c.BodyMessages[1]) != `{"role":"user","content":"list files"}` {+ t.Fatalf("split %q", c.BodyMessages)+ }+ for i, e := range c.BodyMessages {+ if at := cap(c.Body) - cap(e); at < 0 || at+len(e) > len(c.Body) || &c.Body[at] != &e[0] {+ t.Fatalf("message %d is not a slice of the body", i)+ }+ }+ want, werr := codec.SkimRequest(c.Protocol, c.Body)+ got, gerr := codec.SkimSplitRequest(c.Protocol, c.Body, c.BodyMessages)+ if werr != nil || gerr != nil || !reflect.DeepEqual(got, want) {+ t.Errorf("skimmed with the split %+v (%v), without %+v (%v)", got, gerr, want, werr)+ }+}+// Through a real pipeline, a decider that blocks a marker denies a request or reply that carries// it, with its reason and rule in the data Portkey reports to the client.func TestBlockRenderedThroughPipeline(t *testing.T) {