Skip to content

The cross-harness, self-improving memory layer for AI agents. Join our community: https://discord.com/invite/zdNChS2fBu

fix(localingest): reject malformed trailing JSON delimiters (#1124)

Rudy Mizrahi Celekli199955e4c55d
diff --git a/cli/beacon/internal/pricing/overrides.go b/cli/beacon/internal/pricing/overrides.go
index 53a54c04..3683d19a 100644
--- a/cli/beacon/internal/pricing/overrides.go
+++ b/cli/beacon/internal/pricing/overrides.go
@@ -16,6 +16,7 @@ import (
"strings"
endpointconfig "github.com/asymptote-labs/agent-beacon/cli/beacon/internal/endpoint/config"
+ "github.com/asymptote-labs/agent-beacon/pkg/asymptoteobserve"
)
// OverridesSchema is the schema string an overrides file must declare. A layout change gets a
@@ -216,14 +217,9 @@ func LoadOverrides(data []byte, catalog *Catalog) (*Overrides, error) {
// unpublished and price its tokens by a fallback the author never asked for.
dec.DisallowUnknownFields()
var file overridesJSON
- if err := dec.Decode(&file); err != nil {
+ if err := asymptoteobserve.DecodeOnly(dec, &file); err != nil {
return nil, fmt.Errorf("parse: %w", err)
}
- // More() reports false before a stray closing bracket or brace, so the end of input is
- // required explicitly: anything after the document, even a lone "]", is refused.
- if _, err := dec.Token(); err != io.EOF {
- return nil, fmt.Errorf("parse: trailing data after the document")
- }
if file.Schema != OverridesSchema {
return nil, fmt.Errorf("schema %q, want %q", file.Schema, OverridesSchema)
}
diff --git a/pkg/asymptoteobserve/jsondecode.go b/pkg/asymptoteobserve/jsondecode.go
new file mode 100644
index 00000000..2ca64cca
--- /dev/null
+++ b/pkg/asymptoteobserve/jsondecode.go
@@ -0,0 +1,22 @@
+package asymptoteobserve
+
+import (
+ "encoding/json"
+ "errors"
+ "io"
+)
+
+// DecodeOnly decodes one value from dec into v and requires the input to end after it, apart
+// from whitespace. Configure dec (UseNumber, DisallowUnknownFields) before calling.
+//
+// Decoder.More is not that check: it reports false in front of a stray "]" or "}", so a value
+// followed by one, or by "] {...}", would pass.
+func DecodeOnly(dec *json.Decoder, v any) error {
+ if err := dec.Decode(v); err != nil {
+ return err
+ }
+ if _, err := dec.Token(); err != io.EOF {
+ return errors.New("trailing data after the JSON value")
+ }
+ return nil
+}
diff --git a/pkg/asymptoteobserve/lens.go b/pkg/asymptoteobserve/lens.go
index 49aef201..8fb84ea9 100644
--- a/pkg/asymptoteobserve/lens.go
+++ b/pkg/asymptoteobserve/lens.go
@@ -212,11 +212,8 @@ func ParseLensManifest(html []byte) (LensManifestV1, error) {
decoder := json.NewDecoder(bytes.NewReader(body[:end]))
decoder.DisallowUnknownFields()
var manifest LensManifestV1
- if err := decoder.Decode(&manifest); err != nil {
+ if err := DecodeOnly(decoder, &manifest); err != nil {
return LensManifestV1{}, fmt.Errorf("lens manifest: %w", err)
}
- if decoder.More() {
- return LensManifestV1{}, errors.New("lens manifest holds more than one JSON value")
- }
return manifest, nil
}
diff --git a/pkg/asymptoteobserve/lens_test.go b/pkg/asymptoteobserve/lens_test.go
index 8c253299..ffb91da3 100644
--- a/pkg/asymptoteobserve/lens_test.go
+++ b/pkg/asymptoteobserve/lens_test.go
@@ -103,7 +103,10 @@ func TestParseLensManifest(t *testing.T) {
{"duplicate", open + good + "</script>" + open + good + "</script>", "more than one manifest"},
{"unclosed", open + good, "not closed"},
{"unknown field", open + `{"id":"x","title":"X","version":1,"api":"beacon.lens.v1","permissions":["net"]}` + "</script>", "unknown field"},
- {"trailing value", open + good + good + "</script>", "more than one JSON value"},
+ {"trailing value", open + good + good + "</script>", "trailing data"},
+ // Decoder.More reports false in front of a stray closer, so these need the end-of-input check.
+ {"trailing brace", open + good + "}" + "</script>", "trailing data"},
+ {"trailing bracket then value", open + good + `] {"id":"y"}` + "</script>", "trailing data"},
{"not json", open + "id: x" + "</script>", "lens manifest"},
}
for _, tc := range cases {
diff --git a/pkg/asymptoteobserve/localingest/localingest.go b/pkg/asymptoteobserve/localingest/localingest.go
index 1926c572..2738b30d 100644
--- a/pkg/asymptoteobserve/localingest/localingest.go
+++ b/pkg/asymptoteobserve/localingest/localingest.go
@@ -238,19 +238,12 @@ func decodeObject(line []byte) (map[string]interface{}, error) {
dec := json.NewDecoder(bytes.NewReader(line))
dec.UseNumber()
var event map[string]interface{}
- if err := dec.Decode(&event); err != nil {
+ if err := asymptoteobserve.DecodeOnly(dec, &event); err != nil {
return nil, fmt.Errorf("not a JSON object: %v", err)
}
if event == nil {
return nil, errors.New("not a JSON object")
}
- if dec.More() {
- return nil, errors.New("more than one JSON value on the line")
- }
- var trailing json.RawMessage
- if err := dec.Decode(&trailing); err == nil {
- return nil, errors.New("more than one JSON value on the line")
- }
return event, nil
}
diff --git a/pkg/asymptoteobserve/localingest/localingest_test.go b/pkg/asymptoteobserve/localingest/localingest_test.go
index ca22b524..44b4da01 100644
--- a/pkg/asymptoteobserve/localingest/localingest_test.go
+++ b/pkg/asymptoteobserve/localingest/localingest_test.go
@@ -186,6 +186,32 @@ func TestStampLineRejects(t *testing.T) {
}
}
+// A line must be exactly one JSON value. In particular, Decoder.More is not a
+// top-level EOF check: unmatched closing delimiters must not be silently dropped.
+func TestStampLineRejectsTrailingClosingDelimiters(t *testing.T) {
+ for _, suffix := range []string{"}", "]", " }", "\t]", "} {", "] null"} {
+ t.Run(suffix, func(t *testing.T) {
+ line := append(validLine(t, nil), suffix...)
+ if json.Valid(line) {
+ t.Fatal("fixture unexpectedly contains valid JSON")
+ }
+ if out, err := StampLine(line, Peer{UID: "501"}, nil); !errors.Is(err, ErrRejected) {
+ t.Fatalf("err = %v, out = %s; want ErrRejected for invalid JSON", err, out)
+ }
+ })
+ }
+}
+
+func TestStampLineAcceptsTrailingJSONWhitespace(t *testing.T) {
+ line := append(validLine(t, nil), " \t\r"...)
+ if !json.Valid(line) {
+ t.Fatal("fixture is not valid JSON")
+ }
+ if _, err := StampLine(line, Peer{UID: "501"}, nil); err != nil {
+ t.Fatalf("valid trailing whitespace rejected: %v", err)
+ }
+}
+
func TestStampLineRejectsALineThatOutgrowsTheLimitWhenStamped(t *testing.T) {
base := validLine(t, func(e map[string]interface{}) { delete(e, "user") })
pad := MaxLineBytes - len(base) - 10