Skip to content

The cross-harness, self-improving memory layer for AI agents. Join our community: https://discord.com/invite/zdNChS2fBu

perf(sanitize): sanitize typed event fields without JSON round trips (#1144)

kyle-elliott-asymptoteef45d33f3b25
diff --git a/pkg/asymptoteobserve/privacy.go b/pkg/asymptoteobserve/privacy.go
index 4d05f4ce..94c16e8b 100644
--- a/pkg/asymptoteobserve/privacy.go
+++ b/pkg/asymptoteobserve/privacy.go
@@ -892,17 +892,17 @@ func SanitizeEvent(event Event, maxBytes int) Event {
event.Prompt.Text = CleanString(event.Prompt.Text, DefaultStringLimit, true)
}
if event.MCP != nil {
- event.MCP = sanitizeTyped(event.MCP, PrivacyOptions{RedactSecrets: true, StringLimit: DefaultRawStringLimit})
+ event.MCP = sanitizeMCP(event.MCP)
}
if event.SystemContext != nil {
- event.SystemContext = sanitizeTyped(event.SystemContext, PrivacyOptions{RedactSecrets: true, StringLimit: DefaultRawStringLimit})
+ event.SystemContext = sanitizeSystemContext(event.SystemContext)
}
if event.GenAI != nil {
description := ""
if event.GenAI.Tool != nil {
description = event.GenAI.Tool.Description
}
- event.GenAI = sanitizeTyped(event.GenAI, PrivacyOptions{RedactSecrets: true, StringLimit: DefaultRawStringLimit})
+ event.GenAI = sanitizeGenAI(event.GenAI)
// A tool description is text the model reads, so it gets the prompt-text limit rather than
// the raw-attribute one: under 2 KB a padded description would hide its instructions past
// the point the poisoning rule sees.
@@ -913,8 +913,12 @@ func SanitizeEvent(event Event, maxBytes int) Event {
if event.Raw != nil {
event.Raw = SanitizeMap(event.Raw, PrivacyOptions{RedactSecrets: true, StringLimit: DefaultRawStringLimit})
}
- if data, err := json.Marshal(event); err == nil && len(data) > maxBytes {
- event.Truncated = true
+ // Marshaling the event only to measure it was a third of SanitizeEvent's cost. A bound on its
+ // size decides most events without it; a marshal that fails leaves Truncated as it was.
+ if n, ok := jsonSizeBound(&event, maxBytes); !ok || n > maxBytes {
+ if data, err := json.Marshal(event); err == nil && len(data) > maxBytes {
+ event.Truncated = true
+ }
}
return event
}
diff --git a/pkg/asymptoteobserve/privacy_bench_test.go b/pkg/asymptoteobserve/privacy_bench_test.go
index 68e722a3..56d58e34 100644
--- a/pkg/asymptoteobserve/privacy_bench_test.go
+++ b/pkg/asymptoteobserve/privacy_bench_test.go
@@ -6,10 +6,11 @@ import (
"testing"
obs "github.com/asymptote-labs/agent-beacon/pkg/asymptoteobserve"
+ "github.com/asymptote-labs/agent-beacon/pkg/asymptoteobserve/threatrules"
)
-// The benchmarks in this file time redaction on the text a hook redacts most: a file read's result,
-// a system prompt, a shell command, and text whose every line holds a secret.
+// The benchmarks in this file time redaction and event sanitizing on what a hook handles most: a
+// file read's result, a system prompt, a shell command, and text whose every line holds a secret.
var benchGoLines = []string{
"package cache", "", "import (", "\t\"container/list\"", "\t\"sync\"", "\t\"time\"", ")",
@@ -82,3 +83,62 @@ func BenchmarkRedactAdversarial(b *testing.B) {
})
}
}
+
+// benchEvents are a file read carrying its 3 KB result, the session.context event of a 12 KB system
+// prompt, and a shell command, as a writer builds them before sanitizing.
+func benchEvents() []struct {
+ name string
+ event obs.Event
+} {
+ base := func(action, category string) obs.Event {
+ e := obs.NewEvent(obs.NewEventOptions{Action: action, Category: category, Harness: obs.HarnessInfo{Name: "claude_code"}})
+ e.Session = &obs.SessionInfo{ID: "0b6f4c1e-1d5a-4c83-9a57-2c5b8e0f3d11", WorkingDirectory: "/home/user/project"}
+ e.Content = &obs.ContentInfo{Retention: obs.ContentRetentionFull, Included: true}
+ e.Raw = map[string]interface{}{"hook_event_name": "PostToolUse", "tool_use_id": "toolu_01ABCDEF"}
+ return e
+ }
+ read := base("file.read", "file")
+ read.Tool = &obs.ToolInfo{Name: "Read", Path: "/home/user/project/internal/cache/lru.go"}
+ read.File = &obs.FileInfo{Path: "/home/user/project/internal/cache/lru.go", Operation: "read"}
+ read.GenAI = &obs.GenAIInfo{Tool: &obs.GenAIToolInfo{Name: "Read", Call: &obs.GenAIToolCallInfo{
+ ID: "toolu_01ABCDEF", Arguments: map[string]interface{}{"file_path": "/home/user/project/internal/cache/lru.go"}, Result: benchFileRead(3000),
+ }}}
+ context := base("session.context", "session")
+ context.SystemContext = &obs.SystemContextInfo{Kind: "system_prompt", Source: obs.SystemContextSourceSystemPrompt}
+ context.GenAI = &obs.GenAIInfo{SystemInstructions: obs.SystemInstructionParts(benchSystemPrompt(12000))}
+ command := base("command.executed", "command")
+ command.Tool = &obs.ToolInfo{Name: "Bash", Command: "go test ./internal/cache/ -run TestLRUEviction -count=20"}
+ command.Command = &obs.CommandInfo{Command: "go test ./internal/cache/ -run TestLRUEviction -count=20"}
+ command.GenAI = &obs.GenAIInfo{Tool: &obs.GenAIToolInfo{Name: "Bash", Call: &obs.GenAIToolCallInfo{
+ ID: "toolu_01ABCDEF", Arguments: map[string]interface{}{"command": "go test ./internal/cache/ -run TestLRUEviction -count=20", "description": "Run the test"},
+ }}}
+ return []struct {
+ name string
+ event obs.Event
+ }{{"file.read_3KB", read}, {"session.context_12KB", context}, {"command.executed", command}}
+}
+
+// BenchmarkSanitizeEvent sanitizes each event. Sanitizing writes cleaned text through the event's
+// pointers, which for these events leaves it as it was, so each iteration does the same work.
+func BenchmarkSanitizeEvent(b *testing.B) {
+ for _, bc := range benchEvents() {
+ b.Run(bc.name, func(b *testing.B) {
+ b.ReportAllocs()
+ for b.Loop() {
+ obs.SanitizeEvent(bc.event, 65536)
+ }
+ })
+ }
+}
+
+// BenchmarkToolResultText derives gen_ai.tool.call.result_text from the sanitized file read.
+func BenchmarkToolResultText(b *testing.B) {
+ e := obs.SanitizeEvent(benchEvents()[0].event, 65536)
+ if threatrules.ToolResultText(e) == "" {
+ b.Fatal("no result text")
+ }
+ b.ReportAllocs()
+ for b.Loop() {
+ threatrules.ToolResultText(e)
+ }
+}
diff --git a/pkg/asymptoteobserve/sanitize_typed.go b/pkg/asymptoteobserve/sanitize_typed.go
new file mode 100644
index 00000000..aef9f7c5
--- /dev/null
+++ b/pkg/asymptoteobserve/sanitize_typed.go
@@ -0,0 +1,720 @@
+package asymptoteobserve
+
+import (
+ "encoding"
+ "encoding/json"
+ "math"
+ "reflect"
+ "strconv"
+ "strings"
+ "sync"
+ "unicode/utf8"
+)
+
+// This file is how SanitizeEvent (privacy.go) sanitizes the typed parts of an event and decides
+// whether it is too large, without the JSON round trips it used:
+//
+// - MCP, SystemContext and GenAI were sanitized by sanitizeTyped: marshal the value, decode it
+// into a map, run SanitizeMap over the map, marshal the map, and decode it back into the type.
+// Each field is now converted directly into what that round trip leaves of it
+// (genericSanitizer), and the round trip itself runs only for a value the conversion does not
+// know, or one the round trip fails on, so its fallback is kept too.
+// - Truncated was set by marshaling the whole event and comparing the length with maxBytes. An
+// upper bound on that length is now added up first (jsonSizeBound), and the event is marshaled
+// only when the bound exceeds maxBytes.
+//
+// sanitize_typed_test.go checks the conversion's effects one by one (TestSanitizeEventTypedFields),
+// that it copies every field of the types it converts (TestTypedConversionKeepsEveryField), that
+// the bound never undercounts (TestJSONSizeBoundIsABound), and that Truncated is set exactly when
+// the marshaled event exceeds maxBytes (TestSanitizeEventTruncatesAtItsMarshaledSize).
+
+// genericSanitizer converts typed fields and the values interface{} fields hold into what
+// sanitizeTyped's round trip leaves of them, sanitized the way SanitizeMap and SanitizeSlice
+// sanitize the decoded value:
+//
+// - A string passes through JSON with each byte of invalid UTF-8 replaced by U+FFFD, as the
+// encoder writes it, and is then sanitized.
+// - A number in an interface{} decodes as float64, and a float32 as the float64 nearest its
+// shortest decimal form. An integer field reads back the float64 as the encoder writes it, its
+// shortest decimal form, so beyond 2^53 it changes value (2^63-1024 comes back as
+// 9223372036854775000), and where that form is out of range (from about 2^63-512 up, and at
+// -2^63) the decode fails, which fails the round trip.
+// - NaN and infinities fail the marshal, which fails the round trip.
+// - A nil map, slice or pointer, in an interface{} or a field, and an omitempty field that is
+// empty, decode as nil; an empty map or slice in an interface{} stays empty, not nil. A non-nil
+// pointer to an empty struct decodes as an empty struct.
+// - `json:"-"` fields (ResultText, SystemInstructionsText) are not written, so they come back
+// empty.
+//
+// failed is set for anything it does not convert (a value of another type, a map key that is not
+// valid UTF-8, which the encoder can merge with another, nesting deeper than maxGenericDepth) and
+// for what fails the round trip; the caller then runs sanitizeTyped, which handles each of these
+// as it always did.
+type genericSanitizer struct {
+ limit int
+ failed bool
+}
+
+// maxGenericDepth bounds the nesting genericSanitizer converts and jsonSizeBound adds up. The
+// decoder fails beyond 10000 levels and the encoder detects cycles after 1000.
+const maxGenericDepth = 100
+
+// text is a typed string field, none of which is named for a credential
+// (TestTypedFieldNamesAreNotSecret).
+func (g *genericSanitizer) text(s string) string {
+ return CleanString(jsonString(s), g.limit, true)
+}
+
+// jsonString is s as the encoder writes it and the decoder reads it back: each byte that is not
+// part of a valid UTF-8 encoding becomes U+FFFD.
+func jsonString(s string) string {
+ if utf8.ValidString(s) {
+ return s
+ }
+ var b strings.Builder
+ b.Grow(len(s) + 16)
+ for i := 0; i < len(s); {
+ r, size := utf8.DecodeRuneInString(s[i:])
+ if r == utf8.RuneError && size == 1 {
+ b.WriteString("\uFFFD")
+ } else {
+ b.WriteString(s[i : i+size])
+ }
+ i += size
+ }
+ return b.String()
+}
+
+// integer is an integer field's value after the round trip.
+func (g *genericSanitizer) integer(v int64) int64 {
+ if -1<<53 <= v && v <= 1<<53 {
+ return v
+ }
+ n, err := strconv.ParseInt(strconv.FormatFloat(float64(v), 'f', -1, 64), 10, 64)
+ if err != nil {
+ g.failed = true
+ }
+ return n
+}
+
+func (g *genericSanitizer) intp(p *int) *int {
+ if p == nil {
+ return nil
+ }
+ n := g.integer(int64(*p))
+ if n < math.MinInt || n > math.MaxInt { // cannot happen for an int; out of range, the round trip fails too
+ g.failed = true
+ return nil
+ }
+ v := int(n)
+ return &v
+}
+
+func (g *genericSanitizer) int64p(p *int64) *int64 {
+ if p == nil {
+ return nil
+ }
+ v := g.integer(*p)
+ return &v
+}
+
+func (g *genericSanitizer) float64p(p *float64) *float64 {
+ if p == nil {
+ return nil
+ }
+ if math.IsNaN(*p) || math.IsInf(*p, 0) {
+ g.failed = true
+ return nil
+ }
+ v := *p
+ return &v
+}
+
+func boolp(p *bool) *bool {
+ if p == nil {
+ return nil
+ }
+ v := *p
+ return &v
+}
+
+// strings is a []string field, which SanitizeSlice sanitizes: none is named for a credential.
+func (g *genericSanitizer) strings(in []string) []string {
+ if len(in) == 0 {
+ return nil
+ }
+ out := make([]string, len(in))
+ for i, s := range in {
+ s = jsonString(s)
+ if s != "" && i > 0 && IsSecretFlag(jsonString(in[i-1])) {
+ out[i] = redactedValue
+ continue
+ }
+ out[i] = CleanString(s, g.limit, true)
+ }
+ return out
+}
+
+// field is the value of an interface{} field, none of which is named for a credential.
+func (g *genericSanitizer) field(v any) any {
+ return g.value(v, "", 1)
+}
+
+// value converts v, held under key in a map, and sanitizes it as SanitizeMap does.
+func (g *genericSanitizer) value(v any, key string, depth int) any {
+ if depth > maxGenericDepth {
+ g.failed = true
+ return nil
+ }
+ switch x := v.(type) {
+ case string:
+ x = jsonString(x)
+ if x != "" && IsSecretField(key, x) {
+ return redactedValue
+ }
+ return CleanString(x, g.limit, true)
+ case []any, []string:
+ items, ok := g.items(x, depth)
+ if !ok || items == nil {
+ return nil
+ }
+ if IsSecretKeyName(key) {
+ return g.redactAll(items, depth)
+ }
+ return g.slice(items, depth)
+ }
+ return g.scalarOrObject(v, depth)
+}
+
+// scalarOrObject converts v when it is neither a string nor a list, which SanitizeMap,
+// SanitizeSlice and redactAllStrings all treat the same.
+func (g *genericSanitizer) scalarOrObject(v any, depth int) any {
+ switch x := v.(type) {
+ case nil:
+ return nil
+ case map[string]any:
+ if x == nil {
+ return nil
+ }
+ out := make(map[string]any, len(x))
+ for k, item := range x {
+ if !utf8.ValidString(k) {
+ g.failed = true
+ return nil
+ }
+ out[k] = g.value(item, k, depth+1)
+ }
+ return out
+ case map[string]string:
+ if x == nil {
+ return nil
+ }
+ out := make(map[string]any, len(x))
+ for k, item := range x {
+ if !utf8.ValidString(k) {
+ g.failed = true
+ return nil
+ }
+ out[k] = g.value(item, k, depth+1)
+ }
+ return out
+ case bool:
+ return x
+ case float64:
+ if math.IsNaN(x) || math.IsInf(x, 0) {
+ g.failed = true
+ return nil
+ }
+ return x
+ case float32:
+ if math.IsNaN(float64(x)) || math.IsInf(float64(x), 0) {
+ g.failed = true
+ return nil
+ }
+ f, _ := strconv.ParseFloat(strconv.FormatFloat(float64(x), 'g', -1, 32), 64)
+ return f
+ case int:
+ return float64(x)
+ case int8:
+ return float64(x)
+ case int16:
+ return float64(x)
+ case int32:
+ return float64(x)
+ case int64:
+ return float64(x)
+ case uint:
+ return float64(x)
+ case uint8:
+ return float64(x)
+ case uint16:
+ return float64(x)
+ case uint32:
+ return float64(x)
+ case uint64:
+ return float64(x)
+ }
+ g.failed = true
+ return nil
+}
+
+// items returns a list as it decodes: nil for a nil list, its items otherwise, with strings as
+// the encoder writes them.
+func (g *genericSanitizer) items(v any, depth int) ([]any, bool) {
+ if depth > maxGenericDepth {
+ g.failed = true
+ return nil, false
+ }
+ switch x := v.(type) {
+ case []any:
+ if x == nil {
+ return nil, true
+ }
+ out := make([]any, len(x))
+ for i, item := range x {
+ if s, ok := item.(string); ok {
+ item = jsonString(s)
+ }
+ out[i] = item
+ }
+ return out, true
+ case []string:
+ if x == nil {
+ return nil, true
+ }
+ out := make([]any, len(x))
+ for i, item := range x {
+ out[i] = jsonString(item)
+ }
+ return out, true
+ }
+ return nil, false
+}
+
+// slice is SanitizeSlice over decoded items. A flag named for a secret is read from the item as
+// decoded, before it is sanitized in place.
+func (g *genericSanitizer) slice(items []any, depth int) []any {
+ var prev any
+ for i, item := range items {
+ previous := prev
+ prev = item
+ switch x := item.(type) {
+ case string:
+ if x != "" && i > 0 && followsSecretFlag(previous) {
+ items[i] = redactedValue
+ continue
+ }
+ items[i] = CleanString(x, g.limit, true)
+ case []any, []string:
+ nested, ok := g.items(x, depth+1)
+ if !ok || nested == nil {
+ items[i] = nil
+ continue
+ }
+ items[i] = g.slice(nested, depth+1)
+ default:
+ items[i] = g.scalarOrObject(item, depth+1)
+ }
+ }
+ return items
+}
+
+// redactAll is redactAllStrings over decoded items.
+func (g *genericSanitizer) redactAll(items []any, depth int) []any {
+ for i, item := range items {
+ switch x := item.(type) {
+ case string:
+ if x != "" {
+ items[i] = redactedValue
+ }
+ case []any, []string:
+ nested, ok := g.items(x, depth+1)
+ if !ok || nested == nil {
+ items[i] = nil
+ continue
+ }
+ items[i] = g.redactAll(nested, depth+1)
+ default:
+ items[i] = g.scalarOrObject(item, depth+1)
+ }
+ }
+ return items
+}
+
+func sanitizeMCP(in *MCPInfo) *MCPInfo {
+ g := genericSanitizer{limit: DefaultRawStringLimit}
+ out := &MCPInfo{Server: g.text(in.Server), Tool: g.text(in.Tool)}
+ if in.Method != nil {
+ out.Method = &MCPMethodInfo{Name: g.text(in.Method.Name)}
+ }
+ if in.Protocol != nil {
+ out.Protocol = &MCPProtocolInfo{Version: g.text(in.Protocol.Version)}
+ }
+ if in.Resource != nil {
+ out.Resource = &MCPResourceInfo{URI: g.text(in.Resource.URI)}
+ }
+ if in.Session != nil {
+ out.Session = &MCPSessionInfo{ID: g.text(in.Session.ID)}
+ }
+ return out
+}
+
+func sanitizeSystemContext(in *SystemContextInfo) *SystemContextInfo {
+ g := genericSanitizer{limit: DefaultRawStringLimit}
+ out := &SystemContextInfo{Kind: g.text(in.Kind), Source: g.text(in.Source)}
+ if len(in.Skills) > 0 {
+ out.Skills = make([]SkillRefInfo, len(in.Skills))
+ for i, s := range in.Skills {
+ out.Skills[i] = SkillRefInfo{SkillName: g.text(s.SkillName), SkillNameHash: g.text(s.SkillNameHash)}
+ }
+ }
+ return out
+}
+
+// sanitizeGenAI is sanitizeTyped for GenAI, except that a converted tool's Description is left for
+// SanitizeEvent to fill.
+func sanitizeGenAI(in *GenAIInfo) *GenAIInfo {
+ g := genericSanitizer{limit: DefaultRawStringLimit}
+ out := &GenAIInfo{}
+ if a := in.Agent; a != nil {
+ out.Agent = &GenAIAgentInfo{Description: g.text(a.Description), ID: g.text(a.ID), Name: g.text(a.Name), Version: g.text(a.Version)}
+ }
+ if c := in.Context; c != nil {
+ out.Context = &GenAIContextInfo{UsedTokens: g.int64p(c.UsedTokens), LimitTokens: g.int64p(c.LimitTokens)}
+ }
+ if c := in.Conversation; c != nil {
+ out.Conversation = &GenAIConversationInfo{ID: g.text(c.ID)}
+ }
+ if d := in.DataSource; d != nil {
+ out.DataSource = &GenAIDataSourceInfo{ID: g.text(d.ID)}
+ }
+ if e := in.Embeddings; e != nil {
+ out.Embeddings = &GenAIEmbeddingsInfo{DimensionCount: g.intp(e.DimensionCount)}
+ }
+ if e := in.Evaluation; e != nil {
+ out.Evaluation = &GenAIEvaluationInfo{Explanation: g.text(e.Explanation), Name: g.text(e.Name)}
+ if s := e.Score; s != nil {
+ out.Evaluation.Score = &GenAIEvaluationScoreInfo{Label: g.text(s.Label), Value: g.float64p(s.Value)}
+ }
+ }
+ if i := in.Input; i != nil {
+ out.Input = &GenAIInputInfo{Messages: g.field(i.Messages)}
+ }
+ if o := in.Operation; o != nil {
+ out.Operation = &GenAIOperationInfo{Name: g.text(o.Name)}
+ }
+ if o := in.Output; o != nil {
+ out.Output = &GenAIOutputInfo{Messages: g.field(o.Messages), Type: g.text(o.Type)}
+ }
+ if p := in.Prompt; p != nil {
+ out.Prompt = &GenAIPromptInfo{Name: g.text(p.Name)}
+ }
+ if p := in.Provider; p != nil {
+ out.Provider = &GenAIProviderInfo{Name: g.text(p.Name)}
+ }
+ if r := in.Request; r != nil {
+ out.Request = &GenAIRequestInfo{
+ ChoiceCount: g.intp(r.ChoiceCount),
+ EncodingFormats: g.strings(r.EncodingFormats),
+ FrequencyPenalty: g.float64p(r.FrequencyPenalty),
+ MaxTokens: g.intp(r.MaxTokens),
+ Model: g.text(r.Model),
+ PresencePenalty: g.float64p(r.PresencePenalty),
+ Seed: g.intp(r.Seed),
+ StopSequences: g.strings(r.StopSequences),
+ Stream: boolp(r.Stream),
+ Temperature: g.float64p(r.Temperature),
+ TopK: g.float64p(r.TopK),
+ TopP: g.float64p(r.TopP),
+ }
+ }
+ if r := in.Response; r != nil {
+ out.Response = &GenAIResponseInfo{
+ FinishReasons: g.strings(r.FinishReasons),
+ ID: g.text(r.ID),
+ Model: g.text(r.Model),
+ TimeToFirstChunk: g.float64p(r.TimeToFirstChunk),
+ }
+ }
+ if r := in.Retrieval; r != nil {
+ out.Retrieval = &GenAIRetrievalInfo{Documents: g.field(r.Documents), QueryText: g.text(r.QueryText)}
+ }
+ out.SystemInstructions = g.field(in.SystemInstructions)
+ if t := in.Token; t != nil {
+ out.Token = &GenAITokenInfo{Type: g.text(t.Type)}
+ }
+ if t := in.Tool; t != nil {
+ // Description is left empty: SanitizeEvent replaces a non-empty one, from the description as
+ // given, and an empty one is empty either way.
+ out.Tool = &GenAIToolInfo{Definitions: g.field(t.Definitions), Name: g.text(t.Name), Type: g.text(t.Type)}
+ if c := t.Call; c != nil {
+ out.Tool.Call = &GenAIToolCallInfo{Arguments: g.field(c.Arguments), ID: g.text(c.ID), Result: g.field(c.Result)}
+ }
+ }
+ if u := in.Usage; u != nil {
+ out.Usage = &GenAIUsageInfo{
+ CostUSD: g.float64p(u.CostUSD),
+ InputTokens: g.int64p(u.InputTokens),
+ OutputTokens: g.int64p(u.OutputTokens),
+ }
+ if c := u.CacheCreation; c != nil {
+ out.Usage.CacheCreation = &GenAIUsageCacheCreationInfo{InputTokens: g.int64p(c.InputTokens), Ephemeral1hInputTokens: g.int64p(c.Ephemeral1hInputTokens)}
+ }
+ if c := u.CacheRead; c != nil {
+ out.Usage.CacheRead = &GenAIUsageCacheReadInfo{InputTokens: g.int64p(c.InputTokens)}
+ }
+ if r := u.Reasoning; r != nil {
+ out.Usage.Reasoning = &GenAIUsageReasoningInfo{OutputTokens: g.int64p(r.OutputTokens)}
+ }
+ }
+ if w := in.Workflow; w != nil {
+ out.Workflow = &GenAIWorkflowInfo{Name: g.text(w.Name)}
+ }
+ if g.failed {
+ return sanitizeTyped(in, PrivacyOptions{RedactSecrets: true, StringLimit: DefaultRawStringLimit})
+ }
+ return out
+}
+
+// ---- the size bound ----
+
+// jsonSizeBound returns an upper bound on len(json.Marshal(v)), giving up (ok false) once it passes
+// limit, or at a value whose encoding it does not bound: a type with its own marshaler, a map whose
+// keys are not strings, a struct with embedded or ",string" fields, a channel or a function, or
+// nesting deeper than maxGenericDepth. A string is bounded at six bytes a byte, which no escape
+// exceeds, and a number at 32 bytes.
+func jsonSizeBound(v any, limit int) (n int, ok bool) {
+ b := sizeBounder{limit: limit}
+ b.any(v, 0)
+ return b.n, b.ok()
+}
+
+type sizeBounder struct {
+ n, limit int
+ failed bool
+}
+
+func (b *sizeBounder) ok() bool { return !b.failed && b.n <= b.limit }
+
+func (b *sizeBounder) str(n int) { b.n += 2 + 6*n }
+
+// any bounds a value held in an interface{}: the shapes JSON decodes to directly, anything else by
+// reflection.
+func (b *sizeBounder) any(v any, depth int) {
+ if !b.ok() {
+ return
+ }
+ switch x := v.(type) {
+ case nil:
+ b.n += 4
+ case string:
+ b.str(len(x))
+ case bool:
+ b.n += 5
+ case float64, int, int64:
+ b.n += 32
+ case map[string]any:
+ b.object(x, depth)
+ case []any:
+ if x == nil {
+ b.n += 4
+ return
+ }
+ if depth > maxGenericDepth {
+ b.failed = true
+ return
+ }
+ b.n += 2
+ for _, item := range x {
+ b.n++
+ b.any(item, depth+1)
+ }
+ default:
+ rv := reflect.ValueOf(v)
+ b.value(rv, boundOf(rv.Type()), depth)
+ }
+}
+
+func (b *sizeBounder) object(m map[string]any, depth int) {
+ if m == nil {
+ b.n += 4
+ return
+ }
+ if depth > maxGenericDepth {
+ b.failed = true
+ return
+ }
+ b.n += 2
+ for k, item := range m {
+ b.str(len(k))
+ b.n += 2
+ b.any(item, depth+1)
+ if !b.ok() {
+ return
+ }
+ }
+}
+
+var (
+ jsonMarshalerType = reflect.TypeFor[json.Marshaler]()
+ textMarshalerType = reflect.TypeFor[encoding.TextMarshaler]()
+ genericMapType = reflect.TypeFor[map[string]any]()
+)
+
+// typeBound is what bounding a type needs to know of it, worked out once per type, with the
+// bounds of the types it holds linked in, so a walk looks a type up only where an interface{}
+// holds it.
+type typeBound struct {
+ kind reflect.Kind
+ // custom is set when the type, or a pointer to it, marshals itself, or is a struct whose
+ // fields json flattens or quotes.
+ custom bool
+ // generic is set for map[string]any, walked without reflection.
+ generic bool
+ elem, key *typeBound
+ fields []fieldBound
+}
+
+type fieldBound struct {
+ index int
+ name int // the encoded name with its quotes and colon, plus a comma
+ bound *typeBound
+}
+
+var (
+ typeBounds sync.Map // reflect.Type -> *typeBound, complete
+ typeBoundsMu sync.Mutex
+)
+
+func boundOf(t reflect.Type) *typeBound {
+ if tb, ok := typeBounds.Load(t); ok {
+ return tb.(*typeBound)
+ }
+ typeBoundsMu.Lock()
+ defer typeBoundsMu.Unlock()
+ building := map[reflect.Type]*typeBound{}
+ tb := buildBound(t, building)
+ for t, tb := range building {
+ typeBounds.Store(t, tb)
+ }
+ return tb
+}
+
+func buildBound(t reflect.Type, building map[reflect.Type]*typeBound) *typeBound {
+ if tb, ok := typeBounds.Load(t); ok {
+ return tb.(*typeBound)
+ }
+ if tb, ok := building[t]; ok {
+ return tb
+ }
+ tb := &typeBound{
+ kind: t.Kind(), generic: t == genericMapType,
+ custom: t.Implements(jsonMarshalerType) || t.Implements(textMarshalerType) ||
+ reflect.PointerTo(t).Implements(jsonMarshalerType) || reflect.PointerTo(t).Implements(textMarshalerType),
+ }
+ building[t] = tb
+ switch t.Kind() {
+ case reflect.Pointer, reflect.Slice, reflect.Array:
+ tb.elem = buildBound(t.Elem(), building)
+ case reflect.Map:
+ tb.key, tb.elem = buildBound(t.Key(), building), buildBound(t.Elem(), building)
+ case reflect.Struct:
+ for i := range t.NumField() {
+ f := t.Field(i)
+ tag := f.Tag.Get("json")
+ if tag == "-" {
+ continue
+ }
+ name, opts, _ := strings.Cut(tag, ",")
+ if f.Anonymous || strings.Contains(opts, "string") {
+ tb.custom = true
+ break
+ }
+ if !f.IsExported() {
+ continue
+ }
+ // The encoder falls back to the Go name for a tag name it rejects; the longer of the
+ // two covers both.
+ tagged, _ := json.Marshal(name)
+ named, _ := json.Marshal(f.Name)
+ tb.fields = append(tb.fields, fieldBound{index: i, name: max(len(tagged), len(named)) + 2, bound: buildBound(f.Type, building)})
+ }
+ }
+ return tb
+}
+
+func (b *sizeBounder) value(v reflect.Value, tb *typeBound, depth int) {
+ if !b.ok() {
+ return
+ }
+ if depth > maxGenericDepth {
+ b.failed = true
+ return
+ }
+ switch tb.kind {
+ case reflect.Pointer, reflect.Interface:
+ if v.IsNil() {
+ b.n += 4
+ return
+ }
+ }
+ if tb.custom {
+ b.failed = true
+ return
+ }
+ switch tb.kind {
+ case reflect.String:
+ b.str(v.Len())
+ case reflect.Bool:
+ b.n += 5
+ case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64,
+ reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64, reflect.Uintptr,
+ reflect.Float32, reflect.Float64:
+ b.n += 32
+ case reflect.Pointer:
+ b.value(v.Elem(), tb.elem, depth+1)
+ case reflect.Interface:
+ b.any(v.Interface(), depth+1)
+ case reflect.Struct:
+ b.n += 2
+ for _, f := range tb.fields {
+ b.n += f.name
+ b.value(v.Field(f.index), f.bound, depth+1)
+ }
+ case reflect.Map:
+ if v.IsNil() {
+ b.n += 4
+ return
+ }
+ if tb.generic {
+ b.object(v.Interface().(map[string]any), depth)
+ return
+ }
+ if tb.key.kind != reflect.String || tb.key.custom {
+ b.failed = true
+ return
+ }
+ b.n += 2
+ for it := v.MapRange(); it.Next() && b.ok(); {
+ b.str(it.Key().Len())
+ b.n += 2
+ b.value(it.Value(), tb.elem, depth+1)
+ }
+ case reflect.Slice, reflect.Array:
+ if tb.kind == reflect.Slice && v.IsNil() {
+ b.n += 4
+ return
+ }
+ if tb.kind == reflect.Slice && tb.elem.kind == reflect.Uint8 && !tb.elem.custom {
+ b.n += 2 + (v.Len()+2)/3*4
+ return
+ }
+ b.n += 2
+ for i := range v.Len() {
+ b.n++
+ b.value(v.Index(i), tb.elem, depth+1)
+ }
+ default:
+ b.failed = true
+ }
+}
diff --git a/pkg/asymptoteobserve/sanitize_typed_test.go b/pkg/asymptoteobserve/sanitize_typed_test.go
new file mode 100644
index 00000000..1f7e1993
--- /dev/null
+++ b/pkg/asymptoteobserve/sanitize_typed_test.go
@@ -0,0 +1,361 @@
+package asymptoteobserve
+
+import (
+ "encoding/json"
+ "fmt"
+ "math"
+ "math/rand/v2"
+ "reflect"
+ "strings"
+ "testing"
+ "time"
+)
+
+// SanitizeEvent's typed conversion has every effect the JSON round trip it replaced had, checked
+// one by one: invalid UTF-8, integers beyond 2^53, float32, empty omitempty fields, json:"-" fields,
+// secret-named keys judged with their values, secret flags in lists, nil and empty lists in
+// interface{} values, and the tool description at the prompt-text limit.
+func TestSanitizeEventTypedFields(t *testing.T) {
+ beyond := int64(1<<53 + 1)
+ count := int64(1200)
+ arguments := map[string]any{
+ "api_key": "plain", "token_type": "bearer", "max_tokens": "4096", "n": 3, "f": float32(0.1),
+ "args": []string{"-y", "--api-key", "k", "ok"}, "cookie": []any{"a", "", []any{"b"}},
+ "empty": []any{}, "nil": []any(nil), "bad": "x\xffy",
+ }
+ e := Event{
+ MCP: &MCPInfo{Server: "notes\xff", Tool: "get"},
+ SystemContext: &SystemContextInfo{Kind: SystemContextSkillListing, Skills: []SkillRefInfo{}},
+ GenAI: &GenAIInfo{
+ Usage: &GenAIUsageInfo{InputTokens: &beyond, OutputTokens: &count},
+ Request: &GenAIRequestInfo{StopSequences: []string{"--password", "hunter2", "END"}, EncodingFormats: []string{}},
+ Tool: &GenAIToolInfo{Name: "mcp__notes__get", Description: "Fetch a note. token=abcdefgh12345678", Call: &GenAIToolCallInfo{
+ ID: "t1", ResultText: "asserted", Arguments: arguments,
+ Result: "export GITHUB_TOKEN=ghp_0123456789abcdefghijklmnopqrstuvwxyz",
+ }},
+ Agent: &GenAIAgentInfo{},
+ SystemInstructionsText: "asserted",
+ },
+ }
+ got := SanitizeEvent(e, 65536)
+
+ if got.MCP.Server != "notes\uFFFD" || got.MCP.Tool != "get" {
+ t.Errorf("mcp = %+v", got.MCP)
+ }
+ if got.SystemContext.Kind != SystemContextSkillListing || got.SystemContext.Skills != nil {
+ t.Errorf("system context = %+v, want the empty skills list gone", got.SystemContext)
+ }
+ g := got.GenAI
+ if *g.Usage.InputTokens != 1<<53 || *g.Usage.OutputTokens != 1200 {
+ t.Errorf("usage = %d, %d; want 2^53+1 read back as 2^53, and 1200", *g.Usage.InputTokens, *g.Usage.OutputTokens)
+ }
+ if want := []string{"--password", "[REDACTED]", "END"}; !reflect.DeepEqual(g.Request.StopSequences, want) || g.Request.EncodingFormats != nil {
+ t.Errorf("request = %+v", g.Request)
+ }
+ if g.Agent == nil || *g.Agent != (GenAIAgentInfo{}) {
+ t.Errorf("an empty agent block = %+v, want kept empty", g.Agent)
+ }
+ if g.SystemInstructionsText != "" || g.Tool.Call.ResultText != "" {
+ t.Errorf("derived fields kept: %q, %q", g.SystemInstructionsText, g.Tool.Call.ResultText)
+ }
+ if g.Tool.Description != "Fetch a note. token=[REDACTED]" {
+ t.Errorf("description = %q", g.Tool.Description)
+ }
+ if g.Tool.Call.Result != "export GITHUB_TOKEN=[REDACTED]" {
+ t.Errorf("result = %q", g.Tool.Call.Result)
+ }
+ want := map[string]any{
+ "api_key": "[REDACTED]", "token_type": "bearer", "max_tokens": "4096", "n": float64(3), "f": 0.1,
+ "args": []any{"-y", "--api-key", "[REDACTED]", "ok"}, "cookie": []any{"[REDACTED]", "", []any{"[REDACTED]"}},
+ "empty": []any{}, "nil": nil, "bad": "x\uFFFDy",
+ }
+ if !reflect.DeepEqual(g.Tool.Call.Arguments, want) {
+ t.Errorf("arguments = %#v\nwant %#v", g.Tool.Call.Arguments, want)
+ }
+ if arguments["api_key"] != "plain" || e.GenAI.Tool.Call.ResultText != "asserted" {
+ t.Error("SanitizeEvent changed the caller's gen_ai")
+ }
+}
+
+// Truncated is set exactly when the sanitized event's JSON is longer than maxBytes, whether the
+// size bound decides it or the event is marshaled: on events right around the size, with values the
+// bound gives up on, and on random events.
+func TestSanitizeEventTruncatesAtItsMarshaledSize(t *testing.T) {
+ const maxBytes = 65536
+ check := func(what string, e Event) {
+ t.Helper()
+ got := SanitizeEvent(e, maxBytes)
+ measured := got
+ measured.Truncated = e.Truncated
+ data, err := json.Marshal(measured)
+ want := e.Truncated || err == nil && len(data) > maxBytes
+ if got.Truncated != want {
+ t.Fatalf("%s: truncated %v, want %v (%d bytes, %v)", what, got.Truncated, want, len(data), err)
+ }
+ }
+ base := func(pad int) Event {
+ raw := map[string]any{}
+ for i := range 31 {
+ raw[fmt.Sprintf("k%02d", i)] = strings.Repeat("<", 340)
+ }
+ raw["pad"] = strings.Repeat("a", pad)
+ return Event{Vendor: Vendor, Message: "m", Raw: raw}
+ }
+ for pad := 0; pad < 1200; pad += 7 {
+ check(fmt.Sprintf("pad %d", pad), base(pad))
+ }
+ for _, v := range []any{
+ make(chan int), func() {}, math.NaN(), json.Number("12"), json.RawMessage(`{"a":1}`), time.Unix(0, 0),
+ map[int]string{1: strings.Repeat("x", 70000)}, []byte(strings.Repeat("x", 70000)), struct{ A string }{strings.Repeat("x", 70000)},
+ } {
+ for _, pad := range []int{0, 600} {
+ e := base(pad)
+ e.Raw["v"] = v
+ check(fmt.Sprintf("raw %T, pad %d", v, pad), e)
+ }
+ }
+ r := rand.New(rand.NewPCG(1, 2))
+ for i := range 1000 {
+ check(fmt.Sprintf("random event %d", i), randomEvent(r))
+ }
+}
+
+// The size bound never undercounts what json.Marshal writes.
+func TestJSONSizeBoundIsABound(t *testing.T) {
+ r := rand.New(rand.NewPCG(3, 4))
+ bounded := 0
+ for i := range 2000 {
+ e := randomEvent(r)
+ data, err := json.Marshal(e)
+ n, ok := jsonSizeBound(&e, math.MaxInt)
+ if err == nil && ok {
+ bounded++
+ if n < len(data) {
+ t.Fatalf("event %d: bound %d, marshaled %d bytes", i, n, len(data))
+ }
+ }
+ }
+ if bounded < 500 {
+ t.Fatalf("only %d of 2000 events were bounded", bounded)
+ }
+}
+
+// The typed conversion sanitizes typed string and list fields without asking whether their name
+// marks them as a credential, which holds only while no such field has a name that does.
+func TestTypedFieldNamesAreNotSecret(t *testing.T) {
+ seen := map[reflect.Type]bool{}
+ var walk func(t reflect.Type, path string)
+ walk = func(typ reflect.Type, path string) {
+ for typ.Kind() == reflect.Pointer || typ.Kind() == reflect.Slice {
+ typ = typ.Elem()
+ }
+ if typ.Kind() != reflect.Struct || seen[typ] {
+ return
+ }
+ seen[typ] = true
+ for i := range typ.NumField() {
+ f := typ.Field(i)
+ name, _, _ := strings.Cut(f.Tag.Get("json"), ",")
+ if name == "-" {
+ continue
+ }
+ k := f.Type.Kind()
+ if (k == reflect.String || k == reflect.Slice || k == reflect.Interface) && IsSecretKeyName(name) {
+ t.Errorf("%s.%s (%q) is named for a credential; sanitize_typed.go must check it", path, f.Name, name)
+ }
+ walk(f.Type, path+"."+f.Name)
+ }
+ }
+ for _, v := range []any{MCPInfo{}, SystemContextInfo{}, GenAIInfo{}} {
+ walk(reflect.TypeOf(v), reflect.TypeOf(v).Name())
+ }
+}
+
+// The typed conversion copies every field of the types it converts: with values sanitizing leaves
+// alone, the result is the input, but for the fields that never survive the round trip (json:"-")
+// and the tool description, which SanitizeEvent fills. A field added to these types and not to
+// sanitize_typed.go fails here.
+func TestTypedConversionKeepsEveryField(t *testing.T) {
+ var fill func(v reflect.Value)
+ fill = func(v reflect.Value) {
+ switch v.Kind() {
+ case reflect.String:
+ v.SetString("v")
+ case reflect.Bool:
+ v.SetBool(true)
+ case reflect.Int, reflect.Int64:
+ v.SetInt(7)
+ case reflect.Float64:
+ v.SetFloat(1.5)
+ case reflect.Pointer:
+ v.Set(reflect.New(v.Type().Elem()))
+ fill(v.Elem())
+ case reflect.Slice:
+ v.Set(reflect.MakeSlice(v.Type(), 1, 1))
+ fill(v.Index(0))
+ case reflect.Interface:
+ v.Set(reflect.ValueOf(map[string]any{"k": "v", "n": 1.5, "l": []any{"v", true, nil}}))
+ case reflect.Struct:
+ for i := range v.NumField() {
+ fill(v.Field(i))
+ }
+ default:
+ t.Fatalf("no value for a %s field", v.Kind())
+ }
+ }
+ var mcp MCPInfo
+ var context SystemContextInfo
+ var genAI GenAIInfo
+ for _, v := range []any{&mcp, &context, &genAI} {
+ fill(reflect.ValueOf(v).Elem())
+ }
+ if got := sanitizeMCP(&mcp); !reflect.DeepEqual(*got, mcp) {
+ t.Errorf("mcp = %+v, want %+v", *got, mcp)
+ }
+ if got := sanitizeSystemContext(&context); !reflect.DeepEqual(*got, context) {
+ t.Errorf("system context = %+v, want %+v", *got, context)
+ }
+ want := genAI
+ want.SystemInstructionsText = ""
+ tool := *want.Tool
+ tool.Description = ""
+ call := *tool.Call
+ call.ResultText = ""
+ tool.Call, want.Tool = &call, &tool
+ if got := sanitizeGenAI(&genAI); !reflect.DeepEqual(*got, want) {
+ t.Errorf("gen_ai = %s\nwant %s", mustJSON(t, got), mustJSON(t, want))
+ }
+}
+
+func mustJSON(t *testing.T, v any) string {
+ data, err := json.Marshal(v)
+ if err != nil {
+ t.Fatal(err)
+ }
+ return string(data)
+}
+
+// ---- random events ----
+
+func randomTestString(r *rand.Rand) string {
+ pieces := []string{
+ "token=abcdefgh12345678", `"api_key": "x y z"`, "--password hunter2", "Authorization: Bearer abc.def",
+ "ghp_0123456789abcdefghijklmnopqrstuvwxyz", "postgres://user:pw@host/db", "input_tokens: 812",
+ " ", "\n", "\t", `"`, "'", ",", "<&>", "\u2028", "\x01", "\xff", "é", "--api-key", "plain text",
+ }
+ var s strings.Builder
+ for n := r.IntN(5); n >= 0; n-- {
+ if r.IntN(10) == 0 {
+ // Long text, around the limits.
+ n := []int{100, 2040, 2048, 2060, 4090, 4100, 8200, 12300}[r.IntN(8)]
+ s.WriteString(strings.Repeat("ab cd ", n/6+1)[:n])
+ continue
+ }
+ s.WriteString(pieces[r.IntN(len(pieces))])
+ }
+ return s.String()
+}
+
+func randomTestKey(r *rand.Rand) string {
+ keys := []string{"password", "api_key", "GITHUB_TOKEN", "Cookie", "max_tokens", "token_type", "name", "text", "args", "", "\xffkey"}
+ return keys[r.IntN(len(keys))]
+}
+
+// randomGeneric returns a value of a shape an interface{} field or raw can hold: what JSON decodes
+// to, and the Go values an in-process caller puts there.
+func randomGeneric(r *rand.Rand, depth int) any {
+ n := 16
+ if depth > 3 {
+ n = 9
+ }
+ switch r.IntN(n) {
+ case 0:
+ return nil
+ case 1, 2, 3:
+ return randomTestString(r)
+ case 4:
+ return r.IntN(2) == 0
+ case 5:
+ return []float64{0, math.Copysign(0, -1), 1.5, 1e21, 1e-7, 1 << 53, math.MaxFloat64, math.NaN(), math.Inf(1)}[r.IntN(9)]
+ case 6:
+ return []any{int(42), int64(math.MaxInt64), int64(math.MinInt64), int32(-7), uint64(math.MaxUint64), float32(0.1)}[r.IntN(6)]
+ case 7:
+ return []any{json.Number("12"), []byte("bytes"), struct{ N int }{3}, map[int]string{1: "x"}, []string{"--token", "t"}}[r.IntN(5)]
+ case 8:
+ return []any{[]any(nil), map[string]any(nil), []any{}, map[string]any{}}[r.IntN(4)]
+ case 9, 10, 11, 12:
+ out := make([]any, r.IntN(4))
+ for i := range out {
+ out[i] = randomGeneric(r, depth+1)
+ }
+ return out
+ default:
+ out := map[string]any{}
+ for range r.IntN(4) {
+ out[randomTestKey(r)] = randomGeneric(r, depth+1)
+ }
+ return out
+ }
+}
+
+// fillRandom sets v, of any type an event holds, at random, leaving fields zero now and then.
+func fillRandom(r *rand.Rand, v reflect.Value, depth int) {
+ if depth > 0 && r.IntN(4) == 0 {
+ return
+ }
+ switch v.Kind() {
+ case reflect.String:
+ v.SetString(randomTestString(r))
+ case reflect.Bool:
+ v.SetBool(r.IntN(2) == 0)
+ case reflect.Int, reflect.Int64, reflect.Int32:
+ v.SetInt([]int64{0, 1, -1, 1<<53 + 1, math.MaxInt64, math.MinInt64}[r.IntN(6)])
+ case reflect.Uint, reflect.Uint64:
+ v.SetUint([]uint64{0, 1, math.MaxUint64}[r.IntN(3)])
+ case reflect.Float64:
+ v.SetFloat([]float64{0, 1.5, 1e21, 1e-7, math.NaN()}[r.IntN(5)])
+ case reflect.Pointer:
+ p := reflect.New(v.Type().Elem())
+ fillRandom(r, p.Elem(), depth+1)
+ v.Set(p)
+ case reflect.Interface:
+ if g := randomGeneric(r, depth); g != nil {
+ v.Set(reflect.ValueOf(g))
+ }
+ case reflect.Slice:
+ s := reflect.MakeSlice(v.Type(), r.IntN(3), 3)
+ for i := range s.Len() {
+ fillRandom(r, s.Index(i), depth+1)
+ }
+ v.Set(s)
+ case reflect.Map:
+ if v.Type() == reflect.TypeFor[map[string]any]() {
+ m := map[string]any{}
+ for range r.IntN(5) {
+ m[randomTestKey(r)] = randomGeneric(r, depth)
+ }
+ v.Set(reflect.ValueOf(m))
+ }
+ case reflect.Struct:
+ for i := range v.NumField() {
+ if v.Type().Field(i).IsExported() {
+ fillRandom(r, v.Field(i), depth+1)
+ }
+ }
+ }
+}
+
+func randomEvent(r *rand.Rand) Event {
+ var e Event
+ fillRandom(r, reflect.ValueOf(&e).Elem(), 0)
+ if r.IntN(3) == 0 {
+ // A large raw, near the size an event is marked truncated at.
+ if e.Raw == nil {
+ e.Raw = map[string]any{}
+ }
+ for i := range 28 + r.IntN(8) {
+ e.Raw[fmt.Sprintf("k%d", i)] = strings.Repeat("<a>", 700+r.IntN(30))
+ }
+ }
+ return e
+}