The cross-harness, self-improving memory layer for AI agents. Join our community: https://discord.com/invite/zdNChS2fBu
Go agent-ai-cliagent-harnessesagent-securitybrowser-extensionciclaude-codecloudcodexcursordetection-engineeringjevknowledge-basememorymobile-device-managementobservabilitysecuritysecurity-information-and-event-managementsecurity-toolstelemetrytraces
perf(sanitize): sanitize typed event fields without JSON round trips (#1144)
kyle-elliott-asymptoteef45d33f3b25
diff --git a/pkg/asymptoteobserve/privacy.go b/pkg/asymptoteobserve/privacy.goindex 4d05f4ce..94c16e8b 100644--- a/pkg/asymptoteobserve/privacy.go+++ b/pkg/asymptoteobserve/privacy.go@@ -892,17 +892,17 @@ func SanitizeEvent(event Event, maxBytes int) Event {event.Prompt.Text = CleanString(event.Prompt.Text, DefaultStringLimit, true)}if event.MCP != nil {- event.MCP = sanitizeTyped(event.MCP, PrivacyOptions{RedactSecrets: true, StringLimit: DefaultRawStringLimit})+ event.MCP = sanitizeMCP(event.MCP)}if event.SystemContext != nil {- event.SystemContext = sanitizeTyped(event.SystemContext, PrivacyOptions{RedactSecrets: true, StringLimit: DefaultRawStringLimit})+ event.SystemContext = sanitizeSystemContext(event.SystemContext)}if event.GenAI != nil {description := ""if event.GenAI.Tool != nil {description = event.GenAI.Tool.Description}- event.GenAI = sanitizeTyped(event.GenAI, PrivacyOptions{RedactSecrets: true, StringLimit: DefaultRawStringLimit})+ event.GenAI = sanitizeGenAI(event.GenAI)// A tool description is text the model reads, so it gets the prompt-text limit rather than// the raw-attribute one: under 2 KB a padded description would hide its instructions past// the point the poisoning rule sees.@@ -913,8 +913,12 @@ func SanitizeEvent(event Event, maxBytes int) Event {if event.Raw != nil {event.Raw = SanitizeMap(event.Raw, PrivacyOptions{RedactSecrets: true, StringLimit: DefaultRawStringLimit})}- if data, err := json.Marshal(event); err == nil && len(data) > maxBytes {- event.Truncated = true+ // Marshaling the event only to measure it was a third of SanitizeEvent's cost. A bound on its+ // size decides most events without it; a marshal that fails leaves Truncated as it was.+ if n, ok := jsonSizeBound(&event, maxBytes); !ok || n > maxBytes {+ if data, err := json.Marshal(event); err == nil && len(data) > maxBytes {+ event.Truncated = true+ }}return event}diff --git a/pkg/asymptoteobserve/privacy_bench_test.go b/pkg/asymptoteobserve/privacy_bench_test.goindex 68e722a3..56d58e34 100644--- a/pkg/asymptoteobserve/privacy_bench_test.go+++ b/pkg/asymptoteobserve/privacy_bench_test.go@@ -6,10 +6,11 @@ import ("testing"obs "github.com/asymptote-labs/agent-beacon/pkg/asymptoteobserve"+ "github.com/asymptote-labs/agent-beacon/pkg/asymptoteobserve/threatrules")-// The benchmarks in this file time redaction on the text a hook redacts most: a file read's result,-// a system prompt, a shell command, and text whose every line holds a secret.+// The benchmarks in this file time redaction and event sanitizing on what a hook handles most: a+// file read's result, a system prompt, a shell command, and text whose every line holds a secret.var benchGoLines = []string{"package cache", "", "import (", "\t\"container/list\"", "\t\"sync\"", "\t\"time\"", ")",@@ -82,3 +83,62 @@ func BenchmarkRedactAdversarial(b *testing.B) {})}}++// benchEvents are a file read carrying its 3 KB result, the session.context event of a 12 KB system+// prompt, and a shell command, as a writer builds them before sanitizing.+func benchEvents() []struct {+ name string+ event obs.Event+} {+ base := func(action, category string) obs.Event {+ e := obs.NewEvent(obs.NewEventOptions{Action: action, Category: category, Harness: obs.HarnessInfo{Name: "claude_code"}})+ e.Session = &obs.SessionInfo{ID: "0b6f4c1e-1d5a-4c83-9a57-2c5b8e0f3d11", WorkingDirectory: "/home/user/project"}+ e.Content = &obs.ContentInfo{Retention: obs.ContentRetentionFull, Included: true}+ e.Raw = map[string]interface{}{"hook_event_name": "PostToolUse", "tool_use_id": "toolu_01ABCDEF"}+ return e+ }+ read := base("file.read", "file")+ read.Tool = &obs.ToolInfo{Name: "Read", Path: "/home/user/project/internal/cache/lru.go"}+ read.File = &obs.FileInfo{Path: "/home/user/project/internal/cache/lru.go", Operation: "read"}+ read.GenAI = &obs.GenAIInfo{Tool: &obs.GenAIToolInfo{Name: "Read", Call: &obs.GenAIToolCallInfo{+ ID: "toolu_01ABCDEF", Arguments: map[string]interface{}{"file_path": "/home/user/project/internal/cache/lru.go"}, Result: benchFileRead(3000),+ }}}+ context := base("session.context", "session")+ context.SystemContext = &obs.SystemContextInfo{Kind: "system_prompt", Source: obs.SystemContextSourceSystemPrompt}+ context.GenAI = &obs.GenAIInfo{SystemInstructions: obs.SystemInstructionParts(benchSystemPrompt(12000))}+ command := base("command.executed", "command")+ command.Tool = &obs.ToolInfo{Name: "Bash", Command: "go test ./internal/cache/ -run TestLRUEviction -count=20"}+ command.Command = &obs.CommandInfo{Command: "go test ./internal/cache/ -run TestLRUEviction -count=20"}+ command.GenAI = &obs.GenAIInfo{Tool: &obs.GenAIToolInfo{Name: "Bash", Call: &obs.GenAIToolCallInfo{+ ID: "toolu_01ABCDEF", Arguments: map[string]interface{}{"command": "go test ./internal/cache/ -run TestLRUEviction -count=20", "description": "Run the test"},+ }}}+ return []struct {+ name string+ event obs.Event+ }{{"file.read_3KB", read}, {"session.context_12KB", context}, {"command.executed", command}}+}++// BenchmarkSanitizeEvent sanitizes each event. Sanitizing writes cleaned text through the event's+// pointers, which for these events leaves it as it was, so each iteration does the same work.+func BenchmarkSanitizeEvent(b *testing.B) {+ for _, bc := range benchEvents() {+ b.Run(bc.name, func(b *testing.B) {+ b.ReportAllocs()+ for b.Loop() {+ obs.SanitizeEvent(bc.event, 65536)+ }+ })+ }+}++// BenchmarkToolResultText derives gen_ai.tool.call.result_text from the sanitized file read.+func BenchmarkToolResultText(b *testing.B) {+ e := obs.SanitizeEvent(benchEvents()[0].event, 65536)+ if threatrules.ToolResultText(e) == "" {+ b.Fatal("no result text")+ }+ b.ReportAllocs()+ for b.Loop() {+ threatrules.ToolResultText(e)+ }+}diff --git a/pkg/asymptoteobserve/sanitize_typed.go b/pkg/asymptoteobserve/sanitize_typed.gonew file mode 100644index 00000000..aef9f7c5--- /dev/null+++ b/pkg/asymptoteobserve/sanitize_typed.go@@ -0,0 +1,720 @@+package asymptoteobserve++import (+ "encoding"+ "encoding/json"+ "math"+ "reflect"+ "strconv"+ "strings"+ "sync"+ "unicode/utf8"+)++// This file is how SanitizeEvent (privacy.go) sanitizes the typed parts of an event and decides+// whether it is too large, without the JSON round trips it used:+//+// - MCP, SystemContext and GenAI were sanitized by sanitizeTyped: marshal the value, decode it+// into a map, run SanitizeMap over the map, marshal the map, and decode it back into the type.+// Each field is now converted directly into what that round trip leaves of it+// (genericSanitizer), and the round trip itself runs only for a value the conversion does not+// know, or one the round trip fails on, so its fallback is kept too.+// - Truncated was set by marshaling the whole event and comparing the length with maxBytes. An+// upper bound on that length is now added up first (jsonSizeBound), and the event is marshaled+// only when the bound exceeds maxBytes.+//+// sanitize_typed_test.go checks the conversion's effects one by one (TestSanitizeEventTypedFields),+// that it copies every field of the types it converts (TestTypedConversionKeepsEveryField), that+// the bound never undercounts (TestJSONSizeBoundIsABound), and that Truncated is set exactly when+// the marshaled event exceeds maxBytes (TestSanitizeEventTruncatesAtItsMarshaledSize).++// genericSanitizer converts typed fields and the values interface{} fields hold into what+// sanitizeTyped's round trip leaves of them, sanitized the way SanitizeMap and SanitizeSlice+// sanitize the decoded value:+//+// - A string passes through JSON with each byte of invalid UTF-8 replaced by U+FFFD, as the+// encoder writes it, and is then sanitized.+// - A number in an interface{} decodes as float64, and a float32 as the float64 nearest its+// shortest decimal form. An integer field reads back the float64 as the encoder writes it, its+// shortest decimal form, so beyond 2^53 it changes value (2^63-1024 comes back as+// 9223372036854775000), and where that form is out of range (from about 2^63-512 up, and at+// -2^63) the decode fails, which fails the round trip.+// - NaN and infinities fail the marshal, which fails the round trip.+// - A nil map, slice or pointer, in an interface{} or a field, and an omitempty field that is+// empty, decode as nil; an empty map or slice in an interface{} stays empty, not nil. A non-nil+// pointer to an empty struct decodes as an empty struct.+// - `json:"-"` fields (ResultText, SystemInstructionsText) are not written, so they come back+// empty.+//+// failed is set for anything it does not convert (a value of another type, a map key that is not+// valid UTF-8, which the encoder can merge with another, nesting deeper than maxGenericDepth) and+// for what fails the round trip; the caller then runs sanitizeTyped, which handles each of these+// as it always did.+type genericSanitizer struct {+ limit int+ failed bool+}++// maxGenericDepth bounds the nesting genericSanitizer converts and jsonSizeBound adds up. The+// decoder fails beyond 10000 levels and the encoder detects cycles after 1000.+const maxGenericDepth = 100++// text is a typed string field, none of which is named for a credential+// (TestTypedFieldNamesAreNotSecret).+func (g *genericSanitizer) text(s string) string {+ return CleanString(jsonString(s), g.limit, true)+}++// jsonString is s as the encoder writes it and the decoder reads it back: each byte that is not+// part of a valid UTF-8 encoding becomes U+FFFD.+func jsonString(s string) string {+ if utf8.ValidString(s) {+ return s+ }+ var b strings.Builder+ b.Grow(len(s) + 16)+ for i := 0; i < len(s); {+ r, size := utf8.DecodeRuneInString(s[i:])+ if r == utf8.RuneError && size == 1 {+ b.WriteString("\uFFFD")+ } else {+ b.WriteString(s[i : i+size])+ }+ i += size+ }+ return b.String()+}++// integer is an integer field's value after the round trip.+func (g *genericSanitizer) integer(v int64) int64 {+ if -1<<53 <= v && v <= 1<<53 {+ return v+ }+ n, err := strconv.ParseInt(strconv.FormatFloat(float64(v), 'f', -1, 64), 10, 64)+ if err != nil {+ g.failed = true+ }+ return n+}++func (g *genericSanitizer) intp(p *int) *int {+ if p == nil {+ return nil+ }+ n := g.integer(int64(*p))+ if n < math.MinInt || n > math.MaxInt { // cannot happen for an int; out of range, the round trip fails too+ g.failed = true+ return nil+ }+ v := int(n)+ return &v+}++func (g *genericSanitizer) int64p(p *int64) *int64 {+ if p == nil {+ return nil+ }+ v := g.integer(*p)+ return &v+}++func (g *genericSanitizer) float64p(p *float64) *float64 {+ if p == nil {+ return nil+ }+ if math.IsNaN(*p) || math.IsInf(*p, 0) {+ g.failed = true+ return nil+ }+ v := *p+ return &v+}++func boolp(p *bool) *bool {+ if p == nil {+ return nil+ }+ v := *p+ return &v+}++// strings is a []string field, which SanitizeSlice sanitizes: none is named for a credential.+func (g *genericSanitizer) strings(in []string) []string {+ if len(in) == 0 {+ return nil+ }+ out := make([]string, len(in))+ for i, s := range in {+ s = jsonString(s)+ if s != "" && i > 0 && IsSecretFlag(jsonString(in[i-1])) {+ out[i] = redactedValue+ continue+ }+ out[i] = CleanString(s, g.limit, true)+ }+ return out+}++// field is the value of an interface{} field, none of which is named for a credential.+func (g *genericSanitizer) field(v any) any {+ return g.value(v, "", 1)+}++// value converts v, held under key in a map, and sanitizes it as SanitizeMap does.+func (g *genericSanitizer) value(v any, key string, depth int) any {+ if depth > maxGenericDepth {+ g.failed = true+ return nil+ }+ switch x := v.(type) {+ case string:+ x = jsonString(x)+ if x != "" && IsSecretField(key, x) {+ return redactedValue+ }+ return CleanString(x, g.limit, true)+ case []any, []string:+ items, ok := g.items(x, depth)+ if !ok || items == nil {+ return nil+ }+ if IsSecretKeyName(key) {+ return g.redactAll(items, depth)+ }+ return g.slice(items, depth)+ }+ return g.scalarOrObject(v, depth)+}++// scalarOrObject converts v when it is neither a string nor a list, which SanitizeMap,+// SanitizeSlice and redactAllStrings all treat the same.+func (g *genericSanitizer) scalarOrObject(v any, depth int) any {+ switch x := v.(type) {+ case nil:+ return nil+ case map[string]any:+ if x == nil {+ return nil+ }+ out := make(map[string]any, len(x))+ for k, item := range x {+ if !utf8.ValidString(k) {+ g.failed = true+ return nil+ }+ out[k] = g.value(item, k, depth+1)+ }+ return out+ case map[string]string:+ if x == nil {+ return nil+ }+ out := make(map[string]any, len(x))+ for k, item := range x {+ if !utf8.ValidString(k) {+ g.failed = true+ return nil+ }+ out[k] = g.value(item, k, depth+1)+ }+ return out+ case bool:+ return x+ case float64:+ if math.IsNaN(x) || math.IsInf(x, 0) {+ g.failed = true+ return nil+ }+ return x+ case float32:+ if math.IsNaN(float64(x)) || math.IsInf(float64(x), 0) {+ g.failed = true+ return nil+ }+ f, _ := strconv.ParseFloat(strconv.FormatFloat(float64(x), 'g', -1, 32), 64)+ return f+ case int:+ return float64(x)+ case int8:+ return float64(x)+ case int16:+ return float64(x)+ case int32:+ return float64(x)+ case int64:+ return float64(x)+ case uint:+ return float64(x)+ case uint8:+ return float64(x)+ case uint16:+ return float64(x)+ case uint32:+ return float64(x)+ case uint64:+ return float64(x)+ }+ g.failed = true+ return nil+}++// items returns a list as it decodes: nil for a nil list, its items otherwise, with strings as+// the encoder writes them.+func (g *genericSanitizer) items(v any, depth int) ([]any, bool) {+ if depth > maxGenericDepth {+ g.failed = true+ return nil, false+ }+ switch x := v.(type) {+ case []any:+ if x == nil {+ return nil, true+ }+ out := make([]any, len(x))+ for i, item := range x {+ if s, ok := item.(string); ok {+ item = jsonString(s)+ }+ out[i] = item+ }+ return out, true+ case []string:+ if x == nil {+ return nil, true+ }+ out := make([]any, len(x))+ for i, item := range x {+ out[i] = jsonString(item)+ }+ return out, true+ }+ return nil, false+}++// slice is SanitizeSlice over decoded items. A flag named for a secret is read from the item as+// decoded, before it is sanitized in place.+func (g *genericSanitizer) slice(items []any, depth int) []any {+ var prev any+ for i, item := range items {+ previous := prev+ prev = item+ switch x := item.(type) {+ case string:+ if x != "" && i > 0 && followsSecretFlag(previous) {+ items[i] = redactedValue+ continue+ }+ items[i] = CleanString(x, g.limit, true)+ case []any, []string:+ nested, ok := g.items(x, depth+1)+ if !ok || nested == nil {+ items[i] = nil+ continue+ }+ items[i] = g.slice(nested, depth+1)+ default:+ items[i] = g.scalarOrObject(item, depth+1)+ }+ }+ return items+}++// redactAll is redactAllStrings over decoded items.+func (g *genericSanitizer) redactAll(items []any, depth int) []any {+ for i, item := range items {+ switch x := item.(type) {+ case string:+ if x != "" {+ items[i] = redactedValue+ }+ case []any, []string:+ nested, ok := g.items(x, depth+1)+ if !ok || nested == nil {+ items[i] = nil+ continue+ }+ items[i] = g.redactAll(nested, depth+1)+ default:+ items[i] = g.scalarOrObject(item, depth+1)+ }+ }+ return items+}++func sanitizeMCP(in *MCPInfo) *MCPInfo {+ g := genericSanitizer{limit: DefaultRawStringLimit}+ out := &MCPInfo{Server: g.text(in.Server), Tool: g.text(in.Tool)}+ if in.Method != nil {+ out.Method = &MCPMethodInfo{Name: g.text(in.Method.Name)}+ }+ if in.Protocol != nil {+ out.Protocol = &MCPProtocolInfo{Version: g.text(in.Protocol.Version)}+ }+ if in.Resource != nil {+ out.Resource = &MCPResourceInfo{URI: g.text(in.Resource.URI)}+ }+ if in.Session != nil {+ out.Session = &MCPSessionInfo{ID: g.text(in.Session.ID)}+ }+ return out+}++func sanitizeSystemContext(in *SystemContextInfo) *SystemContextInfo {+ g := genericSanitizer{limit: DefaultRawStringLimit}+ out := &SystemContextInfo{Kind: g.text(in.Kind), Source: g.text(in.Source)}+ if len(in.Skills) > 0 {+ out.Skills = make([]SkillRefInfo, len(in.Skills))+ for i, s := range in.Skills {+ out.Skills[i] = SkillRefInfo{SkillName: g.text(s.SkillName), SkillNameHash: g.text(s.SkillNameHash)}+ }+ }+ return out+}++// sanitizeGenAI is sanitizeTyped for GenAI, except that a converted tool's Description is left for+// SanitizeEvent to fill.+func sanitizeGenAI(in *GenAIInfo) *GenAIInfo {+ g := genericSanitizer{limit: DefaultRawStringLimit}+ out := &GenAIInfo{}+ if a := in.Agent; a != nil {+ out.Agent = &GenAIAgentInfo{Description: g.text(a.Description), ID: g.text(a.ID), Name: g.text(a.Name), Version: g.text(a.Version)}+ }+ if c := in.Context; c != nil {+ out.Context = &GenAIContextInfo{UsedTokens: g.int64p(c.UsedTokens), LimitTokens: g.int64p(c.LimitTokens)}+ }+ if c := in.Conversation; c != nil {+ out.Conversation = &GenAIConversationInfo{ID: g.text(c.ID)}+ }+ if d := in.DataSource; d != nil {+ out.DataSource = &GenAIDataSourceInfo{ID: g.text(d.ID)}+ }+ if e := in.Embeddings; e != nil {+ out.Embeddings = &GenAIEmbeddingsInfo{DimensionCount: g.intp(e.DimensionCount)}+ }+ if e := in.Evaluation; e != nil {+ out.Evaluation = &GenAIEvaluationInfo{Explanation: g.text(e.Explanation), Name: g.text(e.Name)}+ if s := e.Score; s != nil {+ out.Evaluation.Score = &GenAIEvaluationScoreInfo{Label: g.text(s.Label), Value: g.float64p(s.Value)}+ }+ }+ if i := in.Input; i != nil {+ out.Input = &GenAIInputInfo{Messages: g.field(i.Messages)}+ }+ if o := in.Operation; o != nil {+ out.Operation = &GenAIOperationInfo{Name: g.text(o.Name)}+ }+ if o := in.Output; o != nil {+ out.Output = &GenAIOutputInfo{Messages: g.field(o.Messages), Type: g.text(o.Type)}+ }+ if p := in.Prompt; p != nil {+ out.Prompt = &GenAIPromptInfo{Name: g.text(p.Name)}+ }+ if p := in.Provider; p != nil {+ out.Provider = &GenAIProviderInfo{Name: g.text(p.Name)}+ }+ if r := in.Request; r != nil {+ out.Request = &GenAIRequestInfo{+ ChoiceCount: g.intp(r.ChoiceCount),+ EncodingFormats: g.strings(r.EncodingFormats),+ FrequencyPenalty: g.float64p(r.FrequencyPenalty),+ MaxTokens: g.intp(r.MaxTokens),+ Model: g.text(r.Model),+ PresencePenalty: g.float64p(r.PresencePenalty),+ Seed: g.intp(r.Seed),+ StopSequences: g.strings(r.StopSequences),+ Stream: boolp(r.Stream),+ Temperature: g.float64p(r.Temperature),+ TopK: g.float64p(r.TopK),+ TopP: g.float64p(r.TopP),+ }+ }+ if r := in.Response; r != nil {+ out.Response = &GenAIResponseInfo{+ FinishReasons: g.strings(r.FinishReasons),+ ID: g.text(r.ID),+ Model: g.text(r.Model),+ TimeToFirstChunk: g.float64p(r.TimeToFirstChunk),+ }+ }+ if r := in.Retrieval; r != nil {+ out.Retrieval = &GenAIRetrievalInfo{Documents: g.field(r.Documents), QueryText: g.text(r.QueryText)}+ }+ out.SystemInstructions = g.field(in.SystemInstructions)+ if t := in.Token; t != nil {+ out.Token = &GenAITokenInfo{Type: g.text(t.Type)}+ }+ if t := in.Tool; t != nil {+ // Description is left empty: SanitizeEvent replaces a non-empty one, from the description as+ // given, and an empty one is empty either way.+ out.Tool = &GenAIToolInfo{Definitions: g.field(t.Definitions), Name: g.text(t.Name), Type: g.text(t.Type)}+ if c := t.Call; c != nil {+ out.Tool.Call = &GenAIToolCallInfo{Arguments: g.field(c.Arguments), ID: g.text(c.ID), Result: g.field(c.Result)}+ }+ }+ if u := in.Usage; u != nil {+ out.Usage = &GenAIUsageInfo{+ CostUSD: g.float64p(u.CostUSD),+ InputTokens: g.int64p(u.InputTokens),+ OutputTokens: g.int64p(u.OutputTokens),+ }+ if c := u.CacheCreation; c != nil {+ out.Usage.CacheCreation = &GenAIUsageCacheCreationInfo{InputTokens: g.int64p(c.InputTokens), Ephemeral1hInputTokens: g.int64p(c.Ephemeral1hInputTokens)}+ }+ if c := u.CacheRead; c != nil {+ out.Usage.CacheRead = &GenAIUsageCacheReadInfo{InputTokens: g.int64p(c.InputTokens)}+ }+ if r := u.Reasoning; r != nil {+ out.Usage.Reasoning = &GenAIUsageReasoningInfo{OutputTokens: g.int64p(r.OutputTokens)}+ }+ }+ if w := in.Workflow; w != nil {+ out.Workflow = &GenAIWorkflowInfo{Name: g.text(w.Name)}+ }+ if g.failed {+ return sanitizeTyped(in, PrivacyOptions{RedactSecrets: true, StringLimit: DefaultRawStringLimit})+ }+ return out+}++// ---- the size bound ----++// jsonSizeBound returns an upper bound on len(json.Marshal(v)), giving up (ok false) once it passes+// limit, or at a value whose encoding it does not bound: a type with its own marshaler, a map whose+// keys are not strings, a struct with embedded or ",string" fields, a channel or a function, or+// nesting deeper than maxGenericDepth. A string is bounded at six bytes a byte, which no escape+// exceeds, and a number at 32 bytes.+func jsonSizeBound(v any, limit int) (n int, ok bool) {+ b := sizeBounder{limit: limit}+ b.any(v, 0)+ return b.n, b.ok()+}++type sizeBounder struct {+ n, limit int+ failed bool+}++func (b *sizeBounder) ok() bool { return !b.failed && b.n <= b.limit }++func (b *sizeBounder) str(n int) { b.n += 2 + 6*n }++// any bounds a value held in an interface{}: the shapes JSON decodes to directly, anything else by+// reflection.+func (b *sizeBounder) any(v any, depth int) {+ if !b.ok() {+ return+ }+ switch x := v.(type) {+ case nil:+ b.n += 4+ case string:+ b.str(len(x))+ case bool:+ b.n += 5+ case float64, int, int64:+ b.n += 32+ case map[string]any:+ b.object(x, depth)+ case []any:+ if x == nil {+ b.n += 4+ return+ }+ if depth > maxGenericDepth {+ b.failed = true+ return+ }+ b.n += 2+ for _, item := range x {+ b.n+++ b.any(item, depth+1)+ }+ default:+ rv := reflect.ValueOf(v)+ b.value(rv, boundOf(rv.Type()), depth)+ }+}++func (b *sizeBounder) object(m map[string]any, depth int) {+ if m == nil {+ b.n += 4+ return+ }+ if depth > maxGenericDepth {+ b.failed = true+ return+ }+ b.n += 2+ for k, item := range m {+ b.str(len(k))+ b.n += 2+ b.any(item, depth+1)+ if !b.ok() {+ return+ }+ }+}++var (+ jsonMarshalerType = reflect.TypeFor[json.Marshaler]()+ textMarshalerType = reflect.TypeFor[encoding.TextMarshaler]()+ genericMapType = reflect.TypeFor[map[string]any]()+)++// typeBound is what bounding a type needs to know of it, worked out once per type, with the+// bounds of the types it holds linked in, so a walk looks a type up only where an interface{}+// holds it.+type typeBound struct {+ kind reflect.Kind+ // custom is set when the type, or a pointer to it, marshals itself, or is a struct whose+ // fields json flattens or quotes.+ custom bool+ // generic is set for map[string]any, walked without reflection.+ generic bool+ elem, key *typeBound+ fields []fieldBound+}++type fieldBound struct {+ index int+ name int // the encoded name with its quotes and colon, plus a comma+ bound *typeBound+}++var (+ typeBounds sync.Map // reflect.Type -> *typeBound, complete+ typeBoundsMu sync.Mutex+)++func boundOf(t reflect.Type) *typeBound {+ if tb, ok := typeBounds.Load(t); ok {+ return tb.(*typeBound)+ }+ typeBoundsMu.Lock()+ defer typeBoundsMu.Unlock()+ building := map[reflect.Type]*typeBound{}+ tb := buildBound(t, building)+ for t, tb := range building {+ typeBounds.Store(t, tb)+ }+ return tb+}++func buildBound(t reflect.Type, building map[reflect.Type]*typeBound) *typeBound {+ if tb, ok := typeBounds.Load(t); ok {+ return tb.(*typeBound)+ }+ if tb, ok := building[t]; ok {+ return tb+ }+ tb := &typeBound{+ kind: t.Kind(), generic: t == genericMapType,+ custom: t.Implements(jsonMarshalerType) || t.Implements(textMarshalerType) ||+ reflect.PointerTo(t).Implements(jsonMarshalerType) || reflect.PointerTo(t).Implements(textMarshalerType),+ }+ building[t] = tb+ switch t.Kind() {+ case reflect.Pointer, reflect.Slice, reflect.Array:+ tb.elem = buildBound(t.Elem(), building)+ case reflect.Map:+ tb.key, tb.elem = buildBound(t.Key(), building), buildBound(t.Elem(), building)+ case reflect.Struct:+ for i := range t.NumField() {+ f := t.Field(i)+ tag := f.Tag.Get("json")+ if tag == "-" {+ continue+ }+ name, opts, _ := strings.Cut(tag, ",")+ if f.Anonymous || strings.Contains(opts, "string") {+ tb.custom = true+ break+ }+ if !f.IsExported() {+ continue+ }+ // The encoder falls back to the Go name for a tag name it rejects; the longer of the+ // two covers both.+ tagged, _ := json.Marshal(name)+ named, _ := json.Marshal(f.Name)+ tb.fields = append(tb.fields, fieldBound{index: i, name: max(len(tagged), len(named)) + 2, bound: buildBound(f.Type, building)})+ }+ }+ return tb+}++func (b *sizeBounder) value(v reflect.Value, tb *typeBound, depth int) {+ if !b.ok() {+ return+ }+ if depth > maxGenericDepth {+ b.failed = true+ return+ }+ switch tb.kind {+ case reflect.Pointer, reflect.Interface:+ if v.IsNil() {+ b.n += 4+ return+ }+ }+ if tb.custom {+ b.failed = true+ return+ }+ switch tb.kind {+ case reflect.String:+ b.str(v.Len())+ case reflect.Bool:+ b.n += 5+ case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64,+ reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64, reflect.Uintptr,+ reflect.Float32, reflect.Float64:+ b.n += 32+ case reflect.Pointer:+ b.value(v.Elem(), tb.elem, depth+1)+ case reflect.Interface:+ b.any(v.Interface(), depth+1)+ case reflect.Struct:+ b.n += 2+ for _, f := range tb.fields {+ b.n += f.name+ b.value(v.Field(f.index), f.bound, depth+1)+ }+ case reflect.Map:+ if v.IsNil() {+ b.n += 4+ return+ }+ if tb.generic {+ b.object(v.Interface().(map[string]any), depth)+ return+ }+ if tb.key.kind != reflect.String || tb.key.custom {+ b.failed = true+ return+ }+ b.n += 2+ for it := v.MapRange(); it.Next() && b.ok(); {+ b.str(it.Key().Len())+ b.n += 2+ b.value(it.Value(), tb.elem, depth+1)+ }+ case reflect.Slice, reflect.Array:+ if tb.kind == reflect.Slice && v.IsNil() {+ b.n += 4+ return+ }+ if tb.kind == reflect.Slice && tb.elem.kind == reflect.Uint8 && !tb.elem.custom {+ b.n += 2 + (v.Len()+2)/3*4+ return+ }+ b.n += 2+ for i := range v.Len() {+ b.n+++ b.value(v.Index(i), tb.elem, depth+1)+ }+ default:+ b.failed = true+ }+}diff --git a/pkg/asymptoteobserve/sanitize_typed_test.go b/pkg/asymptoteobserve/sanitize_typed_test.gonew file mode 100644index 00000000..1f7e1993--- /dev/null+++ b/pkg/asymptoteobserve/sanitize_typed_test.go@@ -0,0 +1,361 @@+package asymptoteobserve++import (+ "encoding/json"+ "fmt"+ "math"+ "math/rand/v2"+ "reflect"+ "strings"+ "testing"+ "time"+)++// SanitizeEvent's typed conversion has every effect the JSON round trip it replaced had, checked+// one by one: invalid UTF-8, integers beyond 2^53, float32, empty omitempty fields, json:"-" fields,+// secret-named keys judged with their values, secret flags in lists, nil and empty lists in+// interface{} values, and the tool description at the prompt-text limit.+func TestSanitizeEventTypedFields(t *testing.T) {+ beyond := int64(1<<53 + 1)+ count := int64(1200)+ arguments := map[string]any{+ "api_key": "plain", "token_type": "bearer", "max_tokens": "4096", "n": 3, "f": float32(0.1),+ "args": []string{"-y", "--api-key", "k", "ok"}, "cookie": []any{"a", "", []any{"b"}},+ "empty": []any{}, "nil": []any(nil), "bad": "x\xffy",+ }+ e := Event{+ MCP: &MCPInfo{Server: "notes\xff", Tool: "get"},+ SystemContext: &SystemContextInfo{Kind: SystemContextSkillListing, Skills: []SkillRefInfo{}},+ GenAI: &GenAIInfo{+ Usage: &GenAIUsageInfo{InputTokens: &beyond, OutputTokens: &count},+ Request: &GenAIRequestInfo{StopSequences: []string{"--password", "hunter2", "END"}, EncodingFormats: []string{}},+ Tool: &GenAIToolInfo{Name: "mcp__notes__get", Description: "Fetch a note. token=abcdefgh12345678", Call: &GenAIToolCallInfo{+ ID: "t1", ResultText: "asserted", Arguments: arguments,+ Result: "export GITHUB_TOKEN=ghp_0123456789abcdefghijklmnopqrstuvwxyz",+ }},+ Agent: &GenAIAgentInfo{},+ SystemInstructionsText: "asserted",+ },+ }+ got := SanitizeEvent(e, 65536)++ if got.MCP.Server != "notes\uFFFD" || got.MCP.Tool != "get" {+ t.Errorf("mcp = %+v", got.MCP)+ }+ if got.SystemContext.Kind != SystemContextSkillListing || got.SystemContext.Skills != nil {+ t.Errorf("system context = %+v, want the empty skills list gone", got.SystemContext)+ }+ g := got.GenAI+ if *g.Usage.InputTokens != 1<<53 || *g.Usage.OutputTokens != 1200 {+ t.Errorf("usage = %d, %d; want 2^53+1 read back as 2^53, and 1200", *g.Usage.InputTokens, *g.Usage.OutputTokens)+ }+ if want := []string{"--password", "[REDACTED]", "END"}; !reflect.DeepEqual(g.Request.StopSequences, want) || g.Request.EncodingFormats != nil {+ t.Errorf("request = %+v", g.Request)+ }+ if g.Agent == nil || *g.Agent != (GenAIAgentInfo{}) {+ t.Errorf("an empty agent block = %+v, want kept empty", g.Agent)+ }+ if g.SystemInstructionsText != "" || g.Tool.Call.ResultText != "" {+ t.Errorf("derived fields kept: %q, %q", g.SystemInstructionsText, g.Tool.Call.ResultText)+ }+ if g.Tool.Description != "Fetch a note. token=[REDACTED]" {+ t.Errorf("description = %q", g.Tool.Description)+ }+ if g.Tool.Call.Result != "export GITHUB_TOKEN=[REDACTED]" {+ t.Errorf("result = %q", g.Tool.Call.Result)+ }+ want := map[string]any{+ "api_key": "[REDACTED]", "token_type": "bearer", "max_tokens": "4096", "n": float64(3), "f": 0.1,+ "args": []any{"-y", "--api-key", "[REDACTED]", "ok"}, "cookie": []any{"[REDACTED]", "", []any{"[REDACTED]"}},+ "empty": []any{}, "nil": nil, "bad": "x\uFFFDy",+ }+ if !reflect.DeepEqual(g.Tool.Call.Arguments, want) {+ t.Errorf("arguments = %#v\nwant %#v", g.Tool.Call.Arguments, want)+ }+ if arguments["api_key"] != "plain" || e.GenAI.Tool.Call.ResultText != "asserted" {+ t.Error("SanitizeEvent changed the caller's gen_ai")+ }+}++// Truncated is set exactly when the sanitized event's JSON is longer than maxBytes, whether the+// size bound decides it or the event is marshaled: on events right around the size, with values the+// bound gives up on, and on random events.+func TestSanitizeEventTruncatesAtItsMarshaledSize(t *testing.T) {+ const maxBytes = 65536+ check := func(what string, e Event) {+ t.Helper()+ got := SanitizeEvent(e, maxBytes)+ measured := got+ measured.Truncated = e.Truncated+ data, err := json.Marshal(measured)+ want := e.Truncated || err == nil && len(data) > maxBytes+ if got.Truncated != want {+ t.Fatalf("%s: truncated %v, want %v (%d bytes, %v)", what, got.Truncated, want, len(data), err)+ }+ }+ base := func(pad int) Event {+ raw := map[string]any{}+ for i := range 31 {+ raw[fmt.Sprintf("k%02d", i)] = strings.Repeat("<", 340)+ }+ raw["pad"] = strings.Repeat("a", pad)+ return Event{Vendor: Vendor, Message: "m", Raw: raw}+ }+ for pad := 0; pad < 1200; pad += 7 {+ check(fmt.Sprintf("pad %d", pad), base(pad))+ }+ for _, v := range []any{+ make(chan int), func() {}, math.NaN(), json.Number("12"), json.RawMessage(`{"a":1}`), time.Unix(0, 0),+ map[int]string{1: strings.Repeat("x", 70000)}, []byte(strings.Repeat("x", 70000)), struct{ A string }{strings.Repeat("x", 70000)},+ } {+ for _, pad := range []int{0, 600} {+ e := base(pad)+ e.Raw["v"] = v+ check(fmt.Sprintf("raw %T, pad %d", v, pad), e)+ }+ }+ r := rand.New(rand.NewPCG(1, 2))+ for i := range 1000 {+ check(fmt.Sprintf("random event %d", i), randomEvent(r))+ }+}++// The size bound never undercounts what json.Marshal writes.+func TestJSONSizeBoundIsABound(t *testing.T) {+ r := rand.New(rand.NewPCG(3, 4))+ bounded := 0+ for i := range 2000 {+ e := randomEvent(r)+ data, err := json.Marshal(e)+ n, ok := jsonSizeBound(&e, math.MaxInt)+ if err == nil && ok {+ bounded+++ if n < len(data) {+ t.Fatalf("event %d: bound %d, marshaled %d bytes", i, n, len(data))+ }+ }+ }+ if bounded < 500 {+ t.Fatalf("only %d of 2000 events were bounded", bounded)+ }+}++// The typed conversion sanitizes typed string and list fields without asking whether their name+// marks them as a credential, which holds only while no such field has a name that does.+func TestTypedFieldNamesAreNotSecret(t *testing.T) {+ seen := map[reflect.Type]bool{}+ var walk func(t reflect.Type, path string)+ walk = func(typ reflect.Type, path string) {+ for typ.Kind() == reflect.Pointer || typ.Kind() == reflect.Slice {+ typ = typ.Elem()+ }+ if typ.Kind() != reflect.Struct || seen[typ] {+ return+ }+ seen[typ] = true+ for i := range typ.NumField() {+ f := typ.Field(i)+ name, _, _ := strings.Cut(f.Tag.Get("json"), ",")+ if name == "-" {+ continue+ }+ k := f.Type.Kind()+ if (k == reflect.String || k == reflect.Slice || k == reflect.Interface) && IsSecretKeyName(name) {+ t.Errorf("%s.%s (%q) is named for a credential; sanitize_typed.go must check it", path, f.Name, name)+ }+ walk(f.Type, path+"."+f.Name)+ }+ }+ for _, v := range []any{MCPInfo{}, SystemContextInfo{}, GenAIInfo{}} {+ walk(reflect.TypeOf(v), reflect.TypeOf(v).Name())+ }+}++// The typed conversion copies every field of the types it converts: with values sanitizing leaves+// alone, the result is the input, but for the fields that never survive the round trip (json:"-")+// and the tool description, which SanitizeEvent fills. A field added to these types and not to+// sanitize_typed.go fails here.+func TestTypedConversionKeepsEveryField(t *testing.T) {+ var fill func(v reflect.Value)+ fill = func(v reflect.Value) {+ switch v.Kind() {+ case reflect.String:+ v.SetString("v")+ case reflect.Bool:+ v.SetBool(true)+ case reflect.Int, reflect.Int64:+ v.SetInt(7)+ case reflect.Float64:+ v.SetFloat(1.5)+ case reflect.Pointer:+ v.Set(reflect.New(v.Type().Elem()))+ fill(v.Elem())+ case reflect.Slice:+ v.Set(reflect.MakeSlice(v.Type(), 1, 1))+ fill(v.Index(0))+ case reflect.Interface:+ v.Set(reflect.ValueOf(map[string]any{"k": "v", "n": 1.5, "l": []any{"v", true, nil}}))+ case reflect.Struct:+ for i := range v.NumField() {+ fill(v.Field(i))+ }+ default:+ t.Fatalf("no value for a %s field", v.Kind())+ }+ }+ var mcp MCPInfo+ var context SystemContextInfo+ var genAI GenAIInfo+ for _, v := range []any{&mcp, &context, &genAI} {+ fill(reflect.ValueOf(v).Elem())+ }+ if got := sanitizeMCP(&mcp); !reflect.DeepEqual(*got, mcp) {+ t.Errorf("mcp = %+v, want %+v", *got, mcp)+ }+ if got := sanitizeSystemContext(&context); !reflect.DeepEqual(*got, context) {+ t.Errorf("system context = %+v, want %+v", *got, context)+ }+ want := genAI+ want.SystemInstructionsText = ""+ tool := *want.Tool+ tool.Description = ""+ call := *tool.Call+ call.ResultText = ""+ tool.Call, want.Tool = &call, &tool+ if got := sanitizeGenAI(&genAI); !reflect.DeepEqual(*got, want) {+ t.Errorf("gen_ai = %s\nwant %s", mustJSON(t, got), mustJSON(t, want))+ }+}++func mustJSON(t *testing.T, v any) string {+ data, err := json.Marshal(v)+ if err != nil {+ t.Fatal(err)+ }+ return string(data)+}++// ---- random events ----++func randomTestString(r *rand.Rand) string {+ pieces := []string{+ "token=abcdefgh12345678", `"api_key": "x y z"`, "--password hunter2", "Authorization: Bearer abc.def",+ "ghp_0123456789abcdefghijklmnopqrstuvwxyz", "postgres://user:pw@host/db", "input_tokens: 812",+ " ", "\n", "\t", `"`, "'", ",", "<&>", "\u2028", "\x01", "\xff", "é", "--api-key", "plain text",+ }+ var s strings.Builder+ for n := r.IntN(5); n >= 0; n-- {+ if r.IntN(10) == 0 {+ // Long text, around the limits.+ n := []int{100, 2040, 2048, 2060, 4090, 4100, 8200, 12300}[r.IntN(8)]+ s.WriteString(strings.Repeat("ab cd ", n/6+1)[:n])+ continue+ }+ s.WriteString(pieces[r.IntN(len(pieces))])+ }+ return s.String()+}++func randomTestKey(r *rand.Rand) string {+ keys := []string{"password", "api_key", "GITHUB_TOKEN", "Cookie", "max_tokens", "token_type", "name", "text", "args", "", "\xffkey"}+ return keys[r.IntN(len(keys))]+}++// randomGeneric returns a value of a shape an interface{} field or raw can hold: what JSON decodes+// to, and the Go values an in-process caller puts there.+func randomGeneric(r *rand.Rand, depth int) any {+ n := 16+ if depth > 3 {+ n = 9+ }+ switch r.IntN(n) {+ case 0:+ return nil+ case 1, 2, 3:+ return randomTestString(r)+ case 4:+ return r.IntN(2) == 0+ case 5:+ return []float64{0, math.Copysign(0, -1), 1.5, 1e21, 1e-7, 1 << 53, math.MaxFloat64, math.NaN(), math.Inf(1)}[r.IntN(9)]+ case 6:+ return []any{int(42), int64(math.MaxInt64), int64(math.MinInt64), int32(-7), uint64(math.MaxUint64), float32(0.1)}[r.IntN(6)]+ case 7:+ return []any{json.Number("12"), []byte("bytes"), struct{ N int }{3}, map[int]string{1: "x"}, []string{"--token", "t"}}[r.IntN(5)]+ case 8:+ return []any{[]any(nil), map[string]any(nil), []any{}, map[string]any{}}[r.IntN(4)]+ case 9, 10, 11, 12:+ out := make([]any, r.IntN(4))+ for i := range out {+ out[i] = randomGeneric(r, depth+1)+ }+ return out+ default:+ out := map[string]any{}+ for range r.IntN(4) {+ out[randomTestKey(r)] = randomGeneric(r, depth+1)+ }+ return out+ }+}++// fillRandom sets v, of any type an event holds, at random, leaving fields zero now and then.+func fillRandom(r *rand.Rand, v reflect.Value, depth int) {+ if depth > 0 && r.IntN(4) == 0 {+ return+ }+ switch v.Kind() {+ case reflect.String:+ v.SetString(randomTestString(r))+ case reflect.Bool:+ v.SetBool(r.IntN(2) == 0)+ case reflect.Int, reflect.Int64, reflect.Int32:+ v.SetInt([]int64{0, 1, -1, 1<<53 + 1, math.MaxInt64, math.MinInt64}[r.IntN(6)])+ case reflect.Uint, reflect.Uint64:+ v.SetUint([]uint64{0, 1, math.MaxUint64}[r.IntN(3)])+ case reflect.Float64:+ v.SetFloat([]float64{0, 1.5, 1e21, 1e-7, math.NaN()}[r.IntN(5)])+ case reflect.Pointer:+ p := reflect.New(v.Type().Elem())+ fillRandom(r, p.Elem(), depth+1)+ v.Set(p)+ case reflect.Interface:+ if g := randomGeneric(r, depth); g != nil {+ v.Set(reflect.ValueOf(g))+ }+ case reflect.Slice:+ s := reflect.MakeSlice(v.Type(), r.IntN(3), 3)+ for i := range s.Len() {+ fillRandom(r, s.Index(i), depth+1)+ }+ v.Set(s)+ case reflect.Map:+ if v.Type() == reflect.TypeFor[map[string]any]() {+ m := map[string]any{}+ for range r.IntN(5) {+ m[randomTestKey(r)] = randomGeneric(r, depth)+ }+ v.Set(reflect.ValueOf(m))+ }+ case reflect.Struct:+ for i := range v.NumField() {+ if v.Type().Field(i).IsExported() {+ fillRandom(r, v.Field(i), depth+1)+ }+ }+ }+}++func randomEvent(r *rand.Rand) Event {+ var e Event+ fillRandom(r, reflect.ValueOf(&e).Elem(), 0)+ if r.IntN(3) == 0 {+ // A large raw, near the size an event is marked truncated at.+ if e.Raw == nil {+ e.Raw = map[string]any{}+ }+ for i := range 28 + r.IntN(8) {+ e.Raw[fmt.Sprintf("k%d", i)] = strings.Repeat("<a>", 700+r.IntN(30))+ }+ }+ return e+}