The cross-harness, self-improving memory layer for AI agents. Join our community: https://discord.com/invite/zdNChS2fBu
Go agent-ai-cliagent-harnessesagent-securitybrowser-extensionciclaude-codecloudcodexcursordetection-engineeringjevknowledge-basememorymobile-device-managementobservabilitysecuritysecurity-information-and-event-managementsecurity-toolstelemetrytraces
perf(privacy): cut allocations and reflection out of SanitizeEvent (#1239)
kyle-elliott-asymptote9a664f9e0b0b
diff --git a/pkg/asymptoteobserve/privacy.go b/pkg/asymptoteobserve/privacy.goindex 362495c3..59bc94f0 100644--- a/pkg/asymptoteobserve/privacy.go+++ b/pkg/asymptoteobserve/privacy.go@@ -991,12 +991,18 @@ func IsSecretKeyName(key string) bool {// only credential word is "tokens", which is how a count of model tokens is named// (`max_tokens: 4096`, `input_tokens: "1200"`), where `GITHUB_TOKENS: abc` is a credential.func IsSecretField(key string, value interface{}) bool {- secret, onlyTokens := secretKeyName(key)- if !secret || onlyTokens && isNumber(value) {- return false+ if text, ok := value.(string); ok {+ return isSecretText(key, text)}- text, ok := value.(string)- return !ok || !isAuthScheme(text)+ secret, onlyTokens := secretKeyName(key)+ return secret && !(onlyTokens && isNumber(value))+}++// isSecretText is IsSecretField for a string value, which the sanitizers hold as a string: making+// it an interface{} to ask would allocate it again.+func isSecretText(key, text string) bool {+ secret, onlyTokens := secretKeyName(key)+ return secret && !(onlyTokens && isNumberText(text)) && !isAuthScheme(text)}// isNumber reports whether a value is a number, or a string that only spells one.@@ -1005,23 +1011,28 @@ func isNumber(value interface{}) bool {case int, int8, int16, int32, int64, uint, uint8, uint16, uint32, uint64, float32, float64, json.Number:return truecase string:- digits, point := 0, false- for i := range len(typed) {- switch c := typed[i]; {- case c >= '0' && c <= '9':- digits++- case c == '.' && !point:- point = true- case c == '-' && i == 0:- default:- return false- }- }- return digits > 0+ return isNumberText(typed)}return false}+// isNumberText reports whether a string only spells a number.+func isNumberText(text string) bool {+ digits, point := 0, false+ for i := range len(text) {+ switch c := text[i]; {+ case c >= '0' && c <= '9':+ digits+++ case c == '.' && !point:+ point = true+ case c == '-' && i == 0:+ default:+ return false+ }+ }+ return digits > 0+}+// IsSecretFlag reports whether an argument is a flag named for a secret written on its own, so the// next argument is its value: `--api-key`, `--token`, `-password`. SanitizeSlice redacts that next// argument, for callers that walk argument lists of their own.@@ -1038,8 +1049,10 @@ var redactSecretFlagPattern = lazyPattern(`(?i)^` + redactFlagName + `$`)// name that runs one into the words before it (`PGPASSWORD`, `githubtoken`) is marked by how it// ends. `author`, `keyboard` and `PWD` are not secrets.func secretKeyName(key string) (secret, onlyTokens bool) {- var name strings.Builder- name.Grow(len(key))+ // The lower-cased name is built in a buffer on the stack: this runs for every field of every+ // map SanitizeMap sees, and a heap-allocated builder was most of its cost on short names.+ var buf [64]byte+ name := buf[:0]start := 0// tokenPending is a "token" word not yet judged: followed by "count" or "usage" it names a// count (Codex and Gemini log `input_token_count`, `total_token_count`), like "tokens", and a@@ -1048,38 +1061,38 @@ func secretKeyName(key string) (secret, onlyTokens bool) {// endWord reports whether the word that ends here names a credential other than a token count,// and starts the next one.endWord := func() bool {- word := name.String()[start:]- start = name.Len()+ word := name[start:]+ start = len(name)if tokenPending {tokenPending = false- if slices.Contains(tokenCountWords, word) {+ if containsWord(tokenCountWords, word) {onlyTokens = truereturn false}return true}- if word == "tokens" {+ if string(word) == "tokens" {onlyTokens = truereturn false}- if word == "token" {+ if string(word) == "token" {tokenPending = truereturn false}- return slices.Contains(redactKeyWords, word)+ return containsWord(redactKeyWords, word)}for i := range len(key) {c := key[i]switch {case c >= 'a' && c <= 'z' || c >= '0' && c <= '9':- name.WriteByte(c)+ name = append(name, c)case c >= 'A' && c <= 'Z':// A capital starts a word after a lower-case letter or a digit (`apiKey`), and after// other capitals when a lower-case letter follows it (`APIKey`).if i > 0 && (isLowerOrDigit(key[i-1]) || isUpper(key[i-1]) && i+1 < len(key) && key[i+1] >= 'a' && key[i+1] <= 'z') && endWord() {return true, false}- name.WriteByte(c + 'a' - 'A')+ name = append(name, c+'a'-'A')default:if endWord() {return true, false@@ -1089,15 +1102,24 @@ func secretKeyName(key string) (secret, onlyTokens bool) {if endWord() || tokenPending {return true, false}- whole := name.String()for _, suffix := range redactKeySuffixes {- if strings.HasSuffix(whole, suffix) {+ if len(name) >= len(suffix) && string(name[len(name)-len(suffix):]) == suffix {return true, false}}return onlyTokens, onlyTokens}+// containsWord reports whether word is one of words, without converting it to a string.+func containsWord(words []string, word []byte) bool {+ for _, w := range words {+ if string(word) == w {+ return true+ }+ }+ return false+}+func isLowerOrDigit(c byte) bool { return c >= 'a' && c <= 'z' || c >= '0' && c <= '9' }func isUpper(c byte) bool { return c >= 'A' && c <= 'Z' }@@ -1208,7 +1230,7 @@ func sanitizeField(key string, value interface{}, opts PrivacyOptions, limit intcase string:// A value whose field is named for a credential is one, whatever it looks like:// `{"password": "hunter2"}` has no shape a pattern could recognize.- if opts.RedactSecrets && typed != "" && IsSecretField(key, typed) {+ if opts.RedactSecrets && typed != "" && isSecretText(key, typed) {return redactedValue}return CleanString(typed, limit, opts.RedactSecrets)diff --git a/pkg/asymptoteobserve/sanitize_typed.go b/pkg/asymptoteobserve/sanitize_typed.goindex 79b38250..376e6f2a 100644--- a/pkg/asymptoteobserve/sanitize_typed.go+++ b/pkg/asymptoteobserve/sanitize_typed.go@@ -163,11 +163,29 @@ func (g genericSanitizer) strings(in []string) []string {// field is the value of an interface{} field, none of which is named for a credential: decoded as// the round trip decodes it, then sanitized as SanitizeMap sanitizes a value under such a name. A// value nested deeper than maxGenericDepth (or holding a cycle) is marshaled and decoded whole.+//+// A value already in the shape decoding gives (isDecoded), as a payload decoded from JSON is, is+// sanitized as it is: decoding it would only copy every map, list and string in it first.func (g genericSanitizer) field(v any) any {+ if !isDecoded(v, 0) {+ v = decodedField(v)+ }+ return g.sanitizeDecoded(v)+}++// decodedField is v as the round trip decodes it, a value nested too deep marshaled and decoded+// whole.+func decodedField(v any) any {d := decoded(v, 0)if _, deep := d.(tooDeep); deep {- d = roundTrip(v)+ return roundTrip(v)}+ return d+}++// sanitizeDecoded sanitizes a decoded value as SanitizeMap sanitizes a value under a name that+// is not a credential's.+func (g genericSanitizer) sanitizeDecoded(d any) any {switch x := d.(type) {case string:return CleanString(x, g.opts.StringLimit, g.opts.RedactSecrets)@@ -180,6 +198,46 @@ func (g genericSanitizer) field(v any) any {}}+// isDecoded reports whether decoded(v, depth) returns v unchanged: v holds only what decoding JSON+// gives (nil, a bool, a float64 JSON can write, a string of valid UTF-8, and a non-nil []any or+// map[string]any of those with keys of valid UTF-8), nested no deeper than maxGenericDepth. It is+// decoded's own conditions, read without building anything. TestFieldIsDecodedThenSanitized holds+// the two to the same answer.+func isDecoded(v any, depth int) bool {+ if depth > maxGenericDepth {+ return false+ }+ switch x := v.(type) {+ case nil, bool:+ return true+ case string:+ return utf8.ValidString(x)+ case float64:+ return !math.IsNaN(x) && !math.IsInf(x, 0)+ case []any:+ if x == nil {+ return false+ }+ for _, item := range x {+ if !isDecoded(item, depth+1) {+ return false+ }+ }+ return true+ case map[string]any:+ if x == nil {+ return false+ }+ for k, item := range x {+ if !utf8.ValidString(k) || !isDecoded(item, depth+1) {+ return false+ }+ }+ return true+ }+ return false+}+// tooDeep is what decoded returns for a value nested deeper than maxGenericDepth, and for any// value holding one.type tooDeep struct{}diff --git a/pkg/asymptoteobserve/sanitize_typed_test.go b/pkg/asymptoteobserve/sanitize_typed_test.goindex eaf48a57..50a56297 100644--- a/pkg/asymptoteobserve/sanitize_typed_test.go+++ b/pkg/asymptoteobserve/sanitize_typed_test.go@@ -265,6 +265,76 @@ func TestJSONSizeBoundIsABound(t *testing.T) {}}+// field sanitizes a value already in decoded shape as it is, and decodes anything else first.+// isDecoded must hold exactly when decoding leaves the value as it was, and field must give what+// decoding then sanitizing gives, for every shape: random values of every kind a field holds,+// secret-named keys and flags, keys and strings that are not valid UTF-8, NaN, nil and empty+// containers, a cycle, and nesting on either side of maxGenericDepth.+func TestFieldIsDecodedThenSanitized(t *testing.T) {+ nested := func(depth int, leaf any, asMap bool) any {+ v := leaf+ for range depth {+ if asMap {+ v = map[string]any{"k": v}+ } else {+ v = []any{"--token", v}+ }+ }+ return v+ }+ cycle := map[string]any{"a": "x"}+ cycle["self"] = cycle+ values := []any{+ nil, true, 1.5, math.NaN(), math.Inf(-1), float32(0.1), 42, json.Number("12"), []byte("b"),+ "plain", "token=abcdefgh12345678", "bad \xff utf-8",+ []any(nil), map[string]any(nil), []any{}, map[string]any{}, []string{"--token", "t"}, map[string]string{"a": "b"},+ map[string]any{"\xffkey": "x", "a": "token=abcdefgh12345678"},+ map[string]any{"a": map[string]any{"\xffkey": "x"}},+ []any{"--api-key", "secret", "--api-key", map[string]any{"a": 1.0}, json.Number("1"), "x"},+ []any{[]string{"--token", "t"}, "v", "--token", []byte("b")},+ map[string]any{"password": []any{"a", []any{"b"}, map[string]any{"c": "d"}}, "args": []any{"--password", "p"}},+ map[string]any{"max_tokens": "1200", "token_type": "bearer", "GITHUB_TOKENS": "abc", "n": 3.0},+ cycle,+ }+ for _, depth := range []int{maxGenericDepth - 2, maxGenericDepth - 1, maxGenericDepth, maxGenericDepth + 1, maxGenericDepth + 5} {+ for _, asMap := range []bool{false, true} {+ values = append(values, nested(depth, "token=abcdefgh12345678", asMap), nested(depth, 3.0, asMap), nested(depth, []any{}, asMap))+ }+ }+ r := rand.New(rand.NewPCG(7, 8))+ for range 20000 {+ values = append(values, randomGeneric(r, 0))+ }+ asIs := 0+ for i, v := range values {+ if got, want := isDecoded(v, 0), reflect.DeepEqual(decoded(v, 0), v); got != want {+ t.Fatalf("value %d (%#v): isDecoded %v, decoding leaves it as it was %v", i, v, got, want)+ }+ if isDecoded(v, 0) {+ asIs+++ }+ }+ compared := 0+ for _, opts := range []PrivacyOptions{+ {RedactSecrets: true, StringLimit: DefaultRawStringLimit},+ {RedactSecrets: true, StringLimit: 40},+ {RedactSecrets: false, StringLimit: DefaultStringLimit},+ } {+ g := newGenericSanitizer(opts)+ for i, v := range values {+ got, want := g.field(v), g.sanitizeDecoded(decodedField(v))+ if !reflect.DeepEqual(got, want) {+ t.Fatalf("value %d (%#v), %+v:\n got %#v\nwant %#v", i, v, opts, got, want)+ }+ compared+++ }+ }+ if asIs < 1000 || asIs == len(values) {+ t.Fatalf("%d of %d values in decoded shape; want both kinds", asIs, len(values))+ }+ t.Logf("%d values sanitized alike (%d of %d in decoded shape)", compared, asIs, len(values))+}+// The typed conversion sanitizes typed string and list fields without asking whether their name// marks them as a credential, which holds only while no such field has a name that does.func TestTypedFieldNamesAreNotSecret(t *testing.T) {