The cross-harness, self-improving memory layer for AI agents. Join our community: https://discord.com/invite/zdNChS2fBu
Go agent-ai-cliagent-harnessesagent-securitybrowser-extensionciclaude-codecloudcodexcursordetection-engineeringjevknowledge-basememorymobile-device-managementobservabilitysecuritysecurity-information-and-event-managementsecurity-toolstelemetrytraces
End a quoted value only at its own quote; redact secret-named arrays
Claude9f6e1c2de525
diff --git a/pkg/asymptoteobserve/privacy.go b/pkg/asymptoteobserve/privacy.goindex bf164416..a7c71ed0 100644--- a/pkg/asymptoteobserve/privacy.go+++ b/pkg/asymptoteobserve/privacy.go@@ -32,10 +32,22 @@ const redactSeparator = `(?:\\?["'` + "`" + `])?\s*[:=]\s*`// redactOpenQuote opens a quoted value, escaped or not.const redactOpenQuote = `\\?["'` + "`" + `]`-// redactQuoted is a quoted value, up to its closing quote: spaces, commas and brackets inside the-// quotes are part of it. An escaped character is taken whole, except an escaped quote, which is how-// a value quoted inside a JSON string ends.-const redactQuoted = `(?:[^"'` + "`" + `\\\n]|\\[^"'` + "`" + `\n])+`+// redactQuotedBy is a value quoted with q, up to its own closing quote: spaces, commas, brackets+// and the other quote characters inside it are part of it (`"it's-mine"`). An escaped character is+// taken whole, except an escaped q, which is how a value quoted inside a JSON string ends.+func redactQuotedBy(q string) string {+ return `(?:[^` + q + `\\\n]|\\[^` + q + `\n])+`+}++// quotedAssignmentRule redacts a labeled value quoted with q: `token="x"`, `{"api_key": "x"}`.+func quotedAssignmentRule(q string) redactionRule {+ return redactionRule{regexp.MustCompile(`(?i)(\b` + redactLabel + redactSeparator + `\\?` + q + `)(?P<value>` + redactQuotedBy(q) + `)`), "${1}" + redactedValue, isAuthScheme}+}++// quotedFlagRule redacts a secret-named flag's value quoted with q: `--password "x y"`.+func quotedFlagRule(q string) redactionRule {+ return redactionRule{regexp.MustCompile(`(?i)(` + redactFlag + `\\?` + q + `)(` + redactQuotedBy(q) + `)`), "${1}" + redactedValue, nil}+}// redactUnquoted is an unquoted value, up to a quote, a comma or whitespace. Brackets, braces and// backslashes are part of it: a generated password is full of them. It never starts with a quote,@@ -97,7 +109,7 @@ var redactionRules = []redactionRule{// `export GITHUB_TOKEN="x"`, `{"api_key": "x"}`. The closing quote of a quoted label is allowed// between the label and the separator. A quoted value runs to its closing quote; the unquoted// rule never starts at a quote, so it leaves a value the quoted rule already replaced alone.- {regexp.MustCompile(`(?i)(\b` + redactLabel + redactSeparator + redactOpenQuote + `)(?P<value>` + redactQuoted + `)`), "${1}" + redactedValue, isAuthScheme},+ quotedAssignmentRule(`"`), quotedAssignmentRule(`'`), quotedAssignmentRule("`"),// A quoted label with a bare value is JSON (or a YAML or Python mapping) holding a number, a// literal or a structure. The replacement is quoted the way the label is, so the text still// parses where something decodes it again (string-encoded tool arguments do); null, true, false@@ -106,7 +118,7 @@ var redactionRules = []redactionRule{{regexp.MustCompile(`(?i)(\b` + redactLabel + `\s*[:=]\s*)(?P<value>` + redactUnquoted + `)`), "${1}" + redactedValue, isAuthScheme},// A command-line flag: `--password x`, `--token=x`, `--api-key x`, and quoted, the way agents// usually write arguments: `--password "x y"`, `--token='x'`.- {regexp.MustCompile(`(?i)(` + redactFlag + redactOpenQuote + `)(` + redactQuoted + `)`), "${1}" + redactedValue, nil},+ quotedFlagRule(`"`), quotedFlagRule(`'`), quotedFlagRule("`"),{regexp.MustCompile(`(?i)(` + redactFlag + `)([^\s"'-][^\s"']*)`), "${1}" + redactedValue, nil},// The password in a URL's userinfo: `postgres://user:pw@host`, `https://x-access-token:[email protected]`.{regexp.MustCompile(`(?i)(\b[a-z][a-z0-9+.-]*://[^/\s:@"'<>]+:)([^@/\s"'<>]+)(@)`), "${1}" + redactedValue + "${3}", nil},@@ -149,10 +161,11 @@ func (rule redactionRule) apply(value string) string {return string(append(out, value[last:]...))}-// redactAssignedPattern captures the value of a labeled assignment, quoted (${1}) or not (${2}), so-// a bare copy of it elsewhere in the same text is removed too.-var redactAssignedPattern = regexp.MustCompile(`(?i)\b` + redactLabel + redactSeparator +- `(?:` + redactOpenQuote + `(` + redactQuoted + `)|(` + redactUnquoted + `))`)+// redactAssignedPattern captures the value of a labeled assignment, quoted with ", ' or a backtick+// (${1} to ${3}) or not (${4}), so a bare copy of it elsewhere in the same text is+// removed too.+var redactAssignedPattern = regexp.MustCompile(`(?i)\b` + redactLabel + redactSeparator + `(?:` ++ `\\?"(` + redactQuotedBy(`"`) + `)|\\?'(` + redactQuotedBy(`'`) + `)|\\?` + "`(" + redactQuotedBy("`") + `)|(` + redactUnquoted + `))`)// redactWindow is how far past a string's limit redaction reads before the string is cut, so a// secret that straddles the cut is recognized whole rather than left as a recognizable prefix. It@@ -171,7 +184,7 @@ func RedactString(value string) string {// refers to `value` by itself.var assigned []stringfor _, match := range redactAssignedPattern.FindAllStringSubmatch(value, -1) {- copied := match[1] + match[2]+ copied := match[1] + match[2] + match[3] + match[4]if len(copied) >= 8 && copied != redactedValue && !isAuthScheme(copied) {assigned = append(assigned, copied)}@@ -263,6 +276,12 @@ func SanitizeMap(input map[string]interface{}, opts PrivacyOptions) map[string]icase map[string]interface{}:out[key] = SanitizeMap(typed, opts)case []interface{}:+ // A multi-valued header (`{"Cookie": ["sid=1"]}`, OTel's http.request.header.*) is as+ // much a credential as a single value.+ if opts.RedactSecrets && isSecretKeyName(key) {+ out[key] = redactAllStrings(typed, opts)+ continue+ }out[key] = SanitizeSlice(typed, opts)default:out[key] = typed@@ -271,6 +290,29 @@ func SanitizeMap(input map[string]interface{}, opts PrivacyOptions) map[string]ireturn out}+// redactAllStrings replaces every non-empty string in a slice held under a secret-named field,+// at any depth of nested slices; maps inside it are sanitized by their own field names.+func redactAllStrings(input []interface{}, opts PrivacyOptions) []interface{} {+ out := make([]interface{}, len(input))+ for index, value := range input {+ switch typed := value.(type) {+ case string:+ if typed == "" {+ out[index] = typed+ } else {+ out[index] = redactedValue+ }+ case []interface{}:+ out[index] = redactAllStrings(typed, opts)+ case map[string]interface{}:+ out[index] = SanitizeMap(typed, opts)+ default:+ out[index] = typed+ }+ }+ return out+}+func SanitizeSlice(input []interface{}, opts PrivacyOptions) []interface{} {limit := opts.StringLimitif limit <= 0 {diff --git a/pkg/asymptoteobserve/privacy_test.go b/pkg/asymptoteobserve/privacy_test.goindex 7abe47e2..c52ad357 100644--- a/pkg/asymptoteobserve/privacy_test.go+++ b/pkg/asymptoteobserve/privacy_test.go@@ -233,6 +233,15 @@ func TestSanitizeMapRedactsValuesUnderSecretKeys(t *testing.T) {t.Fatalf("SanitizeMap dropped %s: %s", keep, text)}}+ headers := SanitizeMap(map[string]interface{}{+ "Authorization": []interface{}{"Basic dXNlcjpwYXNz"},+ "Cookie": []interface{}{"sid=abcdef123", ""},+ "Accept": []interface{}{"application/json"},+ }, PrivacyOptions{RedactSecrets: true, StringLimit: DefaultRawStringLimit})+ if data, _ := json.Marshal(headers); strings.Contains(string(data), "dXNlcjpwYXNz") || strings.Contains(string(data), "abcdef123") ||+ !strings.Contains(string(data), "application/json") {+ t.Fatalf("multi-valued headers = %s, want credential values redacted and the rest kept", data)+ }if got := SanitizeMap(input, PrivacyOptions{StringLimit: DefaultRawStringLimit}); got["tool_input"].(map[string]interface{})["password"] != "hunter2" {t.Fatal("SanitizeMap redacted by key name with RedactSecrets off")}@@ -264,6 +273,10 @@ func TestRedactStringTakesWholeValues(t *testing.T) {{"quoted with spaces", `secret: "correct horse, battery staple"`, `secret: "[REDACTED]"`},{"single quoted symbols", `password='p]a}s\\s'`, `password='[REDACTED]'`},{"escaped json", `{\"token\": \"a]b}c d\"}`, `{\"token\": \"[REDACTED]\"}`},+ {"apostrophe in double quotes", `DB_PASSWORD="ab'cdEFGH12345" next`, `DB_PASSWORD="[REDACTED]" next`},+ {"apostrophe in json", `{"password": "it's-my-pass"}`, `{"password": "[REDACTED]"}`},+ {"double quote in single quotes", `password='say "hi" 123'`, `password='[REDACTED]'`},+ {"quoted flag with apostrophe", `tool --token "ab'cd ef"`, `tool --token "[REDACTED]"`},{"quoted flag", `docker login --password "hunter2 hunter2" registry`, `docker login --password "[REDACTED]" registry`},{"single quoted flag", `mysql --password 'p@ss' db`, `mysql --password '[REDACTED]' db`},{"quoted flag with equals", `tool --token="opaque-value" run`, `tool --token="[REDACTED]" run`},