Skip to content

The cross-harness, self-improving memory layer for AI agents. Join our community: https://discord.com/invite/zdNChS2fBu

Keep escaped quotes inside a quoted value

Claudef83d6451fc46
diff --git a/pkg/asymptoteobserve/privacy.go b/pkg/asymptoteobserve/privacy.go
index 428d7c1d..b0960a0a 100644
--- a/pkg/asymptoteobserve/privacy.go
+++ b/pkg/asymptoteobserve/privacy.go
@@ -32,21 +32,36 @@ const redactSeparator = `(?:\\?["'` + "`" + `])?\s*[:=]\s*`
// redactOpenQuote opens a quoted value, escaped or not.
const redactOpenQuote = `\\?["'` + "`" + `]`
-// redactQuotedBy is a value quoted with q, up to its own closing quote: spaces, commas, brackets
-// and the other quote characters inside it are part of it (`"it's-mine"`). An escaped character is
-// taken whole, except an escaped q, which is how a value quoted inside a JSON string ends.
-func redactQuotedBy(q string) string {
- return `(?:[^` + q + `\\\n]|\\[^` + q + `\n])+`
+// redactInQuotes is a value quoted with q, up to its own closing quote: spaces, commas, brackets,
+// the other quote characters and escaped characters, an escaped q among them, are part of it
+// (`"it's-mine"`, `"ab\"cd"`).
+func redactInQuotes(q string) string {
+ return `(?:[^` + q + `\\\n]|\\[^\n])+`
}
-// quotedAssignmentRule redacts a labeled value quoted with q: `token="x"`, `{"api_key": "x"}`.
-func quotedAssignmentRule(q string) redactionRule {
- return redactionRule{regexp.MustCompile(`(?i)(\b` + redactLabel + redactSeparator + `\\?` + q + `)(?P<value>` + redactQuotedBy(q) + `)`), "${1}" + redactedValue, isAuthScheme}
+// redactInEscapedQuotes is a value whose opening quote was itself escaped (`\"`), which means it
+// sits inside a JSON string: there the escaped q is its closing quote, and any other escaped
+// character is part of it, a quote escaped twice (`\\\"`, the value's own quote) among them.
+func redactInEscapedQuotes(q string) string {
+ return `(?:[^` + q + `\\\n]|\\\\\\` + q + `|\\[^` + q + `\n])+`
}
-// quotedFlagRule redacts a secret-named flag's value quoted with q: `--password "x y"`.
-func quotedFlagRule(q string) redactionRule {
- return redactionRule{regexp.MustCompile(`(?i)(` + redactFlag + `\\?` + q + `)(` + redactQuotedBy(q) + `)`), "${1}" + redactedValue, nil}
+// quotedValues pairs each way a value can be opened with the body it runs to: an escaped and a
+// plain opening for each quote character. Go has no backreferences, so each pair is its own rule.
+var quotedValues = [][2]string{
+ {`\\"`, redactInEscapedQuotes(`"`)}, {`"`, redactInQuotes(`"`)},
+ {`\\'`, redactInEscapedQuotes(`'`)}, {`'`, redactInQuotes(`'`)},
+ {"\\\\`", redactInEscapedQuotes("`")}, {"`", redactInQuotes("`")},
+}
+
+// quotedRules builds one rule per quotedValues entry from prefix, the pattern before the opening
+// quote.
+func quotedRules(prefix string, keep func(string) bool) []redactionRule {
+ rules := make([]redactionRule, 0, len(quotedValues))
+ for _, quoted := range quotedValues {
+ rules = append(rules, redactionRule{regexp.MustCompile(`(?i)(` + prefix + quoted[0] + `)(?P<value>` + quoted[1] + `)`), "${1}" + redactedValue, keep})
+ }
+ return rules
}
// redactUnquoted is an unquoted value, up to a quote, a comma or whitespace. Brackets, braces and
@@ -102,7 +117,7 @@ func isLiteralOrAuthScheme(value string) bool {
return isAuthScheme(value)
}
-var redactionRules = []redactionRule{
+var redactionRules = concatRules([]redactionRule{
// A private key block. The header must be followed by key material, so source code, a diff or
// grep output that only names the header keeps the text after it. Without an END line (the
// block was cut) the rest of the text goes.
@@ -118,7 +133,7 @@ var redactionRules = []redactionRule{
// `export GITHUB_TOKEN="x"`, `{"api_key": "x"}`. The closing quote of a quoted label is allowed
// between the label and the separator. A quoted value runs to its closing quote; the unquoted
// rule never starts at a quote, so it leaves a value the quoted rule already replaced alone.
- quotedAssignmentRule(`"`), quotedAssignmentRule(`'`), quotedAssignmentRule("`"),
+}, quotedRules(`\b`+redactLabel+redactSeparator, isAuthScheme), []redactionRule{
// A quoted label with a bare value is JSON (or a YAML or Python mapping) holding a number, a
// literal or a structure. The replacement is quoted the way the label is, so the text still
// parses where something decodes it again (string-encoded tool arguments do); null, true, false
@@ -127,7 +142,7 @@ var redactionRules = []redactionRule{
{regexp.MustCompile(`(?i)(\b` + redactLabel + `\s*[:=]\s*)(?P<value>` + redactUnquoted + `)`), "${1}" + redactedValue, isAuthScheme},
// A command-line flag: `--password x`, `--token=x`, `--api-key x`, and quoted, the way agents
// usually write arguments: `--password "x y"`, `--token='x'`.
- quotedFlagRule(`"`), quotedFlagRule(`'`), quotedFlagRule("`"),
+}, quotedRules(redactFlag, nil), []redactionRule{
{regexp.MustCompile(`(?i)(` + redactFlag + `)([^\s"'-][^\s"']*)`), "${1}" + redactedValue, nil},
// The password in a URL's userinfo: `postgres://user:pw@host`, `https://x-access-token:[email protected]`.
{regexp.MustCompile(`(?i)(\b[a-z][a-z0-9+.-]*://[^/\s:@"'<>]+:)([^@/\s"'<>]+)(@)`), "${1}" + redactedValue + "${3}", nil},
@@ -147,6 +162,14 @@ var redactionRules = []redactionRule{
{regexp.MustCompile(`\b(hf_)[A-Za-z0-9]{30,}`), "${1}" + redactedValue, nil}, // Hugging Face
{regexp.MustCompile(`\b(bcn_(?:api|device|mcp)_)[A-Za-z0-9_-]{8,}`), "${1}" + redactedValue, nil}, // Beacon's own keys
{regexp.MustCompile(`\b(eyJ)[A-Za-z0-9_-]{8,}\.eyJ[A-Za-z0-9_-]{8,}\.[A-Za-z0-9_-]{8,}`), "${1}" + redactedValue, nil}, // JWT
+})
+
+func concatRules(groups ...[]redactionRule) []redactionRule {
+ var all []redactionRule
+ for _, group := range groups {
+ all = append(all, group...)
+ }
+ return all
}
func (rule redactionRule) apply(value string) string {
@@ -177,8 +200,16 @@ func (rule redactionRule) apply(value string) string {
// redactAssignedPattern captures the value of a labeled assignment, quoted with ", ' or a backtick
// (${1} to ${3}) or not (${4}), so a bare copy of it elsewhere in the same text is
// removed too.
-var redactAssignedPattern = regexp.MustCompile(`(?i)\b` + redactLabel + redactSeparator + `(?:` +
- `\\?"(` + redactQuotedBy(`"`) + `)|\\?'(` + redactQuotedBy(`'`) + `)|\\?` + "`(" + redactQuotedBy("`") + `)|(` + redactUnquoted + `))`)
+var redactAssignedPattern = regexp.MustCompile(`(?i)\b` + redactLabel + redactSeparator + `(?:` + quotedAlternatives() + `|(` + redactUnquoted + `))`)
+
+// quotedAlternatives is every quotedValues entry as one capturing alternative.
+func quotedAlternatives() string {
+ parts := make([]string, 0, len(quotedValues))
+ for _, quoted := range quotedValues {
+ parts = append(parts, quoted[0]+`(`+quoted[1]+`)`)
+ }
+ return strings.Join(parts, "|")
+}
// redactWindow is how far past a string's limit redaction reads before the string is cut, so a
// secret that straddles the cut is recognized whole rather than left as a recognizable prefix. It
@@ -197,7 +228,7 @@ func RedactString(value string) string {
// refers to `value` by itself.
var assigned []string
for _, match := range redactAssignedPattern.FindAllStringSubmatch(value, -1) {
- copied := match[1] + match[2] + match[3] + match[4]
+ copied := strings.Join(match[1:], "")
if len(copied) >= 8 && copied != redactedValue && !isAuthScheme(copied) {
assigned = append(assigned, copied)
}
diff --git a/pkg/asymptoteobserve/privacy_test.go b/pkg/asymptoteobserve/privacy_test.go
index 646e6faf..b7dd885b 100644
--- a/pkg/asymptoteobserve/privacy_test.go
+++ b/pkg/asymptoteobserve/privacy_test.go
@@ -275,6 +275,9 @@ func TestRedactStringTakesWholeValues(t *testing.T) {
{"quoted with spaces", `secret: "correct horse, battery staple"`, `secret: "[REDACTED]"`},
{"single quoted symbols", `password='p]a}s\\s'`, `password='[REDACTED]'`},
{"escaped json", `{\"token\": \"a]b}c d\"}`, `{\"token\": \"[REDACTED]\"}`},
+ {"escaped quote in shell value", `export TOKEN="ab\"cdefgh1234" next`, `export TOKEN="[REDACTED]" next`},
+ {"escaped quote in json value", `{"password": "pa\"ss123secret"}`, `{"password": "[REDACTED]"}`},
+ {"escaped quote in json string", `{\"password\": \"pa\\\"ss\"}`, `{\"password\": \"[REDACTED]\"}`},
{"apostrophe in double quotes", `DB_PASSWORD="ab'cdEFGH12345" next`, `DB_PASSWORD="[REDACTED]" next`},
{"apostrophe in json", `{"password": "it's-my-pass"}`, `{"password": "[REDACTED]"}`},
{"double quote in single quotes", `password='say "hi" 123'`, `password='[REDACTED]'`},