Skip to content

The cross-harness, self-improving memory layer for AI agents. Join our community: https://discord.com/invite/zdNChS2fBu

Redact whole values and only real private key blocks

Claude454bb277e125
diff --git a/pkg/asymptoteobserve/privacy.go b/pkg/asymptoteobserve/privacy.go
index 77840bd0..4e3ba928 100644
--- a/pkg/asymptoteobserve/privacy.go
+++ b/pkg/asymptoteobserve/privacy.go
@@ -16,51 +16,72 @@ const (
// says what was there.
const redactedValue = "[REDACTED]"
-// secretKeyName is the part of a label that marks the value after it as a secret: `token`,
+// The redaction tables below are named redact*, not secret*: static analysis treats a variable
+// named for a secret as sensitive data, and these hold patterns, not credentials.
+
+// redactLabel is the part of a label that marks the value after it as a secret: `token`,
// `GITHUB_TOKEN`, `api_key`, `AWS_SECRET_ACCESS_KEY`, `"password"`. The label must end in the
// marker, so `input_tokens: 812` and `max_tokens=4096` stay readable.
-const secretKeyName = `[A-Za-z0-9_.-]*?(?:api[_-]?key|access[_-]?key|secret[_-]?key|private[_-]?key|client[_-]?secret|` +
+const redactLabel = `[A-Za-z0-9_.-]*?(?:api[_-]?key|access[_-]?key|secret[_-]?key|private[_-]?key|client[_-]?secret|` +
`token|secret|password|passwd|passphrase|authorization|credentials?)`
-// secretAssignment runs from the end of a label to the start of its value: an optional closing quote
-// (escaped too, for JSON quoted inside a string), the separator, and an optional opening quote.
-const secretAssignment = "(?:\\\\?[\"'`])?\\s*[:=]\\s*(?:\\\\?[\"'`])?"
+// redactSeparator runs from the end of a label to its value: an optional closing quote (escaped
+// too, for JSON quoted inside a string) and the separator.
+const redactSeparator = `(?:\\?["'` + "`" + `])?\s*[:=]\s*`
+
+// redactOpenQuote opens a quoted value, escaped or not.
+const redactOpenQuote = `\\?["'` + "`" + `]`
-// secretValue is the value itself, up to a quote, an escape, a separator or the end of a structure.
-// It never starts with `[`, so a value already replaced with [REDACTED] is left as it is and
-// redacting twice gives the same text as redacting once.
-const secretValue = "[^\"'`,\\s}\\]\\\\\\[][^\"'`,\\s}\\]\\\\]*"
+// redactQuoted is a quoted value, up to its closing quote: spaces, commas and brackets inside the
+// quotes are part of it. An escaped character is taken whole, except an escaped quote, which is how
+// a value quoted inside a JSON string ends.
+const redactQuoted = `(?:[^"'` + "`" + `\\\n]|\\[^"'` + "`" + `\n])+`
-// secretRule is one redaction. ${1} and ${3}, when the pattern has them, are kept around the value.
-type secretRule struct {
+// redactUnquoted is an unquoted value, up to a quote, a comma or whitespace. Brackets, braces and
+// backslashes are part of it: a generated password is full of them. It never starts with a quote,
+// escaped or not, so it leaves a value the quoted rule already replaced alone.
+const redactUnquoted = `(?:[^"'` + "`" + `,\s\\]|\\[^"'` + "`" + `])[^"'` + "`" + `,\s]*`
+
+// redactionRule is one redaction. ${1} and ${3}, when the pattern has them, are kept around the
+// value.
+type redactionRule struct {
pattern *regexp.Regexp
replacement string
// keep, when set, leaves a match alone if its value (${2}) passes.
keep func(value string) bool
}
-// isAuthScheme reports whether a labeled value is an Authorization scheme rather than a
-// credential. `Authorization: Bearer x` is redacted to `Authorization: Bearer [REDACTED]` by the
-// header rule, and the assignment rule must not then take `Bearer` for the value and erase what
-// kind of credential the line held.
+// isAuthScheme reports whether a labeled value is an Authorization scheme, alone or with its
+// credential already redacted, rather than a credential. `Authorization: Bearer x` is redacted to
+// `Authorization: Bearer [REDACTED]` by the header rule, and the assignment rules must not then
+// take the scheme for the value and erase what kind of credential the line held.
func isAuthScheme(value string) bool {
- switch strings.ToLower(value) {
+ fields := strings.Fields(value)
+ if len(fields) == 0 || len(fields) > 2 || (len(fields) == 2 && fields[1] != redactedValue) {
+ return false
+ }
+ switch strings.ToLower(fields[0]) {
case "bearer", "basic", "token", "digest":
return true
}
return false
}
-var secretRules = []secretRule{
- // A private key block, or the rest of the text when the block was cut before its END line.
- {regexp.MustCompile(`-----BEGIN [A-Z0-9 ]*PRIVATE KEY( BLOCK)?-----(?:[\s\S]*?-----END [A-Z0-9 ]*PRIVATE KEY( BLOCK)?-----|[\s\S]*$)`), redactedValue, nil},
+var redactionRules = []redactionRule{
+ // A private key block. The header must be followed by key material, so source code, a diff or
+ // grep output that only names the header keeps the text after it. Without an END line (the
+ // block was cut) the rest of the text goes.
+ {regexp.MustCompile(`-----BEGIN [A-Z0-9 ]*PRIVATE KEY( BLOCK)?-----(?:\s|\\[rn]|(?:Proc-Type|DEK-Info|Comment|Version):[^\n\\]*)*` +
+ `[A-Za-z0-9+/=]{16,}[\s\S]*?(?:-----END [A-Z0-9 ]*PRIVATE KEY( BLOCK)?-----|$)`), redactedValue, nil},
// An Authorization header's credential after its scheme: Bearer, Basic, Token, Digest.
{regexp.MustCompile(`(?i)(authorization["']?\s*[:=]\s*["']?(?:bearer|basic|token|digest)\s+)([^"'\s,]+)()`), "${1}" + redactedValue, nil},
{regexp.MustCompile(`(?i)(\bbearer\s+)([a-z0-9._~+/=-]+)()`), "${1}" + redactedValue, nil},
// A labeled assignment in prose, shell, env files, YAML, TOML or JSON: `token=x`,
// `export GITHUB_TOKEN="x"`, `{"api_key": "x"}`. The closing quote of a quoted label is allowed
- // between the label and the separator.
- {regexp.MustCompile(`(?i)(\b` + secretKeyName + secretAssignment + `)(` + secretValue + `)()`), "${1}" + redactedValue, isAuthScheme},
+ // between the label and the separator. A quoted value runs to its closing quote; the unquoted
+ // rule never starts at a quote, so it leaves a value the quoted rule already replaced alone.
+ {regexp.MustCompile(`(?i)(\b` + redactLabel + redactSeparator + redactOpenQuote + `)(` + redactQuoted + `)()`), "${1}" + redactedValue, isAuthScheme},
+ {regexp.MustCompile(`(?i)(\b` + redactLabel + redactSeparator + `)(` + redactUnquoted + `)()`), "${1}" + redactedValue, isAuthScheme},
// A command-line flag: `--password x`, `--token=x`, `--api-key x`.
{regexp.MustCompile(`(?i)((?:^|\s)--?[a-z0-9-]*(?:password|passwd|passphrase|token|secret|api-?key|access-?key)(?:=|\s+))([^\s"'-][^\s"']*)()`), "${1}" + redactedValue, nil},
// The password in a URL's userinfo: `postgres://user:pw@host`, `https://x-access-token:[email protected]`.
@@ -83,7 +104,7 @@ var secretRules = []secretRule{
{regexp.MustCompile(`\b(eyJ)[A-Za-z0-9_-]{8,}\.eyJ[A-Za-z0-9_-]{8,}\.[A-Za-z0-9_-]{8,}`), "${1}" + redactedValue, nil}, // JWT
}
-func (rule secretRule) apply(value string) string {
+func (rule redactionRule) apply(value string) string {
if rule.keep == nil {
return rule.pattern.ReplaceAllString(value, rule.replacement)
}
@@ -103,9 +124,10 @@ func (rule secretRule) apply(value string) string {
return string(append(out, value[last:]...))
}
-// assignedSecretPattern captures the value of a labeled assignment so a bare copy of it elsewhere in
-// the same text is removed too.
-var assignedSecretPattern = regexp.MustCompile(`(?i)\b` + secretKeyName + secretAssignment + `(` + secretValue + `)`)
+// redactAssignedPattern captures the value of a labeled assignment, quoted (${1}) or not (${2}), so
+// a bare copy of it elsewhere in the same text is removed too.
+var redactAssignedPattern = regexp.MustCompile(`(?i)\b` + redactLabel + redactSeparator +
+ `(?:` + redactOpenQuote + `(` + redactQuoted + `)|(` + redactUnquoted + `))`)
// redactWindow is how far past a string's limit redaction reads before the string is cut, so a
// secret that straddles the cut is recognized whole rather than left as a recognizable prefix. It
@@ -123,12 +145,13 @@ func RedactString(value string) string {
// This is common in model reasoning that quotes `token=value` and later
// refers to `value` by itself.
var assigned []string
- for _, match := range assignedSecretPattern.FindAllStringSubmatch(value, -1) {
- if len(match) > 1 && len(match[1]) >= 8 && match[1] != redactedValue {
- assigned = append(assigned, match[1])
+ for _, match := range redactAssignedPattern.FindAllStringSubmatch(value, -1) {
+ copied := match[1] + match[2]
+ if len(copied) >= 8 && copied != redactedValue && !isAuthScheme(copied) {
+ assigned = append(assigned, copied)
}
}
- for _, rule := range secretRules {
+ for _, rule := range redactionRules {
value = rule.apply(value)
}
// Each copy is searched for and removed; none of it survives into the result.
@@ -150,7 +173,7 @@ func isSecretKeyName(key string) bool {
}
}
name := normalized.String()
- for _, suffix := range secretKeySuffixes {
+ for _, suffix := range redactKeySuffixes {
if strings.HasSuffix(name, suffix) {
return true
}
@@ -158,7 +181,7 @@ func isSecretKeyName(key string) bool {
return false
}
-var secretKeySuffixes = []string{
+var redactKeySuffixes = []string{
"password", "passwd", "passphrase", "secret", "token", "apikey", "accesskey", "secretkey",
"privatekey", "authorization", "credential", "credentials", "cookie",
}
diff --git a/pkg/asymptoteobserve/privacy_test.go b/pkg/asymptoteobserve/privacy_test.go
index 87b07c90..50d8c56d 100644
--- a/pkg/asymptoteobserve/privacy_test.go
+++ b/pkg/asymptoteobserve/privacy_test.go
@@ -251,3 +251,46 @@ func TestCleanStringRedactsBeforeTruncating(t *testing.T) {
t.Fatalf("CleanString kept key material or overran its limit: %d bytes", len(got))
}
}
+
+// A generated password is full of brackets, braces and backslashes, and a quoted one can hold
+// spaces and commas. None of it may survive past the first such character.
+func TestRedactStringTakesWholeValues(t *testing.T) {
+ tests := []struct {
+ name, input, want string
+ }{
+ {"unquoted symbols", "DB_PASSWORD=Xy7]kL9}pQ next", "DB_PASSWORD=[REDACTED] next"},
+ {"unquoted backslash", `password=ab\cdEFgh`, "password=[REDACTED]"},
+ {"leading bracket", "token=[abcDEF123]", "token=[REDACTED]"},
+ {"quoted with spaces", `secret: "correct horse, battery staple"`, `secret: "[REDACTED]"`},
+ {"single quoted symbols", `password='p]a}s\\s'`, `password='[REDACTED]'`},
+ {"escaped json", `{\"token\": \"a]b}c d\"}`, `{\"token\": \"[REDACTED]\"}`},
+ }
+ for _, tc := range tests {
+ t.Run(tc.name, func(t *testing.T) {
+ if got := RedactString(tc.input); got != tc.want {
+ t.Fatalf("RedactString(%q) = %q, want %q", tc.input, got, tc.want)
+ }
+ })
+ }
+}
+
+// Source code, a diff or grep output that names a private key header is not a key, and the text
+// after it stays. A key on one line, as an env file holds it, is still redacted.
+func TestRedactStringPrivateKeyNeedsKeyMaterial(t *testing.T) {
+ code := `if strings.HasPrefix(s, "-----BEGIN RSA PRIVATE KEY-----") {` + "\n\treturn parseRSA(s)\n}"
+ if got := RedactString(code); got != code {
+ t.Fatalf("RedactString changed code that only names a key header:\n%q", got)
+ }
+ grep := "keys.go:12: // -----BEGIN OPENSSH PRIVATE KEY----- marks an OpenSSH key\nkeys.go:13: func parse() {}"
+ if got := RedactString(grep); got != grep {
+ t.Fatalf("RedactString changed grep output that only names a key header:\n%q", got)
+ }
+ escaped := `{"output":"-----BEGIN PRIVATE KEY-----\nMIIEvQIBADANBgkqhkiG9w0BAQEFAASC\n-----END PRIVATE KEY-----\n","ok":true}`
+ if got := RedactString(escaped); strings.Contains(got, "MIIEvQIBADANBgkq") || !strings.Contains(got, `"ok":true`) {
+ t.Fatalf("key quoted in a JSON string = %q, want the key gone and the rest kept", got)
+ }
+ oneLine := `PRIVATE_KEY="-----BEGIN PRIVATE KEY-----MIIEvQIBADANBgkqhkiG9w0BAQEFAASC-----END PRIVATE KEY-----"`
+ if got := RedactString(oneLine); strings.Contains(got, "MIIEvQIBADANBgkq") {
+ t.Fatalf("one-line key = %q, want it redacted", got)
+ }
+}