Skip to content

The cross-harness, self-improving memory layer for AI agents. Join our community: https://discord.com/invite/zdNChS2fBu

Redact common credential formats and secret-named fields

Claude3b8fd10e95d6
diff --git a/docs/security/retention-redaction.mdx b/docs/security/retention-redaction.mdx
index ccf4faed..17539038 100644
--- a/docs/security/retention-redaction.mdx
+++ b/docs/security/retention-redaction.mdx
@@ -11,11 +11,26 @@ Beacon can write supported prompt, command, tool, file, approval, policy, and ru
| Control | Behavior |
|---------|----------|
-| Redaction | Common secret patterns are replaced before retained content is written |
+| Redaction | Common secret patterns are replaced before retained content is written (see [What is redacted](#what-is-redacted)) |
| Sanitization | Runtime payloads are normalized into typed event fields instead of storing source payloads verbatim wherever possible |
| Truncation | Oversized fields are shortened and marked with truncation metadata |
| Event-size limits | Events are bounded before they are written to `runtime.jsonl` or sent to configured destinations |
+## What is redacted
+
+Redaction runs on retained text (prompts, command output, diffs, tool arguments and results, reasons) and on structured payloads. Each match is replaced with `[REDACTED]`, keeping the label in front of it, so `token=abc` becomes `token=[REDACTED]`. Redaction runs before truncation, so a cut never leaves part of a secret behind.
+
+| Form | Examples |
+|------|----------|
+| A value labeled as a secret | `token=…`, `GITHUB_TOKEN=…`, `AWS_SECRET_ACCESS_KEY=…`, `{"api_key": "…"}`, `client_secret: …` |
+| A structured field named for a credential | the value of any field whose name ends in `password`, `secret`, `token`, `api_key`, `access_key`, `private_key`, `authorization`, `credential(s)` or `cookie` |
+| Authorization headers and flags | `Authorization: Bearer …`, `Authorization: Basic …`, `--password …`, `--token=…` |
+| URL credentials | the password in `postgres://user:…@host`, `https://x-access-token:…@github.com` |
+| Private keys | PEM and OpenSSH `PRIVATE KEY` blocks, including a block cut off before its end line |
+| Recognizable tokens | Anthropic and OpenAI `sk-…`, AWS `AKIA…`/`ASIA…`, GitHub `ghp_…`/`github_pat_…`, GitLab `glpat-…`, Slack `xox…-`, Stripe `sk_live_…`, Google `AIza…`, npm, Hugging Face, Beacon `bcn_…` keys, and JWTs |
+
+Redaction is pattern-based. A credential with no label and no recognizable shape, such as a bare random password, is not detected, so treat retained content as sensitive. Metadata-only forwarding is the control for keeping content off Beacon Cloud entirely.
+
## Prompt event example
Prompt content is included only when the source runtime emits it. Secret-like values are redacted before the event is written:
diff --git a/pkg/asymptoteobserve/privacy.go b/pkg/asymptoteobserve/privacy.go
index 9fd3e79e..36a5d703 100644
--- a/pkg/asymptoteobserve/privacy.go
+++ b/pkg/asymptoteobserve/privacy.go
@@ -11,14 +11,69 @@ const (
DefaultRawStringLimit = 2048
)
-var secretPatterns = []*regexp.Regexp{
- regexp.MustCompile(`(?i)authorization\s*[:=]\s*bearer\s+[^"',\s]+`),
- regexp.MustCompile(`(?i)(api[_-]?key|token|secret|password|authorization)\s*[:=]\s*["']?[^"',\s]+`),
- regexp.MustCompile(`(?i)bearer\s+[a-z0-9._~+/=-]+`),
- regexp.MustCompile(`sk-[a-zA-Z0-9]{20,}`),
+// redactedValue replaces a secret. Rules that match a labeled value keep the label (`token=`,
+// `Authorization: Bearer `, a URL's `scheme://user:`) and replace only the value, so the text still
+// says what was there.
+const redactedValue = "[REDACTED]"
+
+// secretKeyName is the part of a label that marks the value after it as a secret: `token`,
+// `GITHUB_TOKEN`, `api_key`, `AWS_SECRET_ACCESS_KEY`, `"password"`. The label must end in the
+// marker, so `input_tokens: 812` and `max_tokens=4096` stay readable.
+const secretKeyName = `[A-Za-z0-9_.-]*?(?:api[_-]?key|access[_-]?key|secret[_-]?key|private[_-]?key|client[_-]?secret|` +
+ `token|secret|password|passwd|passphrase|authorization|credentials?)`
+
+// secretAssignment runs from the end of a label to the start of its value: an optional closing quote
+// (escaped too, for JSON quoted inside a string), the separator, and an optional opening quote.
+const secretAssignment = "(?:\\\\?[\"'`])?\\s*[:=]\\s*(?:\\\\?[\"'`])?"
+
+// secretValue is the value itself, up to a quote, an escape, a separator or the end of a structure.
+// It never starts with `[`, so a value already replaced with [REDACTED] is left as it is and
+// redacting twice gives the same text as redacting once.
+const secretValue = "[^\"'`,\\s}\\]\\\\\\[][^\"'`,\\s}\\]\\\\]*"
+
+// secretRule is one redaction. ${1} and ${3}, when the pattern has them, are kept around the value.
+type secretRule struct {
+ pattern *regexp.Regexp
+ replacement string
+}
+
+var secretRules = []secretRule{
+ // A private key block, or the rest of the text when the block was cut before its END line.
+ {regexp.MustCompile(`-----BEGIN [A-Z0-9 ]*PRIVATE KEY( BLOCK)?-----(?:[\s\S]*?-----END [A-Z0-9 ]*PRIVATE KEY( BLOCK)?-----|[\s\S]*$)`), redactedValue},
+ // An Authorization header's credential after its scheme: Bearer, Basic, Token, Digest.
+ {regexp.MustCompile(`(?i)(authorization["']?\s*[:=]\s*["']?(?:bearer|basic|token|digest)\s+)([^"'\s,]+)()`), "${1}" + redactedValue},
+ {regexp.MustCompile(`(?i)(\bbearer\s+)([a-z0-9._~+/=-]+)()`), "${1}" + redactedValue},
+ // A labeled assignment in prose, shell, env files, YAML, TOML or JSON: `token=x`,
+ // `export GITHUB_TOKEN="x"`, `{"api_key": "x"}`. The closing quote of a quoted label is allowed
+ // between the label and the separator.
+ {regexp.MustCompile(`(?i)(\b` + secretKeyName + secretAssignment + `)(` + secretValue + `)()`), "${1}" + redactedValue},
+ // A command-line flag: `--password x`, `--token=x`, `--api-key x`.
+ {regexp.MustCompile(`(?i)((?:^|\s)--?[a-z0-9-]*(?:password|passwd|passphrase|token|secret|api-?key|access-?key)(?:=|\s+))([^\s"'-][^\s"']*)()`), "${1}" + redactedValue},
+ // The password in a URL's userinfo: `postgres://user:pw@host`, `https://x-access-token:[email protected]`.
+ {regexp.MustCompile(`(?i)(\b[a-z][a-z0-9+.-]*://[^/\s:@"'<>]+:)([^@/\s"'<>]+)(@)`), "${1}" + redactedValue + "${3}"},
+ // Credentials recognizable by their own shape, wherever they appear.
+ {regexp.MustCompile(`\bsk-[A-Za-z0-9_-]{20,}`), redactedValue}, // Anthropic sk-ant-, OpenAI sk-proj- and sk-
+ {regexp.MustCompile(`\b(?:AKIA|ASIA|ABIA|ACCA)[A-Z0-9]{16}\b`), redactedValue}, // AWS access key id
+ {regexp.MustCompile(`\bgh[pousr]_[A-Za-z0-9]{30,}`), redactedValue}, // GitHub tokens
+ {regexp.MustCompile(`\bgithub_pat_[A-Za-z0-9_]{22,}`), redactedValue}, // GitHub fine-grained tokens
+ {regexp.MustCompile(`\bglpat-[A-Za-z0-9_-]{20,}`), redactedValue}, // GitLab
+ {regexp.MustCompile(`\bxox[abposr]-[A-Za-z0-9-]{10,}`), redactedValue}, // Slack
+ {regexp.MustCompile(`\b[rs]k_(?:live|test)_[A-Za-z0-9]{16,}`), redactedValue}, // Stripe
+ {regexp.MustCompile(`\bAIza[0-9A-Za-z_-]{35}`), redactedValue}, // Google API key
+ {regexp.MustCompile(`\bnpm_[A-Za-z0-9]{36}`), redactedValue}, // npm
+ {regexp.MustCompile(`\bhf_[A-Za-z0-9]{30,}`), redactedValue}, // Hugging Face
+ {regexp.MustCompile(`\bbcn_(?:api|device|mcp)_[A-Za-z0-9_-]{8,}`), redactedValue}, // Beacon's own keys
+ {regexp.MustCompile(`\beyJ[A-Za-z0-9_-]{8,}\.eyJ[A-Za-z0-9_-]{8,}\.[A-Za-z0-9_-]{8,}`), redactedValue}, // JWT
}
-var assignedSecretPattern = regexp.MustCompile("(?i)(?:api[_-]?key|token|secret|password|authorization)\\s*[:=]\\s*[\"'`]?([^\"'`,\\s]+)")
+// assignedSecretPattern captures the value of a labeled assignment so a bare copy of it elsewhere in
+// the same text is removed too.
+var assignedSecretPattern = regexp.MustCompile(`(?i)\b` + secretKeyName + secretAssignment + `(` + secretValue + `)`)
+
+// redactWindow is how far past a string's limit redaction reads before the string is cut, so a
+// secret that straddles the cut is recognized whole rather than left as a recognizable prefix. It
+// covers a 4096-bit RSA private key block.
+const redactWindow = 4096
type PrivacyOptions struct {
RedactSecrets bool
@@ -32,27 +87,44 @@ func RedactString(value string) string {
// refers to `value` by itself.
var assigned []string
for _, match := range assignedSecretPattern.FindAllStringSubmatch(value, -1) {
- if len(match) > 1 && len(match[1]) >= 8 {
+ if len(match) > 1 && len(match[1]) >= 8 && match[1] != redactedValue {
assigned = append(assigned, match[1])
}
}
- for _, pattern := range secretPatterns {
- value = pattern.ReplaceAllStringFunc(value, func(match string) string {
- if strings.Contains(match, "=") {
- return match[:strings.Index(match, "=")+1] + "[REDACTED]"
- }
- if strings.Contains(match, ":") {
- return match[:strings.Index(match, ":")+1] + "[REDACTED]"
- }
- return "[REDACTED]"
- })
+ for _, rule := range secretRules {
+ value = rule.pattern.ReplaceAllString(value, rule.replacement)
}
for _, secret := range assigned {
- value = strings.ReplaceAll(value, secret, "[REDACTED]")
+ value = strings.ReplaceAll(value, secret, redactedValue)
}
return value
}
+// isSecretKeyName reports whether a structured field's name says its value is a credential:
+// `password`, `api_key`, `apiKey`, `X-Api-Key`, `GITHUB_TOKEN`, `AWS_SECRET_ACCESS_KEY`,
+// `client_secret`, `Cookie`. The name must end in the marker, so `input_tokens`, `token_type`,
+// `author` and `PWD` are not secrets.
+func isSecretKeyName(key string) bool {
+ var normalized strings.Builder
+ for _, r := range strings.ToLower(key) {
+ if (r >= 'a' && r <= 'z') || (r >= '0' && r <= '9') {
+ normalized.WriteRune(r)
+ }
+ }
+ name := normalized.String()
+ for _, suffix := range secretKeySuffixes {
+ if strings.HasSuffix(name, suffix) {
+ return true
+ }
+ }
+ return false
+}
+
+var secretKeySuffixes = []string{
+ "password", "passwd", "passphrase", "secret", "token", "apikey", "accesskey", "secretkey",
+ "privatekey", "authorization", "credential", "credentials", "cookie",
+}
+
func TruncateString(value string, limit int) string {
if len(value) <= limit {
return value
@@ -63,12 +135,17 @@ func TruncateString(value string, limit int) string {
return value[:limit-15] + "...[truncated]"
}
+// CleanString redacts before it truncates, so a cut can never leave part of a secret behind.
+// Redaction reads only redactWindow bytes past the limit, which keeps a multi-megabyte command output
+// as cheap as before.
func CleanString(value string, limit int, redactSecrets bool) string {
- value = TruncateString(value, limit)
- if redactSecrets {
- value = RedactString(value)
+ if !redactSecrets {
+ return TruncateString(value, limit)
}
- return value
+ if window := limit + redactWindow; limit > 0 && len(value) > window {
+ value = value[:window]
+ }
+ return TruncateString(RedactString(value), limit)
}
func SanitizeMap(input map[string]interface{}, opts PrivacyOptions) map[string]interface{} {
@@ -80,6 +157,12 @@ func SanitizeMap(input map[string]interface{}, opts PrivacyOptions) map[string]i
for key, value := range input {
switch typed := value.(type) {
case string:
+ // A value whose field is named for a credential is one, whatever it looks like:
+ // `{"password": "hunter2"}` has no shape a pattern could recognize.
+ if opts.RedactSecrets && typed != "" && isSecretKeyName(key) {
+ out[key] = redactedValue
+ continue
+ }
out[key] = CleanString(typed, limit, opts.RedactSecrets)
case map[string]interface{}:
out[key] = SanitizeMap(typed, opts)
diff --git a/pkg/asymptoteobserve/privacy_test.go b/pkg/asymptoteobserve/privacy_test.go
index e841eb57..d276ace1 100644
--- a/pkg/asymptoteobserve/privacy_test.go
+++ b/pkg/asymptoteobserve/privacy_test.go
@@ -109,3 +109,133 @@ func TestSanitizeEventRedactsAndTruncates(t *testing.T) {
t.Fatalf("tool path was not truncated: %d", len(sanitized.Tool.Path))
}
}
+
+// Credentials an agent routinely reads (.env files, ~/.aws/credentials, a clone URL, an MCP tool's
+// arguments) must not reach the log in any of the shapes they are written in.
+func TestRedactStringRedactsCommonCredentialFormats(t *testing.T) {
+ tests := []struct {
+ name, input, secret string
+ }{
+ {"aws env", "AWS_SECRET_ACCESS_KEY=wJalrXUtnFEMIK7MDENGbPxRfiCYEXAMPLEKEY", "wJalrXUtnFEMIK7MDENGbPxRfiCYEXAMPLEKEY"},
+ {"aws credentials file", "aws_secret_access_key = wJalrXUtnFEMIK7MDENGbPxRfiCYEXAMPLEKEY", "wJalrXUtnFEMIK7MDENGbPxRfiCYEXAMPLEKEY"},
+ {"aws access key id", "found AKIAIOSFODNN7EXAMPLE in the config", "AKIAIOSFODNN7EXAMPLE"},
+ {"json api_key", `{"api_key": "f00dfacecafebeef1234"}`, "f00dfacecafebeef1234"},
+ {"json token", `{"token":"opaque-session-value-9"}`, "opaque-session-value-9"},
+ {"escaped json", `{\"password\": \"hunter2hunter2\"}`, "hunter2hunter2"},
+ {"export", `export GITHUB_TOKEN="plain-value-1234567"`, "plain-value-1234567"},
+ {"yaml", "client_secret: s3cr3t-value-abc", "s3cr3t-value-abc"},
+ {"anthropic", "key is sk-ant-api03-AbCdEfGhIjKlMnOpQrStUvWxYz0123456789-_AbCd", "sk-ant-api03-AbCdEfGhIjKlMnOpQrStUvWxYz0123456789"},
+ {"openai project", "sk-proj-AbCdEfGhIjKlMnOpQrStUvWxYz0123456789", "AbCdEfGhIjKlMnOpQrStUvWxYz0123456789"},
+ {"github classic", "ghp_AbCdEfGhIjKlMnOpQrStUvWxYz0123456789", "AbCdEfGhIjKlMnOpQrStUvWxYz0123456789"},
+ {"github fine-grained", "github_pat_11ABCDEFG0123456789_abcdefghijklmnopqrstuvwxyz", "abcdefghijklmnopqrstuvwxyz"},
+ {"clone url", "git clone https://x-access-token:[email protected]/org/repo", "opaqueValue123"},
+ {"database url", "DATABASE_URL=postgres://app:[email protected]:5432/app", "pa55w0rd"},
+ {"bare database url", "connect to postgres://app:[email protected]:5432/app", "pa55w0rd"},
+ {"password flag", "mysql --user root --password hunter2hunter2", "hunter2hunter2"},
+ {"token flag", "tool login --token=opaque-flag-value", "opaque-flag-value"},
+ {"basic auth", "Authorization: Basic dXNlcjpwYXNzd29yZA==", "dXNlcjpwYXNzd29yZA=="},
+ {"slack", "xoxb-1234567890-abcdefghijkl", "abcdefghijkl"},
+ {"stripe", "sk_live_AbCdEfGhIjKlMnOpQrSt", "AbCdEfGhIjKlMnOpQrSt"},
+ {"google", "AIzaSyA-abcdefghijklmnopqrstuvwxyz012345", "abcdefghijklmnopqrstuvwxyz012345"},
+ {"jwt", "eyJhbGciOiJIUzI1NiJ9.eyJzdWIiOiIxMjM0NTY3ODkwIn0.dozjgNryP4J3jVmNHl0w5N_XgL0n3I9PlFUP0THsR8U", "dozjgNryP4J3jVmNHl0w5N_XgL0n3I9PlFUP0THsR8U"},
+ {"beacon device key", "bcn_device_abcdefgh_0123456789abcdef", "0123456789abcdef"},
+ {"rsa key", "-----BEGIN RSA PRIVATE KEY-----\nMIIEowIBAAKCAQEAsecretbody\n-----END RSA PRIVATE KEY-----", "MIIEowIBAAKCAQEAsecretbody"},
+ {"openssh key cut short", "cat ~/.ssh/id_ed25519\n-----BEGIN OPENSSH PRIVATE KEY-----\nb3BlbnNzaC1rZXktdjEAAAAA", "b3BlbnNzaC1rZXktdjEAAAAA"},
+ }
+ for _, tc := range tests {
+ t.Run(tc.name, func(t *testing.T) {
+ if got := RedactString(tc.input); strings.Contains(got, tc.secret) || !strings.Contains(got, "[REDACTED]") {
+ t.Fatalf("RedactString(%q) = %q, want %q removed", tc.input, got, tc.secret)
+ }
+ })
+ }
+}
+
+// Redaction keeps the label and the structure around a value, and leaves text that only mentions
+// tokens alone: token counts are the most common numbers in this log.
+func TestRedactStringKeepsLabelsAndOrdinaryText(t *testing.T) {
+ kept := []string{
+ "input_tokens: 812, output_tokens=95, max_tokens=4096",
+ "run go test ./... --max-tokens 10",
+ "the author: Ada",
+ "see https://github.com/org/repo/pull/12 and [email protected]:org/repo.git",
+ "PWD=/home/ada/project",
+ "task-0123456789abcdefghijklmnop",
+ }
+ for _, input := range kept {
+ if got := RedactString(input); got != input {
+ t.Fatalf("RedactString(%q) = %q, want it unchanged", input, got)
+ }
+ }
+ if got := RedactString(`{"api_key": "f00dfacecafebeef1234", "model": "x"}`); got != `{"api_key": "[REDACTED]", "model": "x"}` {
+ t.Fatalf("quoted JSON value = %q, want its quotes and the rest of the object kept", got)
+ }
+ if got := RedactString("git clone https://u:[email protected]/r"); got != "git clone https://u:[REDACTED]@example.com/r" {
+ t.Fatalf("URL credential = %q, want the scheme, user and host kept", got)
+ }
+}
+
+// Events are sanitized again on every read path, so a second pass must not change the first one's
+// output.
+func TestRedactStringIsIdempotent(t *testing.T) {
+ inputs := []string{
+ "config dump\napi_key=sk-abcdefghijklmnopqrstuvwxyz0123\nOPENAI sk-abcdefghijklmnopqrstuvwxyz0123",
+ `{"token": "opaque-session-value-9", "password": "hunter2hunter2"}`,
+ "Authorization: Bearer abc.def.ghi and --password hunter2hunter2",
+ "postgres://app:pa55w0rd@db/app",
+ }
+ for _, input := range inputs {
+ once := RedactString(input)
+ if twice := RedactString(once); twice != once {
+ t.Fatalf("RedactString is not idempotent:\nonce %q\ntwice %q", once, twice)
+ }
+ }
+}
+
+// A field named for a credential is one, whatever its value looks like.
+func TestSanitizeMapRedactsValuesUnderSecretKeys(t *testing.T) {
+ input := map[string]interface{}{
+ "tool_input": map[string]interface{}{
+ "password": "hunter2",
+ "apiKey": "plain",
+ "headers": map[string]interface{}{"X-Api-Key": "abc", "Cookie": "sid=1", "Accept": "application/json"},
+ "env": map[string]interface{}{"GITHUB_TOKEN": "v1", "AWS_SECRET_ACCESS_KEY": "v2", "PWD": "/home/ada"},
+ },
+ "usage": map[string]interface{}{"input_tokens": float64(12)},
+ "token_type": "Bearer",
+ "author": "Ada",
+ }
+ got := SanitizeMap(input, PrivacyOptions{RedactSecrets: true, StringLimit: DefaultRawStringLimit})
+ data, err := json.Marshal(got)
+ if err != nil {
+ t.Fatal(err)
+ }
+ text := string(data)
+ for _, secret := range []string{"hunter2", `"plain"`, `"abc"`, "sid=1", `"v1"`, `"v2"`} {
+ if strings.Contains(text, secret) {
+ t.Fatalf("SanitizeMap leaked %s: %s", secret, text)
+ }
+ }
+ for _, keep := range []string{"application/json", "/home/ada", `"input_tokens":12`, `"token_type":"Bearer"`, `"author":"Ada"`} {
+ if !strings.Contains(text, keep) {
+ t.Fatalf("SanitizeMap dropped %s: %s", keep, text)
+ }
+ }
+ if got := SanitizeMap(input, PrivacyOptions{StringLimit: DefaultRawStringLimit}); got["tool_input"].(map[string]interface{})["password"] != "hunter2" {
+ t.Fatal("SanitizeMap redacted by key name with RedactSecrets off")
+ }
+}
+
+// Truncation must not cut a secret into a prefix that no longer matches its pattern.
+func TestCleanStringRedactsBeforeTruncating(t *testing.T) {
+ secret := "ghp_AbCdEfGhIjKlMnOpQrStUvWxYz0123456789"
+ value := strings.Repeat("x", 90) + " " + secret
+ got := CleanString(value, 120, true)
+ if strings.Contains(got, "ghp_AbCd") {
+ t.Fatalf("CleanString left part of a secret at the cut: %q", got)
+ }
+ key := strings.Repeat("y", 200) + "\n-----BEGIN OPENSSH PRIVATE KEY-----\n" + strings.Repeat("A", 3000) + "\n-----END OPENSSH PRIVATE KEY-----\n" + strings.Repeat("z", 10000)
+ if got := CleanString(key, 1024, true); strings.Contains(got, "AAAA") || len(got) > 1024 {
+ t.Fatalf("CleanString kept key material or overran its limit: %d bytes", len(got))
+ }
+}