The cross-harness, self-improving memory layer for AI agents. Join our community: https://discord.com/invite/zdNChS2fBu
Go agent-ai-cliagent-harnessesagent-securitybrowser-extensionciclaude-codecloudcodexcursordetection-engineeringjevknowledge-basememorymobile-device-managementobservabilitysecuritysecurity-information-and-event-managementsecurity-toolstelemetrytraces
Redact common credential formats and secret-named fields
Claude3b8fd10e95d6
diff --git a/docs/security/retention-redaction.mdx b/docs/security/retention-redaction.mdxindex ccf4faed..17539038 100644--- a/docs/security/retention-redaction.mdx+++ b/docs/security/retention-redaction.mdx@@ -11,11 +11,26 @@ Beacon can write supported prompt, command, tool, file, approval, policy, and ru| Control | Behavior ||---------|----------|-| Redaction | Common secret patterns are replaced before retained content is written |+| Redaction | Common secret patterns are replaced before retained content is written (see [What is redacted](#what-is-redacted)) || Sanitization | Runtime payloads are normalized into typed event fields instead of storing source payloads verbatim wherever possible || Truncation | Oversized fields are shortened and marked with truncation metadata || Event-size limits | Events are bounded before they are written to `runtime.jsonl` or sent to configured destinations |+## What is redacted++Redaction runs on retained text (prompts, command output, diffs, tool arguments and results, reasons) and on structured payloads. Each match is replaced with `[REDACTED]`, keeping the label in front of it, so `token=abc` becomes `token=[REDACTED]`. Redaction runs before truncation, so a cut never leaves part of a secret behind.++| Form | Examples |+|------|----------|+| A value labeled as a secret | `token=…`, `GITHUB_TOKEN=…`, `AWS_SECRET_ACCESS_KEY=…`, `{"api_key": "…"}`, `client_secret: …` |+| A structured field named for a credential | the value of any field whose name ends in `password`, `secret`, `token`, `api_key`, `access_key`, `private_key`, `authorization`, `credential(s)` or `cookie` |+| Authorization headers and flags | `Authorization: Bearer …`, `Authorization: Basic …`, `--password …`, `--token=…` |+| URL credentials | the password in `postgres://user:…@host`, `https://x-access-token:…@github.com` |+| Private keys | PEM and OpenSSH `PRIVATE KEY` blocks, including a block cut off before its end line |+| Recognizable tokens | Anthropic and OpenAI `sk-…`, AWS `AKIA…`/`ASIA…`, GitHub `ghp_…`/`github_pat_…`, GitLab `glpat-…`, Slack `xox…-`, Stripe `sk_live_…`, Google `AIza…`, npm, Hugging Face, Beacon `bcn_…` keys, and JWTs |++Redaction is pattern-based. A credential with no label and no recognizable shape, such as a bare random password, is not detected, so treat retained content as sensitive. Metadata-only forwarding is the control for keeping content off Beacon Cloud entirely.+## Prompt event examplePrompt content is included only when the source runtime emits it. Secret-like values are redacted before the event is written:diff --git a/pkg/asymptoteobserve/privacy.go b/pkg/asymptoteobserve/privacy.goindex 9fd3e79e..36a5d703 100644--- a/pkg/asymptoteobserve/privacy.go+++ b/pkg/asymptoteobserve/privacy.go@@ -11,14 +11,69 @@ const (DefaultRawStringLimit = 2048)-var secretPatterns = []*regexp.Regexp{- regexp.MustCompile(`(?i)authorization\s*[:=]\s*bearer\s+[^"',\s]+`),- regexp.MustCompile(`(?i)(api[_-]?key|token|secret|password|authorization)\s*[:=]\s*["']?[^"',\s]+`),- regexp.MustCompile(`(?i)bearer\s+[a-z0-9._~+/=-]+`),- regexp.MustCompile(`sk-[a-zA-Z0-9]{20,}`),+// redactedValue replaces a secret. Rules that match a labeled value keep the label (`token=`,+// `Authorization: Bearer `, a URL's `scheme://user:`) and replace only the value, so the text still+// says what was there.+const redactedValue = "[REDACTED]"++// secretKeyName is the part of a label that marks the value after it as a secret: `token`,+// `GITHUB_TOKEN`, `api_key`, `AWS_SECRET_ACCESS_KEY`, `"password"`. The label must end in the+// marker, so `input_tokens: 812` and `max_tokens=4096` stay readable.+const secretKeyName = `[A-Za-z0-9_.-]*?(?:api[_-]?key|access[_-]?key|secret[_-]?key|private[_-]?key|client[_-]?secret|` ++ `token|secret|password|passwd|passphrase|authorization|credentials?)`++// secretAssignment runs from the end of a label to the start of its value: an optional closing quote+// (escaped too, for JSON quoted inside a string), the separator, and an optional opening quote.+const secretAssignment = "(?:\\\\?[\"'`])?\\s*[:=]\\s*(?:\\\\?[\"'`])?"++// secretValue is the value itself, up to a quote, an escape, a separator or the end of a structure.+// It never starts with `[`, so a value already replaced with [REDACTED] is left as it is and+// redacting twice gives the same text as redacting once.+const secretValue = "[^\"'`,\\s}\\]\\\\\\[][^\"'`,\\s}\\]\\\\]*"++// secretRule is one redaction. ${1} and ${3}, when the pattern has them, are kept around the value.+type secretRule struct {+ pattern *regexp.Regexp+ replacement string+}++var secretRules = []secretRule{+ // A private key block, or the rest of the text when the block was cut before its END line.+ {regexp.MustCompile(`-----BEGIN [A-Z0-9 ]*PRIVATE KEY( BLOCK)?-----(?:[\s\S]*?-----END [A-Z0-9 ]*PRIVATE KEY( BLOCK)?-----|[\s\S]*$)`), redactedValue},+ // An Authorization header's credential after its scheme: Bearer, Basic, Token, Digest.+ {regexp.MustCompile(`(?i)(authorization["']?\s*[:=]\s*["']?(?:bearer|basic|token|digest)\s+)([^"'\s,]+)()`), "${1}" + redactedValue},+ {regexp.MustCompile(`(?i)(\bbearer\s+)([a-z0-9._~+/=-]+)()`), "${1}" + redactedValue},+ // A labeled assignment in prose, shell, env files, YAML, TOML or JSON: `token=x`,+ // `export GITHUB_TOKEN="x"`, `{"api_key": "x"}`. The closing quote of a quoted label is allowed+ // between the label and the separator.+ {regexp.MustCompile(`(?i)(\b` + secretKeyName + secretAssignment + `)(` + secretValue + `)()`), "${1}" + redactedValue},+ // A command-line flag: `--password x`, `--token=x`, `--api-key x`.+ {regexp.MustCompile(`(?i)((?:^|\s)--?[a-z0-9-]*(?:password|passwd|passphrase|token|secret|api-?key|access-?key)(?:=|\s+))([^\s"'-][^\s"']*)()`), "${1}" + redactedValue},+ // The password in a URL's userinfo: `postgres://user:pw@host`, `https://x-access-token:[email protected]`.+ {regexp.MustCompile(`(?i)(\b[a-z][a-z0-9+.-]*://[^/\s:@"'<>]+:)([^@/\s"'<>]+)(@)`), "${1}" + redactedValue + "${3}"},+ // Credentials recognizable by their own shape, wherever they appear.+ {regexp.MustCompile(`\bsk-[A-Za-z0-9_-]{20,}`), redactedValue}, // Anthropic sk-ant-, OpenAI sk-proj- and sk-+ {regexp.MustCompile(`\b(?:AKIA|ASIA|ABIA|ACCA)[A-Z0-9]{16}\b`), redactedValue}, // AWS access key id+ {regexp.MustCompile(`\bgh[pousr]_[A-Za-z0-9]{30,}`), redactedValue}, // GitHub tokens+ {regexp.MustCompile(`\bgithub_pat_[A-Za-z0-9_]{22,}`), redactedValue}, // GitHub fine-grained tokens+ {regexp.MustCompile(`\bglpat-[A-Za-z0-9_-]{20,}`), redactedValue}, // GitLab+ {regexp.MustCompile(`\bxox[abposr]-[A-Za-z0-9-]{10,}`), redactedValue}, // Slack+ {regexp.MustCompile(`\b[rs]k_(?:live|test)_[A-Za-z0-9]{16,}`), redactedValue}, // Stripe+ {regexp.MustCompile(`\bAIza[0-9A-Za-z_-]{35}`), redactedValue}, // Google API key+ {regexp.MustCompile(`\bnpm_[A-Za-z0-9]{36}`), redactedValue}, // npm+ {regexp.MustCompile(`\bhf_[A-Za-z0-9]{30,}`), redactedValue}, // Hugging Face+ {regexp.MustCompile(`\bbcn_(?:api|device|mcp)_[A-Za-z0-9_-]{8,}`), redactedValue}, // Beacon's own keys+ {regexp.MustCompile(`\beyJ[A-Za-z0-9_-]{8,}\.eyJ[A-Za-z0-9_-]{8,}\.[A-Za-z0-9_-]{8,}`), redactedValue}, // JWT}-var assignedSecretPattern = regexp.MustCompile("(?i)(?:api[_-]?key|token|secret|password|authorization)\\s*[:=]\\s*[\"'`]?([^\"'`,\\s]+)")+// assignedSecretPattern captures the value of a labeled assignment so a bare copy of it elsewhere in+// the same text is removed too.+var assignedSecretPattern = regexp.MustCompile(`(?i)\b` + secretKeyName + secretAssignment + `(` + secretValue + `)`)++// redactWindow is how far past a string's limit redaction reads before the string is cut, so a+// secret that straddles the cut is recognized whole rather than left as a recognizable prefix. It+// covers a 4096-bit RSA private key block.+const redactWindow = 4096type PrivacyOptions struct {RedactSecrets bool@@ -32,27 +87,44 @@ func RedactString(value string) string {// refers to `value` by itself.var assigned []stringfor _, match := range assignedSecretPattern.FindAllStringSubmatch(value, -1) {- if len(match) > 1 && len(match[1]) >= 8 {+ if len(match) > 1 && len(match[1]) >= 8 && match[1] != redactedValue {assigned = append(assigned, match[1])}}- for _, pattern := range secretPatterns {- value = pattern.ReplaceAllStringFunc(value, func(match string) string {- if strings.Contains(match, "=") {- return match[:strings.Index(match, "=")+1] + "[REDACTED]"- }- if strings.Contains(match, ":") {- return match[:strings.Index(match, ":")+1] + "[REDACTED]"- }- return "[REDACTED]"- })+ for _, rule := range secretRules {+ value = rule.pattern.ReplaceAllString(value, rule.replacement)}for _, secret := range assigned {- value = strings.ReplaceAll(value, secret, "[REDACTED]")+ value = strings.ReplaceAll(value, secret, redactedValue)}return value}+// isSecretKeyName reports whether a structured field's name says its value is a credential:+// `password`, `api_key`, `apiKey`, `X-Api-Key`, `GITHUB_TOKEN`, `AWS_SECRET_ACCESS_KEY`,+// `client_secret`, `Cookie`. The name must end in the marker, so `input_tokens`, `token_type`,+// `author` and `PWD` are not secrets.+func isSecretKeyName(key string) bool {+ var normalized strings.Builder+ for _, r := range strings.ToLower(key) {+ if (r >= 'a' && r <= 'z') || (r >= '0' && r <= '9') {+ normalized.WriteRune(r)+ }+ }+ name := normalized.String()+ for _, suffix := range secretKeySuffixes {+ if strings.HasSuffix(name, suffix) {+ return true+ }+ }+ return false+}++var secretKeySuffixes = []string{+ "password", "passwd", "passphrase", "secret", "token", "apikey", "accesskey", "secretkey",+ "privatekey", "authorization", "credential", "credentials", "cookie",+}+func TruncateString(value string, limit int) string {if len(value) <= limit {return value@@ -63,12 +135,17 @@ func TruncateString(value string, limit int) string {return value[:limit-15] + "...[truncated]"}+// CleanString redacts before it truncates, so a cut can never leave part of a secret behind.+// Redaction reads only redactWindow bytes past the limit, which keeps a multi-megabyte command output+// as cheap as before.func CleanString(value string, limit int, redactSecrets bool) string {- value = TruncateString(value, limit)- if redactSecrets {- value = RedactString(value)+ if !redactSecrets {+ return TruncateString(value, limit)}- return value+ if window := limit + redactWindow; limit > 0 && len(value) > window {+ value = value[:window]+ }+ return TruncateString(RedactString(value), limit)}func SanitizeMap(input map[string]interface{}, opts PrivacyOptions) map[string]interface{} {@@ -80,6 +157,12 @@ func SanitizeMap(input map[string]interface{}, opts PrivacyOptions) map[string]ifor key, value := range input {switch typed := value.(type) {case string:+ // A value whose field is named for a credential is one, whatever it looks like:+ // `{"password": "hunter2"}` has no shape a pattern could recognize.+ if opts.RedactSecrets && typed != "" && isSecretKeyName(key) {+ out[key] = redactedValue+ continue+ }out[key] = CleanString(typed, limit, opts.RedactSecrets)case map[string]interface{}:out[key] = SanitizeMap(typed, opts)diff --git a/pkg/asymptoteobserve/privacy_test.go b/pkg/asymptoteobserve/privacy_test.goindex e841eb57..d276ace1 100644--- a/pkg/asymptoteobserve/privacy_test.go+++ b/pkg/asymptoteobserve/privacy_test.go@@ -109,3 +109,133 @@ func TestSanitizeEventRedactsAndTruncates(t *testing.T) {t.Fatalf("tool path was not truncated: %d", len(sanitized.Tool.Path))}}++// Credentials an agent routinely reads (.env files, ~/.aws/credentials, a clone URL, an MCP tool's+// arguments) must not reach the log in any of the shapes they are written in.+func TestRedactStringRedactsCommonCredentialFormats(t *testing.T) {+ tests := []struct {+ name, input, secret string+ }{+ {"aws env", "AWS_SECRET_ACCESS_KEY=wJalrXUtnFEMIK7MDENGbPxRfiCYEXAMPLEKEY", "wJalrXUtnFEMIK7MDENGbPxRfiCYEXAMPLEKEY"},+ {"aws credentials file", "aws_secret_access_key = wJalrXUtnFEMIK7MDENGbPxRfiCYEXAMPLEKEY", "wJalrXUtnFEMIK7MDENGbPxRfiCYEXAMPLEKEY"},+ {"aws access key id", "found AKIAIOSFODNN7EXAMPLE in the config", "AKIAIOSFODNN7EXAMPLE"},+ {"json api_key", `{"api_key": "f00dfacecafebeef1234"}`, "f00dfacecafebeef1234"},+ {"json token", `{"token":"opaque-session-value-9"}`, "opaque-session-value-9"},+ {"escaped json", `{\"password\": \"hunter2hunter2\"}`, "hunter2hunter2"},+ {"export", `export GITHUB_TOKEN="plain-value-1234567"`, "plain-value-1234567"},+ {"yaml", "client_secret: s3cr3t-value-abc", "s3cr3t-value-abc"},+ {"anthropic", "key is sk-ant-api03-AbCdEfGhIjKlMnOpQrStUvWxYz0123456789-_AbCd", "sk-ant-api03-AbCdEfGhIjKlMnOpQrStUvWxYz0123456789"},+ {"openai project", "sk-proj-AbCdEfGhIjKlMnOpQrStUvWxYz0123456789", "AbCdEfGhIjKlMnOpQrStUvWxYz0123456789"},+ {"github classic", "ghp_AbCdEfGhIjKlMnOpQrStUvWxYz0123456789", "AbCdEfGhIjKlMnOpQrStUvWxYz0123456789"},+ {"github fine-grained", "github_pat_11ABCDEFG0123456789_abcdefghijklmnopqrstuvwxyz", "abcdefghijklmnopqrstuvwxyz"},+ {"clone url", "git clone https://x-access-token:[email protected]/org/repo", "opaqueValue123"},+ {"database url", "DATABASE_URL=postgres://app:[email protected]:5432/app", "pa55w0rd"},+ {"bare database url", "connect to postgres://app:[email protected]:5432/app", "pa55w0rd"},+ {"password flag", "mysql --user root --password hunter2hunter2", "hunter2hunter2"},+ {"token flag", "tool login --token=opaque-flag-value", "opaque-flag-value"},+ {"basic auth", "Authorization: Basic dXNlcjpwYXNzd29yZA==", "dXNlcjpwYXNzd29yZA=="},+ {"slack", "xoxb-1234567890-abcdefghijkl", "abcdefghijkl"},+ {"stripe", "sk_live_AbCdEfGhIjKlMnOpQrSt", "AbCdEfGhIjKlMnOpQrSt"},+ {"google", "AIzaSyA-abcdefghijklmnopqrstuvwxyz012345", "abcdefghijklmnopqrstuvwxyz012345"},+ {"jwt", "eyJhbGciOiJIUzI1NiJ9.eyJzdWIiOiIxMjM0NTY3ODkwIn0.dozjgNryP4J3jVmNHl0w5N_XgL0n3I9PlFUP0THsR8U", "dozjgNryP4J3jVmNHl0w5N_XgL0n3I9PlFUP0THsR8U"},+ {"beacon device key", "bcn_device_abcdefgh_0123456789abcdef", "0123456789abcdef"},+ {"rsa key", "-----BEGIN RSA PRIVATE KEY-----\nMIIEowIBAAKCAQEAsecretbody\n-----END RSA PRIVATE KEY-----", "MIIEowIBAAKCAQEAsecretbody"},+ {"openssh key cut short", "cat ~/.ssh/id_ed25519\n-----BEGIN OPENSSH PRIVATE KEY-----\nb3BlbnNzaC1rZXktdjEAAAAA", "b3BlbnNzaC1rZXktdjEAAAAA"},+ }+ for _, tc := range tests {+ t.Run(tc.name, func(t *testing.T) {+ if got := RedactString(tc.input); strings.Contains(got, tc.secret) || !strings.Contains(got, "[REDACTED]") {+ t.Fatalf("RedactString(%q) = %q, want %q removed", tc.input, got, tc.secret)+ }+ })+ }+}++// Redaction keeps the label and the structure around a value, and leaves text that only mentions+// tokens alone: token counts are the most common numbers in this log.+func TestRedactStringKeepsLabelsAndOrdinaryText(t *testing.T) {+ kept := []string{+ "input_tokens: 812, output_tokens=95, max_tokens=4096",+ "run go test ./... --max-tokens 10",+ "the author: Ada",+ "see https://github.com/org/repo/pull/12 and [email protected]:org/repo.git",+ "PWD=/home/ada/project",+ "task-0123456789abcdefghijklmnop",+ }+ for _, input := range kept {+ if got := RedactString(input); got != input {+ t.Fatalf("RedactString(%q) = %q, want it unchanged", input, got)+ }+ }+ if got := RedactString(`{"api_key": "f00dfacecafebeef1234", "model": "x"}`); got != `{"api_key": "[REDACTED]", "model": "x"}` {+ t.Fatalf("quoted JSON value = %q, want its quotes and the rest of the object kept", got)+ }+ if got := RedactString("git clone https://u:[email protected]/r"); got != "git clone https://u:[REDACTED]@example.com/r" {+ t.Fatalf("URL credential = %q, want the scheme, user and host kept", got)+ }+}++// Events are sanitized again on every read path, so a second pass must not change the first one's+// output.+func TestRedactStringIsIdempotent(t *testing.T) {+ inputs := []string{+ "config dump\napi_key=sk-abcdefghijklmnopqrstuvwxyz0123\nOPENAI sk-abcdefghijklmnopqrstuvwxyz0123",+ `{"token": "opaque-session-value-9", "password": "hunter2hunter2"}`,+ "Authorization: Bearer abc.def.ghi and --password hunter2hunter2",+ "postgres://app:pa55w0rd@db/app",+ }+ for _, input := range inputs {+ once := RedactString(input)+ if twice := RedactString(once); twice != once {+ t.Fatalf("RedactString is not idempotent:\nonce %q\ntwice %q", once, twice)+ }+ }+}++// A field named for a credential is one, whatever its value looks like.+func TestSanitizeMapRedactsValuesUnderSecretKeys(t *testing.T) {+ input := map[string]interface{}{+ "tool_input": map[string]interface{}{+ "password": "hunter2",+ "apiKey": "plain",+ "headers": map[string]interface{}{"X-Api-Key": "abc", "Cookie": "sid=1", "Accept": "application/json"},+ "env": map[string]interface{}{"GITHUB_TOKEN": "v1", "AWS_SECRET_ACCESS_KEY": "v2", "PWD": "/home/ada"},+ },+ "usage": map[string]interface{}{"input_tokens": float64(12)},+ "token_type": "Bearer",+ "author": "Ada",+ }+ got := SanitizeMap(input, PrivacyOptions{RedactSecrets: true, StringLimit: DefaultRawStringLimit})+ data, err := json.Marshal(got)+ if err != nil {+ t.Fatal(err)+ }+ text := string(data)+ for _, secret := range []string{"hunter2", `"plain"`, `"abc"`, "sid=1", `"v1"`, `"v2"`} {+ if strings.Contains(text, secret) {+ t.Fatalf("SanitizeMap leaked %s: %s", secret, text)+ }+ }+ for _, keep := range []string{"application/json", "/home/ada", `"input_tokens":12`, `"token_type":"Bearer"`, `"author":"Ada"`} {+ if !strings.Contains(text, keep) {+ t.Fatalf("SanitizeMap dropped %s: %s", keep, text)+ }+ }+ if got := SanitizeMap(input, PrivacyOptions{StringLimit: DefaultRawStringLimit}); got["tool_input"].(map[string]interface{})["password"] != "hunter2" {+ t.Fatal("SanitizeMap redacted by key name with RedactSecrets off")+ }+}++// Truncation must not cut a secret into a prefix that no longer matches its pattern.+func TestCleanStringRedactsBeforeTruncating(t *testing.T) {+ secret := "ghp_AbCdEfGhIjKlMnOpQrStUvWxYz0123456789"+ value := strings.Repeat("x", 90) + " " + secret+ got := CleanString(value, 120, true)+ if strings.Contains(got, "ghp_AbCd") {+ t.Fatalf("CleanString left part of a secret at the cut: %q", got)+ }+ key := strings.Repeat("y", 200) + "\n-----BEGIN OPENSSH PRIVATE KEY-----\n" + strings.Repeat("A", 3000) + "\n-----END OPENSSH PRIVATE KEY-----\n" + strings.Repeat("z", 10000)+ if got := CleanString(key, 1024, true); strings.Contains(got, "AAAA") || len(got) > 1024 {+ t.Fatalf("CleanString kept key material or overran its limit: %d bytes", len(got))+ }+}