Skip to content

The cross-harness, self-improving memory layer for AI agents. Join our community: https://discord.com/invite/zdNChS2fBu

fix(privacy): redact secrets that directly follow another secret (#1153)

kyle-elliott-asymptoteb6439f20a9c1
diff --git a/pkg/asymptoteobserve/privacy.go b/pkg/asymptoteobserve/privacy.go
index dedc01d1..3ac8536e 100644
--- a/pkg/asymptoteobserve/privacy.go
+++ b/pkg/asymptoteobserve/privacy.go
@@ -132,6 +132,12 @@ type redactionRule struct {
// how many bytes before the needle a startNeedle match does. See redact_engine.go.
start redactStart
offset int
+ // shape marks a credential recognized by its own shape: a public type prefix, ${1}, and the
+ // secret after it. Such a rule also redacts a secret written right after its match
+ // (redact_adjacent.go).
+ shape redactShapeKind
+ // runsOn, when set, returns where the text a match replaces ends, at or past the match's end.
+ runsOn func(t string, match []int) int
}
// pattern is the rule's compiled pattern.
@@ -187,6 +193,13 @@ func curlRule(expr, replacement string) redactionRule {
return r
}
+// shapeRule is a rule for a credential recognized by its own shape (redactionRule.shape).
+func shapeRule(expr, replacement string, needles ...string) redactionRule {
+ r := rule(expr, replacement, nil, needles...)
+ r.shape = shapeToken
+ return r
+}
+
// lazyPattern compiles expr the first time it is called.
//
// The rules whose pattern names a credential (redactLabel, redactFlag) are written as struct
@@ -421,30 +434,32 @@ var redactionRules = concatRules([]redactionRule{
foldedRule(`(?im)((?:^\s*|["']\s*|(?:-H|--header)(?:=|\s+)["']?)(?:set-)?cookie["']?\s*:\s*["']?)([^\n"'\\]+)`, "${1}"+redactedValue, isRedactedValue, "cookie").from(startLine),
// The password in a URL's userinfo: `postgres://user:pw@host`, `https://x-access-token:[email protected]`,
// and with no user name, the way Redis URLs carry a password: `redis://:pw@host`.
- foldedRule(`(?i)(\b[a-z][a-z0-9+.-]*://[^/\s:@"'<>]*:)([^@/\s"'<>]+)(@)`, "${1}"+redactedValue+"${3}", nil, "://").from(startScheme),
+ foldedRule(`(?i)(\b[a-z][a-z0-9+.-]*://[^/\s:@"'<>]*:)([^@/\s"'<>]+)(@)`, "${1}"+redactedValue+"${3}", nil, "://").from(startScheme).runningOn(urlRunsOn),
// Credentials recognizable by their own shape, wherever they appear. The type prefix is public
// and stays (`ghp_[REDACTED]`, `AKIA[REDACTED]`), the way secret scanners report a finding: the
// line still says a GitHub token or an AWS key was there, which is what detection rules match on
- // in the log, and none of the credential is left.
- rule(`\b(sk-)[A-Za-z0-9_-]{20,}`, "${1}"+redactedValue, nil, "sk-"), // Anthropic sk-ant-, OpenAI sk-proj- and sk-
+ // in the log, and none of the credential is left. A secret written right after one of these is
+ // redacted with it, keeping its own prefix: `ghp_[REDACTED]ghp_[REDACTED]` (redact_adjacent.go).
+ shapeRule(`\b(sk-)[A-Za-z0-9_-]{20,}`, "${1}"+redactedValue, "sk-"), // Anthropic sk-ant-, OpenAI sk-proj- and sk-
// An AWS secret access key has no shape of its own, but where it follows its key id within a few
// separators (a credentials CSV, `aws configure` output, a table, a JSON array) it is the 40
- // characters after them. This runs before the key id rule below replaces the id it keys on.
- rule(`(\b(?:AKIA|ASIA)[A-Z0-9]{16}\b[\s,:|"']{1,8})[A-Za-z0-9/+]{40}([^A-Za-z0-9/+=]|$)`, "${1}"+redactedValue+"${2}", nil, "AKIA", "ASIA"),
- rule(`\b(AKIA|ASIA|ABIA|ACCA)[A-Z0-9]{16}\b`, "${1}"+redactedValue, nil, "AKIA", "ASIA", "ABIA", "ACCA"), // AWS access key id
- rule(`\b(gh[pousr]_)[A-Za-z0-9]{30,}`, "${1}"+redactedValue, nil, "ghp_", "gho_", "ghu_", "ghs_", "ghr_"), // GitHub tokens
- rule(`\b(github_pat_)[A-Za-z0-9_]{22,}`, "${1}"+redactedValue, nil, "github_pat_"), // GitHub fine-grained tokens
- rule(`\b(glpat-)[A-Za-z0-9_-]{20,}`, "${1}"+redactedValue, nil, "glpat-"), // GitLab
- rule(`\b(xox[abposr]-)[A-Za-z0-9-]{10,}`, "${1}"+redactedValue, nil, "xox"), // Slack
- rule(`\b([rs]k_(?:live|test)_)[A-Za-z0-9]{16,}`, "${1}"+redactedValue, nil, "k_live_", "k_test_").before(1), // Stripe
- rule(`\b(AIza)[0-9A-Za-z_-]{35}`, "${1}"+redactedValue, nil, "AIza"), // Google API key
- rule(`\b(ya29\.)(?:c\.)?[0-9A-Za-z_-]{20,}`, "${1}"+redactedValue, nil, "ya29."), // Google OAuth access token
- rule(`\b(npm_)[A-Za-z0-9]{36}`, "${1}"+redactedValue, nil, "npm_"), // npm
- rule(`\b(hf_)[A-Za-z0-9]{30,}`, "${1}"+redactedValue, nil, "hf_"), // Hugging Face
+ // characters after them. The key id is replaced with it, as the key id rule below would: where
+ // this rule redacts a secret right after another one, the key id rule has already run.
+ shapeRule(redactAWSPair, "${1}"+redactedValue+"${2}"+redactedValue+"${3}", "AKIA", "ASIA"),
+ shapeRule(`\b(AKIA|ASIA|ABIA|ACCA)[A-Z0-9]{16}\b`, "${1}"+redactedValue, "AKIA", "ASIA", "ABIA", "ACCA"), // AWS access key id
+ shapeRule(`\b(gh[pousr]_)[A-Za-z0-9]{30,}`, "${1}"+redactedValue, "ghp_", "gho_", "ghu_", "ghs_", "ghr_"), // GitHub tokens
+ shapeRule(`\b(github_pat_)[A-Za-z0-9_]{22,}`, "${1}"+redactedValue, "github_pat_"), // GitHub fine-grained tokens
+ shapeRule(`\b(glpat-)[A-Za-z0-9_-]{20,}`, "${1}"+redactedValue, "glpat-"), // GitLab
+ shapeRule(`\b(xox[abposr]-)[A-Za-z0-9-]{10,}`, "${1}"+redactedValue, "xox"), // Slack
+ shapeRule(`\b([rs]k_(?:live|test)_)[A-Za-z0-9]{16,}`, "${1}"+redactedValue, "k_live_", "k_test_").before(1), // Stripe
+ shapeRule(`\b(AIza)[0-9A-Za-z_-]{35}`, "${1}"+redactedValue, "AIza"), // Google API key
+ shapeRule(`\b(ya29\.)(?:c\.)?[0-9A-Za-z_-]{20,}`, "${1}"+redactedValue, "ya29."), // Google OAuth access token
+ shapeRule(`\b(npm_)[A-Za-z0-9]{36}`, "${1}"+redactedValue, "npm_"), // npm
+ shapeRule(`\b(hf_)[A-Za-z0-9]{30,}`, "${1}"+redactedValue, "hf_"), // Hugging Face
// Beacon's own keys and tokens: API keys, device and enforcement keys, MCP tokens and their refresh
// tokens, and the CLI sign-in's access token, authorization code, client and exchange values.
- rule(`\b(bcn_(?:api|device|mcprt|mcp|enfdev|cli|code|client|exchange)_)[A-Za-z0-9_-]{8,}`, "${1}"+redactedValue, nil, "bcn_"),
- rule(`\b(eyJ)[A-Za-z0-9_-]{8,}\.eyJ[A-Za-z0-9_-]{8,}\.[A-Za-z0-9_-]{8,}`, "${1}"+redactedValue, nil, "eyJ"), // JWT
+ shapeRule(`\b(bcn_(?:api|device|mcprt|mcp|enfdev|cli|code|client|exchange)_)[A-Za-z0-9_-]{8,}`, "${1}"+redactedValue, "bcn_"),
+ shapeRule(`\b(eyJ)[A-Za-z0-9_-]{8,}\.eyJ[A-Za-z0-9_-]{8,}\.[A-Za-z0-9_-]{8,}`, "${1}"+redactedValue, "eyJ").segmented(), // JWT
})
func concatRules(groups ...[]redactionRule) []redactionRule {
@@ -467,6 +482,20 @@ func (rule redactionRule) before(offset int) redactionRule {
return rule
}
+// segmented marks a shape rule whose secret is a JWT: three dot-separated segments, the first two
+// opening with `eyJ`, each of any length. redact_adjacent.go reads them itself to tell where one
+// ends without reading on to the end of the text each time it asks.
+func (rule redactionRule) segmented() redactionRule {
+ rule.shape = shapeJWT
+ return rule
+}
+
+// runningOn returns the rule with the text its matches replace running on where runsOn says.
+func (rule redactionRule) runningOn(runsOn func(t string, match []int) int) redactionRule {
+ rule.runsOn = runsOn
+ return rule
+}
+
// apply replaces every match of the rule in the gate's text, except one whose value keep passes,
// and a repeating rule until the text stops changing.
func (rule redactionRule) apply(text *redactGate, slot *redactSlot) string {
@@ -482,20 +511,31 @@ func (rule redactionRule) apply(text *redactGate, slot *redactSlot) string {
func (rule redactionRule) applyOnce(text *redactGate, slot *redactSlot) string {
value := text.value
var out []byte
- last := 0
- for match, pattern := range rule.matches(text, slot) {
- if rule.keep != nil {
- index := pattern.SubexpIndex("value")
+ last, slashed := 0, false
+ for match, use := range rule.matches(text, slot) {
+ if match[0] < last {
+ continue // inside a match that ran on (redactionRule.runsOn)
+ }
+ if keep := use.rule.keep; keep != nil {
+ index := use.pattern.SubexpIndex("value")
if index < 0 {
index = 2
}
- if group := 2 * index; match[group] >= 0 && rule.keep(value[match[group]:match[group+1]]) {
+ if group := 2 * index; match[group] >= 0 && keep(value[match[group]:match[group+1]]) {
+ text.hidden = text.hidden || keptLabel(value, match[group+1])
continue
}
}
out = append(out, value[last:match[0]]...)
- out = pattern.ExpandString(out, rule.replacement, value, match)
- last = match[1]
+ replacement, end := use.rule.replacing(value, match)
+ out = use.pattern.ExpandString(out, replacement, value, match)
+ last = end
+ // An AWS secret may hold a `/`, where the URL rule, which runs before the shape rules,
+ // stopped a password; once the secret is replaced, the password can run on to its `@`.
+ slashed = slashed || slashedAWSSecret(use.rule.expr, value, match)
+ }
+ if slashed && strings.Contains(value, "://") {
+ text.hidden = true
}
if out == nil {
return value
@@ -503,6 +543,128 @@ func (rule redactionRule) applyOnce(text *redactGate, slot *redactSlot) string {
return string(append(out, value[last:]...))
}
+// replacing returns the replacement of match in t, and where the text it replaces ends: the
+// match's end, unless the rule's match runs on (runsOn), when what the rule keeps after its value
+// goes too.
+func (rule *redactionRule) replacing(t string, match []int) (string, int) {
+ if rule.runsOn != nil {
+ if end := rule.runsOn(t, match); end >= 0 {
+ return "${1}" + redactedValue, end
+ }
+ }
+ return rule.replacement, match[1]
+}
+
+// urlRunsOn returns where the password of a URL match runs on to when the URL's user name is a
+// secret-named label (`https://x-access-token:…@`), which the label rules redact too, or -1. The
+// two stop at different bytes, a label's value at a quote, a comma or whitespace and a URL's
+// password at an `@`, a slash, a quote or whitespace, and once one has replaced its part, the
+// other's runs on over the marker; so each pass of the rules took one more piece of what follows,
+// starting with the `@`. The value runs on as far as the two take it in turn, which is where
+// every later pass would leave it.
+//
+// The shape rules run after this one, and one of them changes where either run stops: an AWS key
+// id followed by its secret (awsPairAt), a secret recognized only after its key id, which holds
+// `/` and `+`. A label's value that ends in the key id would leave the secret with no key id
+// before it, so it takes the separators and the secret too; the password reads the pair as the
+// markers and separators the shape rule leaves of it.
+func urlRunsOn(t string, m []int) int {
+ if !labelEndsAt(t[:m[3]]) {
+ return -1
+ }
+ end := m[1]
+ for {
+ from := end
+ // The label's value: the marker the URL rule wrote and what follows it. A key id it ends
+ // in, before any separators it takes (`:`, `|`), takes its secret with it.
+ for run := end; ; {
+ for end < len(t) && !strings.ContainsRune("\"'`,", rune(t[end])) && !isRegexpSpace(t[end]) {
+ end++
+ }
+ k := end
+ for k > run && end-k < 8 && (t[k-1] == ':' || t[k-1] == '|') {
+ k--
+ }
+ if k-run < awsKeyIDLen {
+ break
+ }
+ _, secretEnd := awsPairAt(t, k-awsKeyIDLen)
+ if secretEnd < 0 {
+ break
+ }
+ end, run = secretEnd, secretEnd
+ }
+ // The URL's password: that value and what follows it up to an `@`.
+ at := end
+ for at < len(t) && t[at] != '@' && t[at] != '/' && !strings.ContainsRune("\"'<>", rune(t[at])) && !isRegexpSpace(t[at]) {
+ if sep, secretEnd := awsPairAt(t, at); secretEnd >= 0 {
+ // The key id and the secret become markers; the separators between them stay.
+ if stop := strings.IndexAny(t[sep:secretEnd], "\"'"+regexpSpace); stop >= 0 {
+ at = sep + stop
+ break
+ }
+ at = secretEnd
+ continue
+ }
+ at++
+ }
+ redactWorked(at - from)
+ if at == len(t) || t[at] != '@' {
+ return end
+ }
+ end = at + 1
+ }
+}
+
+// redactAWSPair is an AWS access key id followed by its secret access key within a few separators:
+// ${1} the key id's type prefix, ${2} the separators, ${3} what ends the secret.
+const redactAWSPair = `(\b(?:AKIA|ASIA))[A-Z0-9]{16}(\b[\s,:|"']{1,8})[A-Za-z0-9/+]{40}([^A-Za-z0-9/+=]|$)`
+
+// awsKeyIDLen is the length of an AWS access key id.
+const awsKeyIDLen = 20
+
+var redactAWSPairAt = lazyPattern(`^` + redactAWSPair)
+
+// awsPairAt returns, where an AWS key id that the shape rule for a key id and its secret matches
+// starts at s in t, where the separators after the key id start and where the secret ends;
+// otherwise -1, -1.
+func awsPairAt(t string, s int) (sep, secretEnd int) {
+ if s > 0 && isWordByte(t[s-1]) || !strings.HasPrefix(t[s:], "AKIA") && !strings.HasPrefix(t[s:], "ASIA") {
+ return -1, -1
+ }
+ pair := redactAWSPairAt().FindStringSubmatchIndex(t[s:])
+ if pair == nil {
+ redactWorked(min(len(t)-s, 80))
+ return -1, -1
+ }
+ redactWorked(pair[1])
+ return s + pair[4], s + pair[6]
+}
+
+// labelEndsAt reports whether s ends with a secret-named label and its separator, as the label
+// rules read them: one of redactLabel's markers, an optional closing quote, and `:` or `=` with
+// whitespace around. `deploykey:` is no label; `api_key:` is.
+func labelEndsAt(s string) bool {
+ i := len(s)
+ for i > 0 && isRegexpSpace(s[i-1]) {
+ i--
+ }
+ if i == 0 || s[i-1] != ':' && s[i-1] != '=' {
+ return false
+ }
+ for i--; i > 0 && isRegexpSpace(s[i-1]); i-- {
+ }
+ if i > 0 && strings.IndexByte(redactQuotes, s[i-1]) >= 0 {
+ if i--; i > 0 && s[i-1] == '\\' {
+ i--
+ }
+ }
+ return redactLabelEnd().MatchString(s[max(0, i-32):i])
+}
+
+// redactLabelEnd matches text that ends in one of redactLabel's markers.
+var redactLabelEnd = lazyPattern(`(?i)(?:` + strings.TrimPrefix(redactLabel, `[A-Za-z0-9_.-]*?`) + `)$`)
+
// redactAssignedRule captures the value of a labeled assignment, quoted with ", ' or a backtick
// (${1} to ${3}) or not (${4}), so a bare copy of it elsewhere in the same text is
// removed too.
@@ -545,44 +707,150 @@ func RedactString(value string) string {
// label rules cost microseconds on a short string and tens of milliseconds on a tool output of a
// few kilobytes. Almost no text holds a needle of most rules, and one pass over the text finds
// every needle of every rule at once.
+//
+// One pass is enough except where a match hides another that only a later rule uncovers. A rule
+// does not look inside a match whose value it keeps, nor the capture of assigned values inside a
+// value it takes, and that value can itself be a label with its separator (keptLabel): of
+// `token:seCret":token":0` the rule for a quoted label with a bare value keeps `seCret":token`
+// (`token` names an auth scheme) and never sees `token":0`; the rule for unquoted labels then
+// replaces `seCret`, the label that hid it, and only the next pass redacted `0`. Copies of
+// assigned values, replaced after every rule has run, can be where a match starts or ends
+// (copyHidesMatch); and the URL rule stops a password at a `/` in an AWS secret, which the shape
+// rules replace after it. Such text is redacted again until it stops changing, as every later call
+// would redact it. A pass can only uncover what the one before hid; texts the tests and fuzzing
+// have tried took at most three passes that change them, and redactMaxPasses keeps the cost
+// linear whatever the text.
func redactString(value string, gated bool) string {
+ for pass := 1; ; pass++ {
+ out, hidden := redactPass(value, gated)
+ if !hidden || out == value || pass == redactMaxPasses {
+ return out
+ }
+ value = out
+ }
+}
+
+// redactMaxPasses bounds the passes redactString makes over text where a match hid another.
+const redactMaxPasses = 8
+
+// redactPass is one pass of every rule over value, and whether a match may have hidden another
+// (keptLabel, copyHidesMatch).
+func redactPass(value string, gated bool) (string, bool) {
engine := redactEngineFor()
text := redactGate{value: value, enabled: gated, engine: engine}
if text.holdsNothing() {
// Most text: no rule's needle, so no set holds and no rule can match.
- return value
+ return value, false
}
// Remember assignment values before replacing their labeled occurrence so
// repeated bare copies in the same retained prompt/output are also removed.
// This is common in model reasoning that quotes `token=value` and later
// refers to `value` by itself.
var assigned map[string]struct{}
+ var kept bool
if text.holds(redactLabelNeedles) {
- assignedRule := redactAssignedRule()
- for match := range assignedRule.matches(&text, &engine.assigned) {
- var copied strings.Builder
- for g := 2; g+1 < len(match); g += 2 {
- if match[g] >= 0 {
- copied.WriteString(value[match[g]:match[g+1]])
- }
- }
- if s := copied.String(); len(s) >= 8 && s != redactedValue && !isAuthScheme(s) {
- if assigned == nil {
- assigned = map[string]struct{}{}
- }
- assigned[s] = struct{}{}
- }
- }
+ assigned, kept = captureAssigned(&text, engine, assigned)
+ text.copies = byFirstBytes(assigned)
}
+ original := text.value
for i, rule := range redactionRules {
if !text.holds(rule.needles) {
continue
}
text.set(rule.apply(&text, &engine.rules[i]))
}
- value = text.value
- // Each copy is searched for and removed; none of it survives into the result.
- return replaceAssigned(value, assigned)
+ // A value the label rules keep part of can be captured whole and then changed by a later rule:
+ // of `"secret": null}…x-access-token:ghs_…`, the rule for quoted labels keeps `null`, the
+ // capture runs on to the end, and the rule for `x-access-token:` replaces the token inside it,
+ // so no copy of what was captured is left to remove. What the value became is captured too, so
+ // a second pass over the result finds nothing this one did not.
+ if kept && text.value != original {
+ assigned, _ = captureAssigned(&text, engine, assigned)
+ }
+ // Each copy is searched for and removed; none of it survives into the result. A secret right
+ // after a replaced copy, or right after a replaced key, starts a word only now.
+ replaced, glued := replaceAssigned(text.value, assigned)
+ hidden := text.hidden || replaced != text.value && copyHidesMatch(assigned, glued)
+ out, slashed := engine.redactAfterMarkers(replaced)
+ return out, hidden || slashed && strings.Contains(out, "://")
+}
+
+// copyHidesMatch reports whether replacing copies of assigned values, which happens after every
+// rule has run, may have made a match where the rules found none: where a copy that starts with a
+// byte no word starts with was, which some rules' values cannot start with and their marker can;
+// where a copy held a byte a rule's value stops at (`/`, `@`, a quote, a comma, a blank), which
+// its marker does not; or where a word byte comes right before one (glued), which ends a word, or
+// a redacted secret, only now. Of `-pAsswd --token -token:"--token ` the flag finds no value in
+// `--token`, which starts with a dash, until the copy of the value `--token ` is replaced; of
+// `AKIAIOSFODNN7EXAMPLEAuthorization: Authorization: ` the key id rule finds no end before the
+// copy of `Authorization:` that the label holds. The next pass redacts them.
+func copyHidesMatch(assigned map[string]struct{}, glued bool) bool {
+ if glued {
+ return true
+ }
+ for s := range assigned {
+ if !isWordByte(s[0]) || strings.IndexFunc(s, stopsAValue) >= 0 {
+ return true
+ }
+ }
+ return false
+}
+
+// stopsAValue reports whether some rule's value may stop at r: any byte but a word byte, `.` and
+// `-`.
+func stopsAValue(r rune) bool {
+ return r >= utf8.RuneSelf || !isWordByte(byte(r)) && r != '.' && r != '-'
+}
+
+// markersAfterWords counts the markers in t that a word byte comes right before.
+func markersAfterWords(t string) int {
+ n := 0
+ for from := 0; ; {
+ i := strings.Index(t[from:], redactedValue)
+ if i < 0 {
+ return n
+ }
+ if p := from + i; p > 0 && isWordByte(t[p-1]) {
+ n++
+ }
+ from += i + len(redactedValue)
+ }
+}
+
+// keptLabel reports whether a value that ends at end in t, one a rule keeps or the capture takes,
+// is itself a secret-named label followed by its separator, as `token` is in `"secret":token":0`.
+func keptLabel(t string, end int) bool {
+ rest, ok := skipSeparator(t[end:])
+ return ok && labelEndsAt(t[:len(t)-len(rest)])
+}
+
+// captureAssigned adds the value of every labeled assignment in the gate's text to assigned. kept
+// reports whether one, of any length, was unquoted after a quoted label: the one way a value is
+// captured that the label rules may keep part of (redactBare stops where the capture runs on), so
+// that a later rule can change it, or make it long enough to remove.
+func captureAssigned(text *redactGate, engine *redactEngine, assigned map[string]struct{}) (_ map[string]struct{}, kept bool) {
+ value := text.value
+ for match := range redactAssignedRule().matches(text, &engine.assigned) {
+ // The value is in one group: each way of writing it is an alternative.
+ g := 2
+ for g+1 < len(match) && match[g] < 0 {
+ g += 2
+ }
+ if g+1 >= len(match) {
+ continue
+ }
+ kept = kept || g+2 == len(match) && strings.ContainsAny(value[match[0]:match[g]], redactQuotes)
+ // A value that is itself a label hides the assignment it begins from the capture, as a kept
+ // one does from the label rules.
+ text.hidden = text.hidden || keptLabel(value, match[g+1])
+ if s := value[match[g]:match[g+1]]; len(s) >= redactCopyMin && s != redactedValue && !isAuthScheme(s) {
+ if assigned == nil {
+ assigned = map[string]struct{}{}
+ }
+ assigned[s] = struct{}{}
+ }
+ }
+ return assigned, kept
}
// replaceAssigned replaces every copy of each assigned value with redactedValue in one pass.
@@ -590,9 +858,13 @@ func redactString(value string, gated bool) string {
// grow with the text, so the cost was quadratic: Beacon Cloud redacts uploaded shared traces with
// this function, where that let a small gzip body hold a core for minutes. The values are a set,
// longest first, so where two start at the same byte the longer is removed whole.
-func replaceAssigned(value string, assigned map[string]struct{}) string {
+//
+// It also reports whether a word byte comes right before a copy it replaced (copyHidesMatch). The
+// copies are first replaced with a NUL, where the text holds none, so their places can be read;
+// where it holds one, the markers that follow a word byte are counted before and after instead.
+func replaceAssigned(value string, assigned map[string]struct{}) (string, bool) {
if len(assigned) == 0 {
- return value
+ return value, false
}
values := slices.SortedFunc(maps.Keys(assigned), func(a, b string) int {
if len(a) != len(b) {
@@ -600,11 +872,51 @@ func replaceAssigned(value string, assigned map[string]struct{}) string {
}
return strings.Compare(a, b)
})
+ place := "\x00"
+ if strings.IndexByte(value, 0) >= 0 {
+ place = redactedValue
+ }
pairs := make([]string, 0, 2*len(values))
for _, copied := range values {
- pairs = append(pairs, copied, redactedValue)
+ pairs = append(pairs, copied, place)
+ }
+ replaced := strings.NewReplacer(pairs...).Replace(value)
+ if place == redactedValue {
+ return replaced, markersAfterWords(replaced) > markersAfterWords(value)
+ }
+ glued := false
+ for i := strings.IndexByte(replaced, 0); i >= 0 && !glued; {
+ glued = i > 0 && isWordByte(replaced[i-1])
+ j := strings.IndexByte(replaced[i+1:], 0)
+ if j < 0 {
+ break
+ }
+ i += 1 + j
+ }
+ return strings.ReplaceAll(replaced, place, redactedValue), glued
+}
+
+// redactCopyMin is the least length of an assigned value whose copies are removed.
+const redactCopyMin = 8
+
+// redactCopies are the assigned values by their first redactCopyMin bytes, so a place in the text
+// is looked up once to tell whether a copy may start there, and the length of the longest
+// (redactChain.overCopy).
+type redactCopies struct {
+ byFirst map[string][]string
+ longest int
+}
+
+func byFirstBytes(assigned map[string]struct{}) *redactCopies {
+ if len(assigned) == 0 {
+ return nil
+ }
+ copies := &redactCopies{byFirst: map[string][]string{}}
+ for s := range assigned {
+ copies.byFirst[s[:redactCopyMin]] = append(copies.byFirst[s[:redactCopyMin]], s)
+ copies.longest = max(copies.longest, len(s))
}
- return strings.NewReplacer(pairs...).Replace(value)
+ return copies
}
// IsSecretKeyName reports whether a structured field's name has a word naming a credential, for
diff --git a/pkg/asymptoteobserve/privacy_test.go b/pkg/asymptoteobserve/privacy_test.go
index 533a6f36..53e7d7f4 100644
--- a/pkg/asymptoteobserve/privacy_test.go
+++ b/pkg/asymptoteobserve/privacy_test.go
@@ -231,15 +231,7 @@ func TestRedactStringKeepsLabelsAndOrdinaryText(t *testing.T) {
// Events are sanitized again on every read path, so a second pass must not change the first one's
// output.
func TestRedactStringIsIdempotent(t *testing.T) {
- inputs := []string{
- "config dump\napi_key=sk-abcdefghijklmnopqrstuvwxyz0123\nOPENAI sk-abcdefghijklmnopqrstuvwxyz0123",
- `{"token": "opaque-session-value-9", "password": "hunter2hunter2"}`,
- "Authorization: Bearer abc.def.ghi and --password hunter2hunter2",
- "postgres://app:pa55w0rd@db/app",
- "curl -u admin:hunter2hunter2 -H 'Cookie: sid=abcdef123456789' https://x",
- "curl -b 'sessionid=abcdef123456789' --cookie sid=abcdef123456789 https://x",
- }
- for _, input := range inputs {
+ for _, input := range redactIdempotenceInputs {
once := RedactString(input)
if twice := RedactString(once); twice != once {
t.Fatalf("RedactString is not idempotent:\nonce %q\ntwice %q", once, twice)
@@ -247,6 +239,81 @@ func TestRedactStringIsIdempotent(t *testing.T) {
}
}
+// A label hidden inside a value a rule kept is redacted in the same call, as a second call would.
+func TestRedactStringRedactsALabelAKeptValueHid(t *testing.T) {
+ for input, want := range map[string]string{
+ `token:seCret":token":0`: `token:[REDACTED]":token":"[REDACTED]"`,
+ `--token "secret": "token": "abc"`: `--token "[REDACTED]": "token": "[REDACTED]"`,
+ `--password seCret":token":abcdefg`: `--password [REDACTED]":token":"[REDACTED]"`,
+ } {
+ if got := RedactString(input); got != want {
+ t.Errorf("RedactString(%q) = %q, want %q", input, got, want)
+ }
+ }
+}
+
+// redactIdempotenceInputs are texts one pass must leave nothing in for a second.
+// FuzzRedactIsIdempotent starts from them too.
+var redactIdempotenceInputs = []string{
+ "config dump\napi_key=sk-abcdefghijklmnopqrstuvwxyz0123\nOPENAI sk-abcdefghijklmnopqrstuvwxyz0123",
+ `{"token": "opaque-session-value-9", "password": "hunter2hunter2"}`,
+ "Authorization: Bearer abc.def.ghi and --password hunter2hunter2",
+ "postgres://app:pa55w0rd@db/app",
+ "curl -u admin:hunter2hunter2 -H 'Cookie: sid=abcdef123456789' https://x",
+ "curl -b 'sessionid=abcdef123456789' --cookie sid=abcdef123456789 https://x",
+ // A URL whose user name is a secret-named label: the label rules and the URL rule stop at
+ // different bytes, and each used to take one more piece per pass.
+ "https://x-access-token:[email protected]_api_`@host",
+ "A0000000000://token:,@",
+ "A://token:,@0,@0,@",
+ // A curl flag's blanks run over a newline to its value; the command follows a key.
+ "npm_000000000000000000000000000000000000Curl -B\n0=",
+ // The value captured after the quoted label is too short to remove until the URL rule
+ // replaces the comma that ends it.
+ `seCret":}A://:0,@`,
+ // A kept value that is itself a label: `token`, an auth scheme, is kept as the value of
+ // `seCret":` and hides `token":0` from the rule, until a later rule replaces `seCret` (here
+ // the value of `token:`, a flag's value, a GitHub token's tail) and a second pass finds it.
+ `token:seCret":token":0`,
+ `token:seCret":token":abc`,
+ `token:seCret':token':0`,
+ `token:seCret":Token":0`,
+ `--password seCret":token":0`,
+ `--token "secret": "token": "abc"`,
+ `ghp_0123456789abcdefghijklmnopqrstsecret":token":0`,
+ // Two such labels in turn, and one that took three passes to settle.
+ `token:seCret":token":seCret2": "token": "abc"`,
+ `0secret=token :ghp_0123456789abcdefghijklmnopqrstsecret=token :seCret':token':@h'`,
+ // The capture of assigned values takes `token` as the value of `secret` and so never reads
+ // `token`:}token:--token` as an assignment; the label that hid it is replaced.
+ "token:abctoken\":`secret`:`token`:}token:--token \"token\"",
+ // A flag finds no value where a copy of an assigned value that starts with a dash follows it;
+ // once the copy is replaced, a second pass redacted the marker and what follows it.
+ `-pAsswd --token -token:"--token `,
+ "--password --token ghp_0123456789abcdefghijklmnopqrsta=@h:token:'--token ",
+ // An AWS key id that a copy of an assigned value follows ends a word only once the copy is
+ // replaced, after every rule has run.
+ "AKIAIOSFODNN7EXAMPLEAuthorization: Authorization: ",
+ "@AKIAIOSFODNN7EXAMPLEx-access-token:x-access-token:",
+ "AKIAIOSFODNN7EXAMPLEAKIAIOSFODNN7EXAMPLEAuthorization: Authorization: ",
+ // The copy of `Authoriz` makes `ABIA[REDACTED]` of a label, which ends the key id before it.
+ "ABIA0000000000000000ABIAAuthorizAtion:Authoriz",
+ // The copy of `https://` holds the slash a URL's password stops at, and its marker does not.
+ "A://token:https://,ghp_0123456789abcdefghijklmnopqrst==https://@h\n=",
+ // A token runs on over that copy (overCopy), which takes its slash as replacing it would.
+ "A://token:https://,ghp_0000000000000000000000000https://@",
+ // A URL's password stops at the `/` of an AWS secret until the secret is replaced.
+ "https://u:a,AKIAIOSFODNN7EXAMPLE,wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY@host",
+ // The same where a token's item takes the key id and secret, and where they come after a
+ // replaced copy (redactAfterMarkers).
+ "https://u:pw,sk-abcAKIAIOSFODNN7EXAMPLE,wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY@ host",
+ "token=abcdefgh12345678 https://u:pw,abcdefgh12345678AKIAIOSFODNN7EXAMPLE,wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY@ host",
+ // The copy the key id ends before is followed by a marker already: what follows a word byte
+ // is told by place, not by count.
+ "token=abcdefgh12345678 AKIAIOSFODNN7EXAMPLEabcdefgh12345678[REDACTED]",
+ "https://x-access-token:a@b,AKIAIOSFODNN7EXAMPLE,wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY@host",
+}
+
// A field named for a credential is one, whatever its value looks like.
func TestSanitizeMapRedactsValuesUnderSecretKeys(t *testing.T) {
input := map[string]interface{}{
diff --git a/pkg/asymptoteobserve/redact_adjacent.go b/pkg/asymptoteobserve/redact_adjacent.go
new file mode 100644
index 00000000..a4a9f70a
--- /dev/null
+++ b/pkg/asymptoteobserve/redact_adjacent.go
@@ -0,0 +1,1246 @@
+package asymptoteobserve
+
+import (
+ "regexp"
+ "regexp/syntax"
+ "slices"
+ "sort"
+ "strconv"
+ "strings"
+ "unicode/utf8"
+)
+
+// This file is how redaction finds a secret written right after another one.
+//
+// The rules for credentials recognized by their own shape (privacy.go: `\b(ghp_)[A-Za-z0-9]{30,}`
+// and the rest) open with \b, so they match a secret that starts a word. A secret written right
+// after another one does not start a word, and a pass of the rules used to leave it:
+//
+// - A pattern of fixed length stops inside a run of word characters. Of two Google keys back to
+// back, `AIza…AIza…`, the second has a letter before it. Once the first is replaced, `]` is
+// before it, so a second pass redacted what the first left.
+// - An open-ended pattern runs on into the next secret and stops at the first byte its class
+// lacks. Of two GitHub tokens back to back, the first match takes the second's `ghp` and stops
+// at its `_`, and the rest of the second token was left whole by every pass.
+// - A pattern that ends in \b or in a byte that is no key character, the AWS key id's and the
+// AWS secret's, does not match where the next secret follows it with nothing between them.
+// - A rule that runs before the shape rules, for a URL's password, a Bearer token or a curl
+// password, finds no word start after a key that has not been replaced yet; nor does any rule
+// after a bare copy of an assigned value, which is replaced after every rule has run.
+//
+// So the end of a secret counts as the start of a word, and one pass redacts all of it:
+//
+// - When a shape rule matches, redactChain.run looks at what follows. If the match ran on into
+// a secret that starts inside it and ends past it, the match is cut where that secret starts,
+// or, too short to be cut there, runs on to that secret's end; if a secret of any shape rule
+// starts right where the match ends, with no word boundary between them, it is redacted with
+// it; and so on along the chain. Each secret keeps its own prefix: `ghp_[REDACTED]ghp_[REDACTED]`.
+// - A pattern that fails only for the secret right after it matches with the text cut where
+// that secret starts (redactChain.rescue).
+// - After every rule has run and bare copies are replaced, the rules that open with \b and are
+// not label rules are tried once more where a replacement ends right before a word byte
+// (redactAfterMarkers).
+//
+// Text where no secret meets another redacts as it did. Every check reads a slice whose length
+// is bounded by the pattern's own shape, or text that the chain then replaces, so redaction's
+// work stays linear; TestRedactionWorkIsLinear counts it.
+
+// redactShapeKind says whether a rule is a shape rule (redactionRule.shape), and which kind.
+type redactShapeKind uint8
+
+const (
+ shapeNone redactShapeKind = iota
+ shapeToken
+ // shapeJWT: three dot-separated segments of any length, which redactChain reads itself.
+ shapeJWT
+)
+
+// redactShape is what the adjacency checks know of one shape rule's pattern.
+type redactShape struct {
+ rule int
+ needles uint64 // scanner needle bits
+ offset int // the rule's offset: how many bytes before its needle a match starts
+ // tail, when set, is the class of the open-ended repeat the pattern ends in, at least tailMin
+ // long, after at most head bytes. Without it the pattern matches at most bound-1 bytes.
+ tail *[256]bool
+ tailMin int
+ head int
+ // bound is how long a slice from a match's start must be for the pattern to find in it what
+ // it finds in the whole text: the same match, or, for an open-ended pattern, one that reaches
+ // the slice's end where the whole match runs on (redactChain.shapeAt).
+ bound int
+ jwt bool
+ // redacted matches the rule's replacement at the start of a text: its first group, at most
+ // prefix bytes, and a marker.
+ redacted *regexp.Regexp
+ prefix int
+ // trailing is the group the replacement ends with, or 0 when it ends in a marker.
+ trailing int
+}
+
+// redactFollower is a rule that opens with \b and is neither a label rule nor a shape rule: a
+// Bearer token, a curl password or cookie, a URL's password. redactAfterMarkers tries it at the
+// start of a word that a replacement made.
+type redactFollower struct {
+ rule int
+ needles uint64
+ kind redactFollowerKind
+ // charset is every byte the pattern can match, for followRun.
+ charset [256]bool
+ // lead, for followLine, is the pattern anchored at its needle with the lazy run after the
+ // command word captured as `lead`: from any later place on the line up to that run's end, the
+ // pattern ends where it ended from this one (lineFollowerAt).
+ lead *regexp.Regexp
+}
+
+type redactFollowerKind uint8
+
+const (
+ // followRun: a word (its one needle), blanks, and a value: the match lies in the needle, the
+ // blanks after it and the run of its charset without blanks after them (Bearer).
+ followRun redactFollowerKind = iota
+ // followScheme: a URL, which holds no `/` and no whitespace after its `://`.
+ followScheme
+ // followLine: a curl rule, which reads its whole command line and is applied to it until it
+ // stops changing (redactLinesAfterMarkers).
+ followLine
+)
+
+// redactAdjacency is what the adjacency checks know of the rules, built the first time a chain or
+// a marker is looked at.
+type redactAdjacency struct {
+ shapes []redactShape // in rule order
+ followers []redactFollower
+ byRule []int // rule index -> index into shapes, or -1
+}
+
+func (e *redactEngine) adjacency() *redactAdjacency {
+ e.adjacentOnce.Do(func() {
+ a := &redactAdjacency{byRule: make([]int, len(redactionRules))}
+ for i, rule := range redactionRules {
+ a.byRule[i] = -1
+ parsed, err := syntax.Parse(rule.expr, syntax.Perl)
+ if err != nil {
+ panic(err)
+ }
+ if rule.shape != shapeNone {
+ a.byRule[i] = len(a.shapes)
+ a.shapes = append(a.shapes, shapeOf(i, rule, parsed, e.setMasks[rule.needles.id]))
+ continue
+ }
+ if !opensWithBoundary(rule.expr) || rule.start == startLabel {
+ continue
+ }
+ if len(a.shapes) > 0 {
+ panic("asymptoteobserve: a rule that opens with \\b must come before the shape rules: " + rule.expr)
+ }
+ f := redactFollower{rule: i, needles: e.setMasks[rule.needles.id]}
+ switch {
+ case rule.start == startScheme:
+ f.kind = followScheme
+ case rule.repeat:
+ f.kind = followLine
+ f.lead = leadPattern(rule)
+ default:
+ f.kind = followRun
+ addCharset(&f.charset, parsed)
+ for c := utf8.RuneSelf; c < 256; c++ {
+ f.charset[c] = true
+ }
+ if f.charset['['] || f.charset['\n'] || len(rule.needles.needles) != 1 ||
+ !strings.Contains(rule.expr, `\b`+rule.needles.needles[0]+`[ \t]+`) {
+ panic("asymptoteobserve: a run follower has one needle, a blank after it, and cannot match '[' or a newline: " + rule.expr)
+ }
+ }
+ a.followers = append(a.followers, f)
+ }
+ e.adjacent = a
+ })
+ return e.adjacent
+}
+
+// opensWithBoundary reports whether a pattern opens with \b, as redactSlot.prepare reads it.
+func opensWithBoundary(expr string) bool {
+ return slices.ContainsFunc(boundaryPrefixes, func(prefix string) bool { return strings.HasPrefix(expr, prefix) })
+}
+
+// leadPattern is a followLine rule's pattern anchored at its one needle, without the \b it opens
+// with, and with the lazy run after the needle captured as `lead`.
+func leadPattern(rule redactionRule) *regexp.Regexp {
+ const run = `\bcurl\b[^\n]*?`
+ if len(rule.needles.needles) != 1 || !strings.Contains(rule.expr, "("+run) {
+ panic("asymptoteobserve: a line follower opens with `curl` and the lazy run of its line: " + rule.expr)
+ }
+ return regexp.MustCompile("^" + strings.Replace(rule.expr, "("+run, `(curl\b(?P<lead>[^\n]*?)`, 1))
+}
+
+// shapeOf reads a shape rule's pattern: a concatenation that ends, before any trailing assertion,
+// either in an open-ended repeat of one class or in something of bounded length.
+func shapeOf(index int, rule redactionRule, parsed *syntax.Regexp, needles uint64) redactShape {
+ s := redactShape{rule: index, needles: needles, offset: rule.offset, jwt: rule.shape == shapeJWT}
+ parts := []*syntax.Regexp{parsed}
+ if parsed.Op == syntax.OpConcat {
+ parts = parsed.Sub
+ }
+ for len(parts) > 0 && maxLength(parts[len(parts)-1]) == 0 {
+ parts = parts[:len(parts)-1]
+ }
+ last := parts[len(parts)-1]
+ if last.Op == syntax.OpRepeat && last.Max == -1 && last.Sub[0].Op == syntax.OpCharClass {
+ s.tail = new([256]bool)
+ addCharset(s.tail, last.Sub[0])
+ s.tailMin = last.Min
+ s.head = 0
+ for _, part := range parts[:len(parts)-1] {
+ if n := maxLength(part); n < 0 || s.head < 0 {
+ s.head = -1
+ } else {
+ s.head += n
+ }
+ }
+ s.bound = s.head + s.tailMin
+ } else if n := maxLength(parsed); n >= 0 {
+ s.bound = n + 1
+ }
+ group := firstGroup(parsed)
+ if !strings.HasSuffix(rule.replacement, redactedValue) {
+ i := strings.LastIndex(rule.replacement, "${")
+ if i >= 0 && strings.HasSuffix(rule.replacement, "}") {
+ s.trailing, _ = strconv.Atoi(rule.replacement[i+2 : len(rule.replacement)-1])
+ }
+ }
+ if s.jwt != (s.head < 0) || s.tail == nil && s.bound == 0 || group == nil || maxLength(group) < 0 ||
+ s.trailing == 0 && !strings.HasSuffix(rule.replacement, redactedValue) {
+ panic("asymptoteobserve: a shape rule's pattern has a shape the adjacency checks do not know: " + rule.expr)
+ }
+ s.prefix = maxLength(group)
+ s.redacted = regexp.MustCompile(`^(?:` + group.Sub[0].String() + `)` + regexp.QuoteMeta(redactedValue))
+ return s
+}
+
+// firstGroup returns re's first capturing group, or nil.
+func firstGroup(re *syntax.Regexp) *syntax.Regexp {
+ if re.Op == syntax.OpCapture {
+ return re
+ }
+ for _, sub := range re.Sub {
+ if group := firstGroup(sub); group != nil {
+ return group
+ }
+ }
+ return nil
+}
+
+// maxLength returns the most bytes re can match, or -1 when that is unbounded.
+func maxLength(re *syntax.Regexp) int {
+ switch re.Op {
+ case syntax.OpLiteral:
+ n := 0
+ for _, r := range re.Rune {
+ n += utf8.RuneLen(r)
+ }
+ return n
+ case syntax.OpCharClass:
+ if len(re.Rune) > 0 && re.Rune[len(re.Rune)-1] < utf8.RuneSelf {
+ return 1
+ }
+ return utf8.UTFMax
+ case syntax.OpAnyChar, syntax.OpAnyCharNotNL:
+ return utf8.UTFMax
+ case syntax.OpCapture, syntax.OpQuest:
+ return maxLength(re.Sub[0])
+ case syntax.OpConcat, syntax.OpAlternate:
+ n := 0
+ for _, sub := range re.Sub {
+ m := maxLength(sub)
+ if m < 0 {
+ return -1
+ }
+ if re.Op == syntax.OpConcat {
+ n += m
+ } else {
+ n = max(n, m)
+ }
+ }
+ return n
+ case syntax.OpRepeat:
+ if m := maxLength(re.Sub[0]); re.Max >= 0 && m >= 0 {
+ return re.Max * m
+ }
+ return -1
+ case syntax.OpStar, syntax.OpPlus:
+ return -1
+ }
+ return 0 // an assertion or an empty match
+}
+
+// addCharset adds every ASCII byte re can match to set.
+func addCharset(set *[256]bool, re *syntax.Regexp) {
+ switch re.Op {
+ case syntax.OpLiteral:
+ for _, r := range re.Rune {
+ if r < utf8.RuneSelf {
+ set[r] = true
+ if re.Flags&syntax.FoldCase != 0 {
+ set[asciiLower(byte(r))] = true
+ set[asciiUpper(byte(r))] = true
+ }
+ }
+ }
+ case syntax.OpCharClass:
+ for i := 0; i+1 < len(re.Rune); i += 2 {
+ for r := re.Rune[i]; r <= re.Rune[i+1] && r < utf8.RuneSelf; r++ {
+ set[r] = true
+ }
+ }
+ case syntax.OpAnyChar, syntax.OpAnyCharNotNL:
+ for c := range utf8.RuneSelf {
+ set[c] = c != '\n' || re.Op == syntax.OpAnyChar
+ }
+ }
+ for _, sub := range re.Sub {
+ addCharset(set, sub)
+ }
+}
+
+func asciiUpper(c byte) byte {
+ if 'a' <= c && c <= 'z' {
+ return c - 'a' + 'A'
+ }
+ return c
+}
+
+// jwtSegment is the class of a JWT segment's bytes.
+var jwtSegment = func() (set [256]bool) {
+ for c := range 256 {
+ b := byte(c)
+ set[c] = isASCIILetter(b) || '0' <= b && b <= '9' || b == '_' || b == '-'
+ }
+ return set
+}()
+
+// redactChain finds the secrets written right after a shape rule's matches in one text.
+type redactChain struct {
+ engine *redactEngine
+ adj *redactAdjacency
+ t string
+ hits []redactHit
+ // The end of the run of JWT segment bytes from segFrom, and the end of a JWT whose first
+ // segment ends at restAt (-1: none): each is read once however many places ask.
+ segFrom, segEnd int
+ restAt, restEnd int
+ // What secretAt found where it was asked.
+ secrets map[int]redactSecret
+ // The end of the run of URL scheme bytes from schemeFrom, and the URL follower's match from
+ // the run's first place asked (schemeAt; nil: none), which any letter of the run shares.
+ schemeFrom, schemeTo int
+ schemeAt int
+ schemeMatch []int
+ // The end of the line holding lineFrom; and, per followLine follower, the places on a line
+ // where it finds nothing (from the earliest place where it found nothing to the end of that
+ // place's line) and where it finds the match it found from an earlier place (from that place
+ // to the end of its lead, and where the match ends).
+ lineFrom, lineTo int
+ lineFirst int // the end of lineFrom's own line, at or before lineTo
+ lineFailed [][2]int
+ lineFound [][3]int
+ // The end of the run of a followRun follower's charset from runFrom, and per follower the
+ // last place followerAt was asked about and its answer.
+ runFrom, runTo int
+ asked []redactAsked
+ // The assigned values whose copies are removed after every rule has run (overCopy), what
+ // running on over them may still read (copyBudget), and whether running on over one, or
+ // over an AWS secret that holds a `/`, may hide a match from this pass, as replacing a copy
+ // may (copyHidesMatch); matches reports it to the gate.
+ copies *redactCopies
+ copyBudget int
+ hid bool
+ slashed bool
+}
+
+// redactAsked is followerAt's answer for one follower at a place.
+type redactAsked struct {
+ at int
+ m []int
+}
+
+func newRedactChain(engine *redactEngine, t string, hits []redactHit) *redactChain {
+ return &redactChain{engine: engine, adj: engine.adjacency(), t: t, hits: hits, segFrom: -1, restAt: -1,
+ schemeFrom: -1, schemeTo: -1, schemeAt: -1, lineFrom: -1, lineTo: -1, lineFirst: -1, runFrom: -1, runTo: -1,
+ copyBudget: redactCopyWorkPerByte*len(t) + redactCopyWorkBase}
+}
+
+// overCopy may read redactCopyWorkPerByte bytes and copies per byte of the text, and
+// redactCopyWorkBase more; past that it leaves items as they are, as main does.
+const (
+ redactCopyWorkPerByte = 8
+ redactCopyWorkBase = 1 << 16
+)
+
+func redactWorked(n int) {
+ if redactWork != nil {
+ redactWork(n)
+ }
+}
+
+// pattern is rule i's pattern anchored at the start of the text, without its opening \b.
+func (c *redactChain) pattern(i int) *regexp.Regexp {
+ slot := &c.engine.rules[i]
+ slot.prepare(redactionRules[i])
+ return slot.anchored
+}
+
+// find runs rule i's anchored pattern on t[q:end].
+func (c *redactChain) find(i, q, end int) []int {
+ redactWorked(end - q)
+ m := c.pattern(i).FindStringSubmatchIndex(c.t[q:end])
+ for j := range m {
+ if m[j] >= 0 {
+ m[j] += q
+ }
+ }
+ return m
+}
+
+// search returns the first match of re, a shape rule's whole pattern, in t[pos:], as indexes into
+// t. It reads up to the match's end.
+func (c *redactChain) search(re *regexp.Regexp, pos int) []int {
+ m := re.FindStringSubmatchIndex(c.t[pos:])
+ if m == nil {
+ redactWorked(len(c.t) - pos)
+ return nil
+ }
+ redactWorked(m[1])
+ for j := range m {
+ if m[j] >= 0 {
+ m[j] += pos
+ }
+ }
+ return m
+}
+
+// hitsFrom returns the index of the first hit that starts at or after pos.
+func hitsFrom(hits []redactHit, pos int) int {
+ return sort.Search(len(hits), func(i int) bool { return hits[i].start >= pos })
+}
+
+// hasNeedleAt reports whether one of needles occurs at pos.
+func (c *redactChain) hasNeedleAt(needles uint64, pos int) bool {
+ for i := hitsFrom(c.hits, pos); i < len(c.hits) && c.hits[i].start == pos; i++ {
+ if needles&(1<<c.hits[i].id) != 0 {
+ return true
+ }
+ }
+ return false
+}
+
+// shapeAt runs shape si's pattern at q, without the \b it opens with, on a slice that decides it:
+// the match, and whether it runs on past the slice. Its needle must occur at q.
+func (c *redactChain) shapeAt(si, q int) (m []int, open bool) {
+ s := &c.adj.shapes[si]
+ if s.jwt {
+ if end := c.jwtEnd(q, q+3); end >= 0 {
+ return c.find(s.rule, q, end), false
+ }
+ return nil, false
+ }
+ end := min(len(c.t), q+s.bound)
+ m = c.find(s.rule, q, end)
+ return m, m != nil && s.tail != nil && m[1] == end && end < len(c.t) && s.tail[c.t[end]]
+}
+
+// whole returns the match shapeAt found, run on to its end where it is open.
+func (c *redactChain) whole(si, q int, m []int, open bool) []int {
+ if !open {
+ return m
+ }
+ s := &c.adj.shapes[si]
+ end := m[1]
+ for end < len(c.t) && s.tail[c.t[end]] {
+ end++
+ }
+ redactWorked(end - m[1])
+ return c.find(s.rule, q, end)
+}
+
+// rescue returns a match of shape si at q that ends where the next secret starts, or nil. A
+// pattern of bounded length that ends in an assertion or a terminator, the AWS key id's \b and the
+// AWS secret's byte that is no key character, fails where a secret follows with nothing between
+// them; cut where that secret starts, the text ends right after it, and it matches. The next
+// secret may already be replaced, by a rule that runs earlier. The pattern's own match at q, if
+// any, must have been tried first.
+func (c *redactChain) rescue(si, q int) []int {
+ s := &c.adj.shapes[si]
+ if s.tail != nil {
+ return nil
+ }
+ for i := hitsFrom(c.hits, q+1); i < len(c.hits) && c.hits[i].start < q+s.bound+1; i++ {
+ h := c.hits[i]
+ for sj := range c.adj.shapes {
+ x := h.start - c.adj.shapes[sj].offset
+ if c.adj.shapes[sj].needles&(1<<h.id) == 0 || x <= q || x >= q+s.bound {
+ continue
+ }
+ if m := c.find(s.rule, q, x); m != nil && m[1] == x {
+ if _, next := c.secretAt(x); next != nil || c.redactedAt(x) || c.followerStarts(x) {
+ return m
+ }
+ }
+ }
+ }
+ // A URL, Bearer token or curl command right after it.
+ for x := q + 1; x < min(len(c.t), q+s.bound); x++ {
+ if isASCIILetter(c.t[x]) && c.followerStarts(x) {
+ if m := c.find(s.rule, q, x); m != nil && m[1] == x {
+ return m
+ }
+ }
+ }
+ return nil
+}
+
+// redactedAt reports whether a shape rule's replacement starts at x: its prefix and a marker.
+func (c *redactChain) redactedAt(x int) bool {
+ for si := range c.adj.shapes {
+ s := &c.adj.shapes[si]
+ if !c.hasNeedleAt(s.needles, x+s.offset) {
+ continue
+ }
+ end := min(len(c.t), x+s.prefix+len(redactedValue))
+ redactWorked(end - x)
+ if s.redacted.MatchString(c.t[x:end]) {
+ return true
+ }
+ }
+ return false
+}
+
+// rescueBefore returns the first match rescue finds at a place of shape si from pos up to limit,
+// where the pattern opens on a word boundary, or nil.
+func (c *redactChain) rescueBefore(si, pos, limit int) []int {
+ s := &c.adj.shapes[si]
+ if s.tail != nil {
+ return nil
+ }
+ for i := hitsFrom(c.hits, pos+s.offset); i < len(c.hits) && c.hits[i].start-s.offset < limit; i++ {
+ q := c.hits[i].start - s.offset
+ if s.needles&(1<<c.hits[i].id) == 0 || !wordBoundary(c.t, q) {
+ continue
+ }
+ if m := c.rescue(si, q); m != nil {
+ return m
+ }
+ }
+ return nil
+}
+
+// secretAt returns the first shape rule, in rule order, whose pattern matches at p or is rescued
+// there, and its match: the secret that starts at p when p counts as the start of a word. A rescue
+// asks what starts where its match ends, so along a chain of key ids each place is asked about
+// more than once; the answers are kept.
+func (c *redactChain) secretAt(p int) (int, []int) {
+ if found, ok := c.secrets[p]; ok {
+ return found.shape, found.match
+ }
+ shape, match := -1, []int(nil)
+ for si := range c.adj.shapes {
+ s := &c.adj.shapes[si]
+ if !c.hasNeedleAt(s.needles, p+s.offset) {
+ continue
+ }
+ if m, open := c.shapeAt(si, p); m != nil {
+ shape, match = si, c.whole(si, p, m, open)
+ break
+ }
+ if m := c.rescue(si, p); m != nil {
+ shape, match = si, m
+ break
+ }
+ }
+ if c.secrets == nil {
+ c.secrets = map[int]redactSecret{}
+ }
+ c.secrets[p] = redactSecret{shape, match}
+ return shape, match
+}
+
+// redactSecret is what secretAt found at a place.
+type redactSecret struct {
+ shape int
+ match []int
+}
+
+// use is how shape si's matches are yielded: its anchored pattern and its rule.
+func (c *redactChain) use(si int) redactUse {
+ i := c.adj.shapes[si].rule
+ return redactUse{c.pattern(i), &redactionRules[i]}
+}
+
+// run yields m, a match of shape si, and the secrets written right after it, and returns where
+// the last of them ends, or false when yield asked to stop.
+//
+// A secret that starts inside the match and ends past it (crossing) is one the match ran on into.
+// Where the match is still whole without the part from that secret on, it is cut there and the
+// secret is the next item. Where it is not, the two are one item, run on to the second's end: of
+// `sk-eyJhbGciOiJIUzI1NiJ9.eyJ…`, the key's pattern takes the JWT's first segment and stops at
+// the dot, too short a key without it, and the item takes the whole JWT.
+func (c *redactChain) run(si int, m []int, use redactUse, yield func([]int, redactUse) bool) (int, bool) {
+ // item is what is yielded, with use and the groups of shape. si and m are the secret it ends
+ // with, item itself unless merged. least is the first place item can be cut and stay whole.
+ item, shape, merged := m, si, false
+ least := c.least(si, m)
+ for {
+ q, sj, sm := -1, 0, []int(nil)
+ if c.adj.shapes[shape].trailing == 0 && c.adj.shapes[si].trailing == 0 {
+ q, sj, sm = c.crossing(si, m)
+ }
+ if q >= 0 {
+ var cut []int
+ if q >= least && merged {
+ cut = slices.Clone(item)
+ cut[1] = q
+ } else if q >= least {
+ if cut = c.find(c.adj.shapes[shape].rule, item[0], q); cut != nil && cut[1] != q {
+ cut = nil
+ }
+ }
+ if cut != nil {
+ if !yield(cut, use) {
+ return 0, false
+ }
+ item, shape, merged, use, least = sm, sj, false, c.use(sj), c.least(sj, sm)
+ si, m = sj, sm
+ continue
+ }
+ // The item keeps all of itself so far, and a cut can now fall only past that. What the
+ // secret's own replacement would keep after it, it keeps.
+ least, merged = item[1], true
+ item = slices.Clone(item)
+ item[1] = sm[1]
+ if g := c.adj.shapes[sj].trailing; g > 0 && sm[2*g] >= 0 {
+ item[1] = sm[2*g]
+ }
+ // The item is yielded with the first secret's rule: an AWS secret it takes is not seen
+ // as one where the replacements are made (redactionRule.applyOnce).
+ c.slashed = c.slashed || slashedAWSSecret(redactionRules[c.adj.shapes[sj].rule].expr, c.t, sm)
+ si, m = sj, sm
+ continue
+ }
+ // A URL, Bearer token or curl command whose first word the match took: cut the match
+ // before it, and the follower is redacted where the marker ends (redactAfterMarkers); too
+ // short to be cut there, the item runs on over the follower's match.
+ if c.adj.shapes[shape].trailing == 0 {
+ if q, fend := c.followerCrossing(m); q >= 0 {
+ var cut []int
+ if q >= least && merged {
+ cut = slices.Clone(item)
+ cut[1] = q
+ } else if q >= least {
+ if cut = c.find(c.adj.shapes[shape].rule, item[0], q); cut != nil && cut[1] != q {
+ cut = nil
+ }
+ }
+ if cut == nil {
+ cut = slices.Clone(item)
+ cut[1] = fend
+ }
+ return cut[1], yield(cut, use)
+ }
+ }
+ if c.adj.shapes[shape].trailing == 0 {
+ item = c.overCopy(item)
+ }
+ if !yield(item, use) {
+ return 0, false
+ }
+ end := item[1]
+ // A replacement that keeps what follows the secret does not end in a marker.
+ if g := c.adj.shapes[shape].trailing; g > 0 && item[2*g] < item[2*g+1] ||
+ end == len(c.t) || !isWordByte(c.t[end-1]) || !isWordByte(c.t[end]) {
+ return end, true
+ }
+ sj, sm = c.secretAt(end)
+ if sm == nil {
+ return end, true
+ }
+ si, m, item, shape, merged, use, least = sj, sm, sm, sj, false, c.use(sj), c.least(sj, sm)
+ }
+}
+
+// overCopy returns item run on to the end of a copy of an assigned value that starts inside it and
+// ends past it, the item having taken the copy's first bytes; or item. The copy is removed after
+// every rule has run, which finds it only whole: of `hf_…opaque-flag-value` with `opaque-flag-value`
+// assigned, the token's pattern takes `opaque`, and `-flag-value` was left. Such a copy starts
+// less than the longest copy's length before the item's end. What it reads, places, copies looked
+// at and bytes compared, comes out of the chain's copyBudget, linear in the text; once that is
+// spent, items are left as they are.
+func (c *redactChain) overCopy(item []int) []int {
+ if c.copies == nil || c.copyBudget <= 0 {
+ return item
+ }
+ t, work := c.t, 0
+ for x := max(item[0]+1, item[1]-c.copies.longest+1); x < item[1] && x+redactCopyMin <= len(t) && work < c.copyBudget; x++ {
+ work++
+ for _, v := range c.copies.byFirst[t[x:x+redactCopyMin]] {
+ work++
+ if x+len(v) <= item[1] {
+ continue
+ }
+ work += len(v)
+ if strings.HasPrefix(t[x:], v) {
+ item = slices.Clone(item)
+ item[1] = x + len(v)
+ c.hid = true
+ }
+ }
+ }
+ c.copyBudget -= work
+ redactWorked(work)
+ return item
+}
+
+// slashedAWSSecret reports whether m, a match of the pattern expr, is one of the rule for an AWS
+// key id and its secret whose secret holds a `/`, where the URL rule, which runs before the shape
+// rules, stopped a password: once the secret is replaced, the password can run on to its `@`.
+func slashedAWSSecret(expr, t string, m []int) bool {
+ return expr == redactAWSPair && strings.IndexByte(t[m[5]:m[6]], '/') >= 0
+}
+
+// least returns the first place m, a match of shape si, can be cut and still be a whole match: as
+// far into its open-ended tail as the tail's least length. A pattern without one cannot be cut.
+func (c *redactChain) least(si int, m []int) int {
+ s := &c.adj.shapes[si]
+ if s.tail == nil {
+ return len(c.t) + 1
+ }
+ tail := m[1]
+ for tail > m[3] && s.tail[c.t[tail-1]] {
+ tail--
+ }
+ redactWorked(m[1] - tail)
+ return tail + s.tailMin
+}
+
+// crossing looks for a secret that starts inside m, a match of shape si, and ends past it: the
+// match ran on into the next secret and stopped at a byte that secret holds, or the secret
+// started inside a match of fixed length. It returns where the leftmost such secret starts, its
+// shape and its match; or -1.
+func (c *redactChain) crossing(si int, m []int) (int, int, []int) {
+ t, start, end := c.t, m[0], m[1]
+ if end >= len(t) {
+ return -1, 0, nil
+ }
+ stop := t[end]
+ best, bestShape := end, -1
+ var bestMatch []int
+ bestOpen := false
+ for sj := range c.adj.shapes {
+ s := &c.adj.shapes[sj]
+ // run is where the run of the class that could carry s across end starts.
+ run := end
+ var class *[256]bool
+ switch {
+ case s.jwt && (jwtSegment[stop] || stop == '.'):
+ class = &jwtSegment
+ case s.tail != nil && s.tail[stop]:
+ class = s.tail
+ }
+ if class != nil {
+ for run > start+1 && class[t[run-1]] {
+ run--
+ }
+ redactWorked(end - run)
+ }
+ lo := run
+ switch {
+ case s.jwt:
+ if class == nil {
+ continue
+ }
+ case s.tail == nil:
+ lo = end - s.bound
+ default:
+ lo -= s.head
+ }
+ lo = max(lo, start+1)
+ for i := hitsFrom(c.hits, lo+s.offset); i < len(c.hits); i++ {
+ h := c.hits[i]
+ q := h.start - s.offset
+ if q >= best {
+ break
+ }
+ if s.needles&(1<<h.id) == 0 || q < lo {
+ continue
+ }
+ if s.jwt {
+ // The leftmost `eyJ` in the run: a later one has a shorter first segment and the
+ // same rest. Its first segment runs on from end.
+ if jwt := c.jwtEnd(q, end); jwt > end {
+ best, bestShape, bestMatch, bestOpen = q, sj, nil, false
+ }
+ break
+ }
+ sm, open := c.shapeAt(sj, q)
+ if sm == nil {
+ continue
+ }
+ // An open match whose tail runs on unbroken to end crosses it too.
+ if sm[1] > end || open && class != nil && run <= sm[1] {
+ best, bestShape, bestMatch, bestOpen = q, sj, sm, open
+ break
+ }
+ }
+ }
+ if bestShape < 0 {
+ return -1, 0, nil
+ }
+ if bestMatch == nil {
+ bestMatch, bestOpen = c.shapeAt(bestShape, best)
+ }
+ return best, bestShape, c.whole(bestShape, best, bestMatch, bestOpen)
+}
+
+// jwtEnd returns where the JWT that starts at q ends, or -1. Its first segment holds only segment
+// bytes from q+3 to from.
+func (c *redactChain) jwtEnd(q, from int) int {
+ first := c.segmentEnd(from)
+ if first-(q+3) < 8 {
+ return -1
+ }
+ if c.restAt == first {
+ return c.restEnd
+ }
+ t, end := c.t, -1
+ if strings.HasPrefix(t[first:], ".eyJ") {
+ second := segmentEnd(t, first+4)
+ if second-(first+4) >= 8 && second < len(t) && t[second] == '.' {
+ if third := segmentEnd(t, second+1); third-(second+1) >= 8 {
+ end = third
+ }
+ }
+ }
+ c.restAt, c.restEnd = first, end
+ return end
+}
+
+// segmentEnd returns the end of the run of JWT segment bytes from from.
+func (c *redactChain) segmentEnd(from int) int {
+ if c.segFrom <= from && from <= c.segEnd {
+ return c.segEnd
+ }
+ c.segFrom, c.segEnd = from, segmentEnd(c.t, from)
+ return c.segEnd
+}
+
+func segmentEnd(t string, from int) int {
+ end := from
+ for end < len(t) && jwtSegment[t[end]] {
+ end++
+ }
+ redactWorked(end - from)
+ return end
+}
+
+// markerBefore returns the end of the first marker at or after from that a word byte follows, or
+// -1.
+func markerBeforeWord(t string, from int) int {
+ for from < len(t) {
+ i := strings.Index(t[from:], redactedValue)
+ if i < 0 {
+ return -1
+ }
+ p := from + i + len(redactedValue)
+ if p < len(t) && isWordByte(t[p]) {
+ return p
+ }
+ from = p
+ }
+ return -1
+}
+
+// redactAfterMarkers redacts what starts right after a replacement, which starts a word only in
+// the redacted text: a URL's password, a Bearer token or a curl password right after a key, and
+// any secret right after a replaced copy of an assigned value. Where a replacement ends right
+// before a word byte, the followers and then the shape rules, in rule order, are tried there; the
+// first that matches is applied, and a shape rule's chain after it. Each place is tried once, left
+// to right, so a chain that alternates between them is redacted in one pass. The curl rules,
+// which read a whole command line and are applied to it until it stops changing, are then
+// applied to the line from the first such place where a curl command starts.
+//
+// It also reports whether it replaced an AWS secret that holds a `/` (slashedAWSSecret).
+func (e *redactEngine) redactAfterMarkers(t string) (string, bool) {
+ p := markerBeforeWord(t, 0)
+ if p < 0 {
+ return t, false
+ }
+ hits, _ := e.scanner.scan(t, nil)
+ c := newRedactChain(e, t, hits)
+ var out []byte
+ last := 0
+ emit := func(m []int, use redactUse) bool {
+ out = append(out, t[last:m[0]]...)
+ out = use.pattern.ExpandString(out, use.rule.replacement, t, m)
+ last = m[1]
+ c.slashed = c.slashed || slashedAWSSecret(use.rule.expr, t, m)
+ return true
+ }
+ for ; p >= 0; p = markerBeforeWord(t, max(p, last)) {
+ for p >= last && p < len(t) {
+ if m, use := c.followerAt(p); m != nil {
+ // The shape rules run after the followers, also over what a follower keeps of
+ // its match, ${1}: a URL whose scheme starts with a key.
+ replacement, end := use.rule.replacing(t, m)
+ var items [][]int
+ var uses []redactUse
+ if si, sm := c.secretAt(p); sm != nil && strings.HasPrefix(replacement, "${1}") {
+ if chained, _ := c.run(si, sm, c.use(si), func(m []int, use redactUse) bool {
+ items, uses = append(items, m), append(uses, use)
+ return true
+ }); chained > m[3] {
+ items = nil
+ }
+ }
+ for i := range items {
+ emit(items[i], uses[i])
+ }
+ if items != nil {
+ out = append(out, t[last:m[3]]...)
+ replacement = strings.TrimPrefix(replacement, "${1}")
+ } else {
+ out = append(out, t[last:m[0]]...)
+ }
+ out = use.pattern.ExpandString(out, replacement, t, m)
+ last = end
+ break
+ }
+ si, m := c.secretAt(p)
+ if m == nil {
+ break
+ }
+ end, _ := c.run(si, m, c.use(si), emit)
+ // The chain checked for a shape rule at its end; a follower may start there.
+ if end == len(t) || !isWordByte(t[end]) || len(out) == 0 || out[len(out)-1] != ']' {
+ break
+ }
+ p = end
+ }
+ }
+ if out != nil {
+ t = string(append(out, t[last:]...))
+ }
+ return e.redactLinesAfterMarkers(t), c.slashed
+}
+
+// commandLineEnd returns the end of what a curl rule can match from p: its line; the next one,
+// since the blank before a flag (`\s`) can be the newline, so the flag can start that line; and,
+// since the blanks after a flag (`\s*`) can run over newlines, the next line that is not blank.
+func commandLineEnd(t string, p int) int {
+ i := strings.IndexByte(t[p:], '\n')
+ if i < 0 {
+ return len(t)
+ }
+ next := p + i + 1
+ j := strings.IndexByte(t[next:], '\n')
+ if j < 0 {
+ return len(t)
+ }
+ next += j
+ for next < len(t) && isRegexpSpace(t[next]) {
+ next++
+ }
+ if k := strings.IndexByte(t[next:], '\n'); k >= 0 {
+ return next + k
+ }
+ return len(t)
+}
+
+// redactLinesAfterMarkers applies the followLine rules, each as redaction applies it, to every
+// line from the first place right after a marker where one of their needles starts.
+func (e *redactEngine) redactLinesAfterMarkers(t string) string {
+ adj := e.adjacency()
+ var out []byte
+ last := 0
+ for p := markerBeforeWord(t, 0); p >= 0; p = markerBeforeWord(t, max(p, last)) {
+ if p < last || !adj.startsLine(t, p) {
+ continue
+ }
+ // A segment whose last line holds another curl command runs on to that command's value.
+ end := commandLineEnd(t, p)
+ for end < len(t) {
+ ls := strings.LastIndexByte(t[:end], '\n') + 1
+ if ls <= p || !strings.Contains(strings.ToLower(t[ls:end]), "curl") {
+ break
+ }
+ end = commandLineEnd(t, ls)
+ }
+ line := t[p:end]
+ for _, f := range adj.followers {
+ if f.kind != followLine {
+ continue
+ }
+ text := redactGate{value: line, enabled: true, engine: e}
+ if rule := redactionRules[f.rule]; text.holds(rule.needles) {
+ line = rule.apply(&text, &e.rules[f.rule])
+ }
+ }
+ out = append(append(out, t[last:p]...), line...)
+ last = end
+ }
+ if out == nil {
+ return t
+ }
+ return string(append(out, t[last:]...))
+}
+
+// startsLine reports whether one of the followLine rules' needles starts at p.
+func (a *redactAdjacency) startsLine(t string, p int) bool {
+ for _, f := range a.followers {
+ if f.kind != followLine {
+ continue
+ }
+ for _, needle := range redactionRules[f.rule].needles.needles {
+ if len(t)-p >= len(needle) && strings.EqualFold(t[p:p+len(needle)], needle) {
+ return true
+ }
+ }
+ }
+ return false
+}
+
+// followerStarts reports whether a follower matches at x: a URL, a Bearer token or a curl command.
+func (c *redactChain) followerStarts(x int) bool {
+ if m, _ := c.followerAt(x); m != nil {
+ return true
+ }
+ return c.lineFollowerAt(x) != nil
+}
+
+// followerCrossing returns where a follower starts whose first word m took, the match running on
+// over a URL's scheme, `Bearer` or `curl` and stopping at the byte after it, and where the
+// follower's match ends; or -1.
+func (c *redactChain) followerCrossing(m []int) (int, int) {
+ t, start, e := c.t, m[0], m[1]
+ if e >= len(t) || e-1 <= start || c.followerStarts(e) {
+ return -1, 0
+ }
+ // The whole scheme: where the match stopped inside it, at `.`, `-` or `+`, the URL rule found a
+ // scheme of its own after that byte.
+ if isASCIILetter(t[e-1]) && strings.HasPrefix(t[e:], "://") {
+ if fm, _ := c.followerAt(e - 1); fm != nil && fm[1] > e {
+ return e - 1, fm[1]
+ }
+ }
+ for _, f := range c.adj.followers {
+ if f.kind == followScheme {
+ continue
+ }
+ for _, needle := range redactionRules[f.rule].needles.needles {
+ q := e - len(needle)
+ if q <= start || !c.hasNeedleAt(f.needles, q) {
+ continue
+ }
+ if f.kind == followLine {
+ if fm := c.lineFollowerAt(q); fm != nil {
+ return q, fm[1]
+ }
+ } else if fm, _ := c.followerAt(q); fm != nil && fm[1] > e {
+ return q, fm[1]
+ }
+ }
+ }
+ return -1, 0
+}
+
+// schemeEnd returns the end of the run of URL scheme bytes from p: exactly what the URL pattern's
+// case-insensitive `[a-z0-9+.-]*` takes, the ASCII class and the two other characters that fold
+// into it, `K` (U+212A, k) and `ſ` (U+017F, s). schemeFollowerAt shares one answer across the run,
+// so a byte the pattern stops at must end it.
+func (c *redactChain) schemeEnd(p int) int {
+ if c.schemeFrom <= p && p <= c.schemeTo {
+ return c.schemeTo
+ }
+ t, end := c.t, p
+ for end < len(t) {
+ switch {
+ case schemeClass[t[end]]:
+ end++
+ continue
+ case strings.HasPrefix(t[end:], "\u212a"):
+ end += len("\u212a")
+ continue
+ case strings.HasPrefix(t[end:], "\u017f"):
+ end += len("\u017f")
+ continue
+ }
+ break
+ }
+ redactWorked(end - p)
+ c.schemeFrom, c.schemeTo = p, end
+ return end
+}
+
+// lineFollowerAt returns the match of the first followLine rule that matches at q, or nil (only
+// its start and end: the callers read no group). It hands the pattern slices that double until
+// the match ends inside one or one reaches the line's end. A place on a line where the needle
+// ends a word and the rule finds nothing means no later place on the line finds anything (from
+// the earlier `curl`, the pattern's lazy run takes the bytes up to the later one); and where it
+// finds a match, a later place up to the end of that lazy run finds one that ends at the same
+// byte, the flags between having failed from the earlier place too. So each line is read a
+// bounded number of times however many places on it are asked about.
+func (c *redactChain) lineFollowerAt(q int) []int {
+ t := c.t
+ if c.lineFailed == nil {
+ c.lineFailed = make([][2]int, len(c.adj.followers))
+ c.lineFound = make([][3]int, len(c.adj.followers))
+ for k := range c.lineFailed {
+ c.lineFailed[k] = [2]int{-1, -1}
+ c.lineFound[k] = [3]int{-1, -1, -1}
+ }
+ }
+ for k, f := range c.adj.followers {
+ if f.kind != followLine || !c.hasNeedleAt(f.needles, q) {
+ continue
+ }
+ // The pattern opens with the needle and \b: where a word byte follows the needle it finds
+ // nothing, which says nothing of later places.
+ word := len(redactionRules[f.rule].needles.needles[0])
+ if q+word < len(t) && isWordByte(t[q+word]) {
+ continue
+ }
+ failed, found := &c.lineFailed[k], &c.lineFound[k]
+ if failed[0] <= q && q <= failed[1] {
+ continue
+ }
+ if found[0] <= q && q <= found[1] {
+ return []int{q, found[2]}
+ }
+ if q < c.lineFrom || q > c.lineFirst {
+ c.lineFrom, c.lineTo = q, commandLineEnd(t, q)
+ c.lineFirst = c.lineTo
+ if i := strings.IndexByte(t[q:c.lineTo], '\n'); i >= 0 {
+ c.lineFirst = q + i
+ }
+ redactWorked(c.lineTo - q)
+ }
+ for n := 256; ; n *= 2 {
+ end := min(c.lineTo, q+n)
+ redactWorked(end - q)
+ m := f.lead.FindStringSubmatchIndex(t[q:end])
+ if m != nil && q+m[1] < end || end == c.lineTo {
+ if m != nil {
+ lead := q + m[2*f.lead.SubexpIndex("lead")+1]
+ *found = [3]int{q, lead - word, q + m[1]}
+ return []int{q, q + m[1]}
+ }
+ break
+ }
+ }
+ // The places are not asked in order: each shape's rescue asks from its own start.
+ if failed[1] == c.lineFirst && failed[0] >= 0 {
+ failed[0] = min(failed[0], q)
+ } else {
+ *failed = [2]int{q, c.lineFirst}
+ }
+ }
+ return nil
+}
+
+// followerAt returns the match of the first followRun or followScheme rule, in rule order, that
+// matches at p and keeps no value of it, and how to expand it. Each answer is kept for the place
+// asked last, and a URL's for every letter of its scheme's run: the places of a run share it.
+func (c *redactChain) followerAt(p int) ([]int, redactUse) {
+ t := c.t
+ if c.asked == nil {
+ c.asked = make([]redactAsked, len(c.adj.followers))
+ for k := range c.asked {
+ c.asked[k].at = -1
+ }
+ }
+ for k := range c.adj.followers {
+ f := &c.adj.followers[k]
+ var m []int
+ switch {
+ case f.kind == followLine:
+ continue
+ case c.asked[k].at == p:
+ m = c.asked[k].m
+ case f.kind == followScheme:
+ if p >= len(t) || !isASCIILetter(t[p]) {
+ continue
+ }
+ m = c.schemeFollowerAt(f, p)
+ c.asked[k] = redactAsked{p, m}
+ default:
+ if !c.hasNeedleAt(f.needles, p) {
+ continue
+ }
+ // The word, the blanks after it, and a run of the charset without blanks. A run
+ // follower's pattern needs a blank after its word (adjacency checks it).
+ end := p + len(redactionRules[f.rule].needles.needles[0])
+ if end >= len(t) || t[end] != ' ' && t[end] != '\t' {
+ c.asked[k] = redactAsked{p, nil}
+ continue
+ }
+ for end < len(t) && (t[end] == ' ' || t[end] == '\t') {
+ end++
+ }
+ m = c.find(f.rule, p, min(len(t), c.runEnd(f, end)+1))
+ c.asked[k] = redactAsked{p, m}
+ }
+ if m == nil {
+ continue
+ }
+ rule, re := &redactionRules[f.rule], c.pattern(f.rule)
+ if rule.keep != nil {
+ index := re.SubexpIndex("value")
+ if index < 0 {
+ index = 2
+ }
+ if g := 2 * index; m[g] >= 0 && rule.keep(t[m[g]:m[g+1]]) {
+ continue
+ }
+ }
+ return m, redactUse{re, rule}
+ }
+ return nil, redactUse{}
+}
+
+// schemeFollowerAt returns the URL follower's match at p, a letter, or nil. From any letter of a
+// run of scheme bytes the pattern reads the run to its `://` and the same text after it, so the
+// match from the first letter asked serves every other: only its start moves.
+func (c *redactChain) schemeFollowerAt(f *redactFollower, p int) []int {
+ t := c.t
+ r := c.schemeEnd(p)
+ if !strings.HasPrefix(t[r:], "://") {
+ return nil
+ }
+ if c.schemeFrom <= c.schemeAt && c.schemeAt <= r {
+ if c.schemeMatch == nil {
+ return nil
+ }
+ m := slices.Clone(c.schemeMatch)
+ m[0], m[2] = p, p
+ return m
+ }
+ end := r + 3
+ for end < len(t) && t[end] != '/' && !isRegexpSpace(t[end]) {
+ end++
+ }
+ redactWorked(end - r)
+ c.schemeAt, c.schemeMatch = p, c.find(f.rule, p, end)
+ return c.schemeMatch
+}
+
+// runEnd returns the end of the run of follower f's charset, without blanks, from s.
+func (c *redactChain) runEnd(f *redactFollower, s int) int {
+ if c.runFrom <= s && s <= c.runTo {
+ return c.runTo
+ }
+ end := s
+ for end < len(c.t) && f.charset[c.t[end]] && c.t[end] != ' ' && c.t[end] != '\t' {
+ end++
+ }
+ redactWorked(end - s)
+ c.runFrom, c.runTo = s, end
+ return end
+}
diff --git a/pkg/asymptoteobserve/redact_adjacent_test.go b/pkg/asymptoteobserve/redact_adjacent_test.go
new file mode 100644
index 00000000..46d745f6
--- /dev/null
+++ b/pkg/asymptoteobserve/redact_adjacent_test.go
@@ -0,0 +1,243 @@
+package asymptoteobserve
+
+import (
+ "math/rand/v2"
+ "regexp"
+ "strings"
+ "testing"
+)
+
+const (
+ adjacentGitHub = "ghp_0123456789abcdefghijklmnopqrstuvwxyz"
+ adjacentGoogle = "AIzaSyA-abcdefghijklmnopqrstuvwxyz01234"
+ adjacentNPM = "npm_abcdefghijklmnopqrstuvwxyz0123456789"
+ adjacentJWT = "eyJhbGciOiJIUzI1NiJ9.eyJzdWIiOiIxMjM0NTY3ODkwIn0.abcdefghijk"
+)
+
+// adjacentCases are secrets written right after another secret, or right after text a pass
+// replaces, each in the shape it is written in. A pass of the rules used to leave the second
+// secret, or part of it.
+var adjacentCases = []struct{ name, input, want string }{
+ {"two Google keys", adjacentGoogle + adjacentGoogle, "AIza[REDACTED]AIza[REDACTED]"},
+ {"two npm tokens", adjacentNPM + adjacentNPM, "npm_[REDACTED]npm_[REDACTED]"},
+ // The first token's pattern takes the second's `ghp` and stops at its `_`.
+ {"two GitHub tokens", adjacentGitHub + adjacentGitHub, "ghp_[REDACTED]ghp_[REDACTED]"},
+ {"two short GitHub tokens", "ghp_abcdefghijklmnopqrstuvwxyz0123ghp_abcdefghijklmnopqrstuvwxyz0123", "ghp_[REDACTED]ghp_[REDACTED]"},
+ {"two Hugging Face tokens", "hf_abcdefghijklmnopqrstuvwxyz01234567hf_abcdefghijklmnopqrstuvwxyz01234567", "hf_[REDACTED]hf_[REDACTED]"},
+ {"two Stripe keys", "sk_live_abcdefghijklmnopsk_live_abcdefghijklmnop", "sk_live_[REDACTED]sk_live_[REDACTED]"},
+ {"a Slack token into a GitHub token", "xoxb-1234567890-abc" + adjacentGitHub, "xoxb-[REDACTED]ghp_[REDACTED]"},
+ {"a GitHub token into a Google key", adjacentGitHub + adjacentGoogle, "ghp_[REDACTED]AIza[REDACTED]"},
+ {"a GitHub token into a JWT", adjacentGitHub + adjacentJWT, "ghp_[REDACTED]eyJ[REDACTED]"},
+ {"a JWT into a JWT", adjacentJWT + adjacentJWT, "eyJ[REDACTED]eyJ[REDACTED]"},
+ {"an API key into an OAuth token", "sk-abcdefghijklmnopqrst-ya29.abcdefghijklmnopqrstuvwxyz", "sk-[REDACTED]ya29.[REDACTED]"},
+ // The key's pattern takes the JWT's first segment and stops at its dot, too short a key
+ // without it: the two are redacted as one.
+ {"an API key into a JWT it cannot be cut before", "sk-" + adjacentJWT, "sk-[REDACTED]"},
+ {"curl cookies after an npm token", adjacentNPM + "curl -b 'a=1' -b 'b=2'", "npm_[REDACTED]curl -b '[REDACTED]' -b '[REDACTED]'"},
+ // A token's pattern runs on over a URL's scheme, `Bearer` or `curl`: the token is cut before
+ // it, and the follower is redacted.
+ {"a URL password after a GitHub token", adjacentGitHub + "postgres://user:pw@host/db", "ghp_[REDACTED]s://user:[REDACTED]@host/db"},
+ {"a URL password after a JWT", adjacentJWT + "postgres://user:pw@host/db", "eyJ[REDACTED]s://user:[REDACTED]@host/db"},
+ {"a Bearer token after a Hugging Face token", "hf_abcdefghijklmnopqrstuvwxyz01234567Bearer abcdefghijk", "hf_[REDACTED]Bearer [REDACTED]"},
+ {"a Bearer token after a Slack token", "xoxb-1234567890-abcBearer abcdefghijk", "xoxb-[REDACTED]Bearer [REDACTED]"},
+ {"a curl password after an API key", "sk-abcdefghijklmnopqrstuvwxyz0123curl -u admin:hunter2 https://x", "sk-[REDACTED]curl -u admin:[REDACTED] https://x"},
+ {"a curl password after a GitLab token", "glpat-abcdefghijklmnopqrstuvcurl -u admin:hunter2", "glpat-[REDACTED]curl -u admin:[REDACTED]"},
+ {"a URL password after an AWS key id", "AKIAIOSFODNN7EXAMPLEpostgres://user:pw@host/db", "AKIA[REDACTED]postgres://user:[REDACTED]@host/db"},
+ {"a Bearer token after an AWS key id", "AKIAIOSFODNN7EXAMPLEBearer abcdefghijk", "AKIA[REDACTED]Bearer [REDACTED]"},
+ {"a curl password after an AWS key id", "AKIAIOSFODNN7EXAMPLEcurl -u admin:hunter2", "AKIA[REDACTED]curl -u admin:[REDACTED]"},
+ // A URL's scheme run ends at a byte its pattern stops at (`é`), and goes on over one that
+ // folds into it (`K`, Kelvin): what is asked before the first is not the answer after it.
+ {"a URL password after an AWS key id, past a non-ASCII letter", "AKIAIOSFODNN7EXAMPLEa\u00e9AKIAIOSFODNN7EXAMPLEpostgres://u:hunter2@ h", "AKIAIOSFODNN7EXAMPLEa\u00e9AKIA[REDACTED]postgres://u:[REDACTED]@ h"},
+ {"a URL password after AWS key ids, over a Kelvin sign", "AKIAIOSFODNN7EXAMPLEa\u212aAKIAIOSFODNN7EXAMPLEpostgres://u:hunter2@ h", "AKIA[REDACTED]a\u212aAKIA[REDACTED]postgres://u:[REDACTED]@ h"},
+ // When the key id rule asks whether a curl command follows the key id, the rule for a key id
+ // and its secret has already asked about the line from its later `curl`, which has none.
+ {"a curl password after a Google key and an AWS key id", adjacentGoogle + "AKIAIOSFODNN7EXAMPLEcurl -u a:curl -u a:", "AIza[REDACTED]AKIA[REDACTED]curl -u a:[REDACTED] -u a:"},
+ // A `curl` a word byte follows (`curl_x`) is no command, and says nothing of a later one.
+ {"a curl password after a token, past a glued curl_", adjacentGitHub + "curl_x " + adjacentGitHub + "curl -u admin:hunter2", "ghp_[REDACTED]_x ghp_[REDACTED]curl -u admin:[REDACTED]"},
+ // The flag starts the line after the command, and its value is on the line after that.
+ {"a curl flag alone on the line after a key", adjacentNPM + "curl x\n-u\na:hunter2", "npm_[REDACTED]curl x\n-u\na:[REDACTED]"},
+ {"a Google key, then a GitHub token", adjacentGoogle + adjacentGitHub, "AIza[REDACTED]ghp_[REDACTED]"},
+ {"npm tokens and Google keys in turn", adjacentNPM + adjacentGoogle + adjacentNPM + adjacentGoogle, "npm_[REDACTED]AIza[REDACTED]npm_[REDACTED]AIza[REDACTED]"},
+ // The key id's pattern ends in \b, which a token right after it takes away.
+ {"an AWS key id, then a GitHub token", "AKIAIOSFODNN7EXAMPLE" + adjacentGitHub, "AKIA[REDACTED]ghp_[REDACTED]"},
+ {"an AWS key id, then a Google key", "AKIAIOSFODNN7EXAMPLE" + adjacentGoogle, "AKIA[REDACTED]AIza[REDACTED]"},
+ {"an AWS secret, then a GitHub token", "AKIAIOSFODNN7EXAMPLE,wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY" + adjacentGitHub, "AKIA[REDACTED],[REDACTED]ghp_[REDACTED]"},
+ {"a GitHub token, then an AWS key id and secret", adjacentGitHub + "AKIAIOSFODNN7EXAMPLE,wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY\n", "ghp_[REDACTED]AKIA[REDACTED],[REDACTED]\n"},
+ // The rules for a URL's password, a Bearer token and a curl password run before the shape
+ // rules, while the key before them is still there.
+ {"a URL password after an npm token", adjacentNPM + "postgres://user:pw@host/db", "npm_[REDACTED]postgres://user:[REDACTED]@host/db"},
+ {"a Bearer token after an npm token", adjacentNPM + "Bearer abcdefghijk", "npm_[REDACTED]Bearer [REDACTED]"},
+ {"a curl password after an npm token", adjacentNPM + "curl -u admin:hunter2 https://x", "npm_[REDACTED]curl -u admin:[REDACTED] https://x"},
+ {"curl passwords after npm tokens", adjacentNPM + "curl -u a:b1 " + adjacentNPM + "curl -u c:d2", "npm_[REDACTED]curl -u a:[REDACTED] npm_[REDACTED]curl -u c:[REDACTED]"},
+ {"a URL password after a Google key, then a token", adjacentGoogle + "https://x:pw@h/" + adjacentNPM, "AIza[REDACTED]https://x:[REDACTED]@h/npm_[REDACTED]"},
+ // A bare copy of an assigned value is replaced after every rule has run.
+ {"a GitHub token after a copy of an assigned value", "token=abcdefgh12345678 abcdefgh12345678" + adjacentGitHub, "token=[REDACTED] [REDACTED]ghp_[REDACTED]"},
+ {"a URL password after a copy of an assigned value", "token=abcdefgh12345678 abcdefgh12345678redis://:pw@h", "token=[REDACTED] [REDACTED]redis://:[REDACTED]@h"},
+ // The quoted-label rule keeps `null`, the capture of the value runs on past it, and the rule
+ // for `x-access-token:` then replaces the token inside what was captured.
+ {"a value changed after it was captured", `"secret": null}bcn_code_--https://x-access-token:[email protected]/o/r`, `"secret": [REDACTED]`},
+ // A token's pattern takes the first bytes of a copy of an assigned value written right after
+ // it, and the copy, removed only whole, was left from there on: the item runs on to its end.
+ {"a copy of an assigned value after a token", "hf_abcdefghijklmnopqrstuvwxyz01234567opaque-flag-value tool login --token=opaque-flag-value", "hf_[REDACTED] tool login --token=[REDACTED]"},
+}
+
+func TestRedactStringRedactsSecretsWrittenRightAfterAnother(t *testing.T) {
+ for _, tc := range adjacentCases {
+ if got := RedactString(tc.input); got != tc.want {
+ t.Errorf("%s: RedactString(%q) = %q, want %q", tc.name, tc.input, got, tc.want)
+ }
+ if got := redactString(tc.input, false); got != tc.want {
+ t.Errorf("%s: with every rule's whole pattern, %q, want %q", tc.name, got, tc.want)
+ }
+ withBudget(t, 0, -1, func() {
+ if got := RedactString(tc.input); got != tc.want {
+ t.Errorf("%s: from safe starts, %q, want %q", tc.name, got, tc.want)
+ }
+ })
+ }
+}
+
+// Text where a secret only ends next to other text, or is followed by something shaped like one
+// that is not, redacts as before.
+func TestRedactStringKeepsWhatFollowsASecretWhenItIsNone(t *testing.T) {
+ for input, want := range map[string]string{
+ adjacentGoogle + "xyz": "AIza[REDACTED]xyz",
+ adjacentNPM + "postgres": "npm_[REDACTED]postgres",
+ adjacentNPM + "ghp_short": "npm_[REDACTED]ghp_short",
+ "AKIAIOSFODNN7EXAMPLExoxz": "AKIAIOSFODNN7EXAMPLExoxz",
+ "AKIAIOSFODNN7EXAMPLEghp_short": "AKIAIOSFODNN7EXAMPLEghp_short",
+ adjacentGitHub + "_suffix": "ghp_[REDACTED]_suffix",
+ "x" + adjacentGitHub: "x" + adjacentGitHub,
+ "token=abcdefgh12345678 abcdefgh12345678xyz": "token=[REDACTED] [REDACTED]xyz",
+ "[REDACTED]Bearer": "[REDACTED]Bearer",
+ // A URL whose user name only ends in a word a label holds keeps its host and path; one
+ // that is a label (api_key) is redacted as far as the label rules take it.
+ "https://deploykey:pw@host/repo": "https://deploykey:[REDACTED]@host/repo",
+ "https://key:pw@host/repo": "https://key:[REDACTED]@host/repo",
+ "https://keys:pw@host/repo": "https://keys:[REDACTED]@host/repo",
+ } {
+ if got := RedactString(input); got != want {
+ t.Errorf("RedactString(%q) = %q, want %q", input, got, want)
+ }
+ }
+}
+
+// A URL's password that runs on as the label rules read it (urlRunsOn) reads an AWS key id and
+// its secret as the shape rule for the pair leaves them: the run that takes the key id takes the
+// secret too, and the pair does not stop the password where the secret's `/` would.
+func TestRedactStringRunsAURLPasswordOnOverAnAWSKeyAndSecret(t *testing.T) {
+ const pair = "AKIAIOSFODNN7EXAMPLE,wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY"
+ for input, want := range map[string]string{
+ "A://token:,@0,@0,@" + pair + "*S-1": "A://token:[REDACTED]",
+ "A://token:,@AKIAIOSFODNN7EXAMPLE:,wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY x": "A://token:[REDACTED] x",
+ "A://token:,@0," + pair + "@h": "A://token:[REDACTED]",
+ "A://token:,@0,AKIAIOSFODNN7EXAMPLE wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY@h": "A://token:[REDACTED],AKIA[REDACTED] [REDACTED]@h",
+ } {
+ got := RedactString(input)
+ if got != want {
+ t.Errorf("RedactString(%q) = %q, want %q", input, got, want)
+ }
+ if again := RedactString(got); again != got {
+ t.Errorf("RedactString(%q) = %q, and again %q", input, got, again)
+ }
+ }
+}
+
+// The run schemeFollowerAt shares one answer across is exactly what the URL pattern's
+// case-insensitive scheme class takes, non-ASCII characters that fold into it among it.
+func TestSchemeEndIsTheURLPatternsRun(t *testing.T) {
+ run := regexp.MustCompile(`^(?i)[a-z0-9+.-]*`)
+ pieces := []string{"a", "Z", "0", "+", ".", "-", "k", "s", "\u212a", "\u017f", "\u00e9", "\u212b", ":", "/", " ", "\xff", "\xe2\x84"}
+ r := rand.New(rand.NewPCG(7, 7))
+ for range 20000 {
+ var b strings.Builder
+ for n := r.IntN(12); n >= 0; n-- {
+ b.WriteString(pieces[r.IntN(len(pieces))])
+ }
+ text := b.String()
+ c := newRedactChain(redactEngineFor(), text, nil)
+ if got, want := c.schemeEnd(0), len(run.FindString(text)); got != want {
+ t.Fatalf("schemeEnd(%q) = %d, the pattern's run ends at %d", text, got, want)
+ }
+ }
+}
+
+// One pass leaves nothing for a second one: events are sanitized again on every read path, and a
+// derived field that redacts text a writer already redacted must find nothing new.
+func FuzzRedactIsIdempotent(f *testing.F) {
+ for _, tc := range adjacentCases {
+ f.Add(tc.input)
+ }
+ for _, input := range redactIdempotenceInputs {
+ f.Add(input)
+ }
+ r := rand.New(rand.NewPCG(5, 5))
+ for range 64 {
+ f.Add(randomAdjacentInput(r))
+ }
+ f.Fuzz(func(t *testing.T, input string) {
+ once := RedactString(input)
+ if twice := RedactString(once); twice != once {
+ t.Fatalf("RedactString is not idempotent on %q:\nonce %q\ntwice %q", input, once, twice)
+ }
+ if all := redactString(input, false); all != once {
+ t.Fatalf("redaction of %q = %q, every rule's whole pattern gives %q", input, once, all)
+ }
+ })
+}
+
+// randomAdjacentInput is randomRedactInput's fragments with whole secrets among them, so they
+// often meet.
+func randomAdjacentInput(r *rand.Rand) string {
+ var b strings.Builder
+ for n := r.IntN(8); n >= 0; n-- {
+ if r.IntN(3) == 0 {
+ b.WriteString(adjacentSecrets[r.IntN(len(adjacentSecrets))])
+ } else {
+ b.WriteString(redactGateFragments[r.IntN(len(redactGateFragments))])
+ }
+ }
+ return b.String()
+}
+
+var adjacentSecrets = []string{
+ adjacentGitHub, adjacentGoogle, adjacentNPM, adjacentJWT,
+ "ghs_abcdefghijklmnopqrstuvwxyz0123", "hf_abcdefghijklmnopqrstuvwxyz01234567",
+ "sk-abcdefghijklmnopqrstuvwxyz0123", "sk-ant-api03-abcdefghijklmnopqrstuvwxyz", "AKIAIOSFODNN7EXAMPLE",
+ "AKIAIOSFODNN7EXAMPLE,wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY", "github_pat_abcdefghijklmnopqrstuvwxyz",
+ "glpat-abcdefghijklmnopqrstuv", "xoxb-1234567890-abc", "sk_live_abcdefghijklmnop", "ya29.abcdefghijklmnopqrstuvwxyz",
+ "bcn_api_abcdefgh", "postgres://user:pw@host/db", "https://x-access-token:[email protected]", "token=abcdefgh12345678",
+ "Bearer abcdefghijk", "curl -u admin:hunter2 ", "abcdefgh12345678",
+}
+
+// Idempotence and the gate's equivalence over many random texts where secrets meet, beyond what
+// the fuzz corpus holds, every fourth also from safe starts.
+func TestRedactIsIdempotentWhereSecretsMeet(t *testing.T) {
+ backToBack := regexp.MustCompile(`\[REDACTED\][A-Za-z0-9_.-]{1,16}\[REDACTED\]`)
+ r := rand.New(rand.NewPCG(6, 6))
+ met := 0
+ for i := range 10000 {
+ input := randomAdjacentInput(r)
+ once := RedactString(input)
+ if twice := RedactString(once); twice != once {
+ t.Fatalf("RedactString is not idempotent on %q:\nonce %q\ntwice %q", input, once, twice)
+ }
+ if all := redactString(input, false); all != once {
+ t.Fatalf("redaction of %q = %q, every rule's whole pattern gives %q", input, once, all)
+ }
+ if i%4 == 0 {
+ withBudget(t, 0, -1, func() {
+ if got := RedactString(input); got != once {
+ t.Fatalf("redaction of %q from safe starts = %q, want %q", input, got, once)
+ }
+ })
+ }
+ if backToBack.MatchString(once) {
+ met++
+ }
+ }
+ // The texts have to put secrets next to each other for the check to mean anything.
+ if met < 600 {
+ t.Fatalf("only %d texts redacted a secret right after another", met)
+ }
+}
diff --git a/pkg/asymptoteobserve/redact_engine.go b/pkg/asymptoteobserve/redact_engine.go
index fd4e5cfe..6c835bc9 100644
--- a/pkg/asymptoteobserve/redact_engine.go
+++ b/pkg/asymptoteobserve/redact_engine.go
@@ -35,6 +35,9 @@ import (
// case-insensitive rules run their whole patterns, as they always did. redactString(value, false)
// runs every rule's whole pattern on any text, and TestRedactGateNeverChangesAResult and
// FuzzRedactGate hold the two to the same result.
+//
+// A shape rule's matches also take in the secrets written right after them (redact_adjacent.go),
+// the same way whichever way the matches are found.
// redactStart says where, around an occurrence of one of a rule's needles that passes the needle
// set's precededBy and followedBy (and those of the sets it is within), a match of the rule's
@@ -70,15 +73,20 @@ type redactSlot struct {
// The assertions the whole pattern makes, which decide where it can be run on a suffix of the
// text with the same result (safeStart): \b or \B, (?m)^, and ^ or \A.
wordAssert, lineStart, textStart bool
+ // index is the rule's index in redactionRules, or -1.
+ index int
}
+// boundaryPrefixes are the ways a pattern opens with \b.
+var boundaryPrefixes = []string{`(?i)(\b`, `(?i)\b`, `(\b`, `\b`}
+
// prepare compiles the rule's pattern anchored at the start of the text, without the \b it opens
// with, if any: the anchored pattern cannot see the byte before its start, so the boundary is
// checked against the whole text instead.
func (s *redactSlot) prepare(rule redactionRule) {
s.once.Do(func() {
expr := rule.expr
- for _, prefix := range []string{`(?i)(\b`, `(?i)\b`, `(\b`, `\b`} {
+ for _, prefix := range boundaryPrefixes {
if strings.HasPrefix(expr, prefix) {
s.boundary = true
expr = prefix[:len(prefix)-2] + expr[len(prefix):]
@@ -157,10 +165,17 @@ type redactEngine struct {
setMasks [64]uint64 // redactNeedles id -> the needles of the set, as scanner needle bits
rules []redactSlot
assigned redactSlot
+ // adjacent is what redact_adjacent.go knows of the rules, built the first time it is needed.
+ adjacentOnce sync.Once
+ adjacent *redactAdjacency
}
var redactEngineFor = sync.OnceValue(func() *redactEngine {
e := &redactEngine{rules: make([]redactSlot, len(redactionRules))}
+ for i := range e.rules {
+ e.rules[i].index = i
+ }
+ e.assigned.index = -1
ids := map[redactNeedle]int{}
var needles []redactNeedle
var register func(set *redactNeedles)
@@ -219,6 +234,13 @@ type redactGate struct {
// known and present are bit sets over redactNeedles ids: which sets were looked for in this
// version of the text, and which were found.
known, present uint64
+ // hidden is set when a match may hide another from this pass: a value a rule keeps, or the
+ // capture takes, that is itself a label and its separator (keptLabel), inside a match that
+ // neither looks into; a copy a token ran on over, as replacing a copy may (copyHidesMatch); or a
+ // replaced AWS secret's `/`, where a URL's password stopped.
+ hidden bool
+ // copies are the assigned values whose copies are removed after every rule has run.
+ copies *redactCopies
}
func (g *redactGate) set(value string) {
@@ -325,23 +347,69 @@ func (set *redactNeedles) passes(engine *redactEngine, t string, h redactHit, wi
// When the places would hand the pattern more text than the budget allows (redactAnchoredPerByte),
// the whole pattern runs once from a safe start before the first of them instead. No match starts
// before the first place, and from a safe start the pattern finds what it finds in the whole text.
-func (rule redactionRule) matches(text *redactGate, slot *redactSlot) iter.Seq2[[]int, *regexp.Regexp] {
- return func(yield func([]int, *regexp.Regexp) bool) {
+//
+// A shape rule's match is followed by the secrets written right after it (redactChain.run), each
+// yielded with the rule it is a match of, and the next match is looked for from where they end,
+// in both ways of finding it. Text with none gets exactly FindAll's matches.
+func (rule redactionRule) matches(text *redactGate, slot *redactSlot) iter.Seq2[[]int, redactUse] {
+ return func(yield func([]int, redactUse) bool) {
t := text.value
+ var chain *redactChain
+ if rule.shape != shapeNone {
+ text.scan()
+ slot.prepare(rule)
+ chain = newRedactChain(text.e(), t, text.hits)
+ chain.copies = text.copies
+ defer func() {
+ text.hidden = text.hidden || chain.hid || chain.slashed && strings.Contains(t, "://")
+ }()
+ }
whole := func(from int) {
- if redactWork != nil {
- redactWork(len(t) - from)
- }
re := rule.pattern()
- for _, m := range re.FindAllStringSubmatchIndex(t[from:], -1) {
- for i := range m {
- if m[i] >= 0 {
- m[i] += from
+ use := redactUse{re, &rule}
+ if chain == nil {
+ redactWorked(len(t) - from)
+ for _, m := range re.FindAllStringSubmatchIndex(t[from:], -1) {
+ for i := range m {
+ if m[i] >= 0 {
+ m[i] += from
+ }
+ }
+ if !yield(m, use) {
+ return
}
}
- if !yield(m, re) {
+ return
+ }
+ // FindAll's loop, from where each chain ends. The pattern run on t[pos:] takes pos for
+ // the start of a word; where t does not, the search goes on from the next byte. The
+ // first match from aheadFrom on (nil: none) stays the first from any later pos up to
+ // its start, so a rescued match before it does not make the search read on to it again.
+ si := chain.adj.byRule[slot.index]
+ var ahead []int
+ aheadFrom := -1
+ for pos := from; pos < len(t); {
+ if aheadFrom < 0 || ahead != nil && ahead[0] < pos {
+ ahead, aheadFrom = chain.search(re, pos), pos
+ if ahead != nil && ahead[0] == pos && pos > from && slot.boundary && !wordBoundary(t, pos) {
+ ahead = chain.search(re, pos+1)
+ }
+ }
+ m, limit := ahead, len(t)
+ if m != nil {
+ limit = m[0]
+ }
+ if rescued := chain.rescueBefore(si, pos, limit); rescued != nil {
+ m = rescued
+ }
+ if m == nil {
return
}
+ end, ok := chain.run(si, m, use, yield)
+ if !ok {
+ return
+ }
+ pos = end
}
}
if !text.enabled || rule.needles.folded && text.unfoldable() {
@@ -400,23 +468,42 @@ func (rule redactionRule) matches(text *redactGate, slot *redactSlot) iter.Seq2[
redactWork(len(t) - q)
}
m := slot.anchored.FindStringSubmatchIndex(t[q:])
+ for i := range m {
+ if m[i] >= 0 {
+ m[i] += q
+ }
+ }
+ if m == nil && chain != nil {
+ m = chain.rescue(chain.adj.byRule[slot.index], q)
+ }
if m == nil {
failed = q
continue
}
- for i := range m {
- if m[i] >= 0 {
- m[i] += q
+ use := redactUse{slot.anchored, &rule}
+ if chain != nil {
+ end, ok := chain.run(chain.adj.byRule[slot.index], m, use, yield)
+ if !ok {
+ return
}
+ pos = end
+ continue
}
pos = m[1]
- if !yield(m, slot.anchored) {
+ if !yield(m, use) {
return
}
}
}
}
+// redactUse is what matches yields with a match: the pattern to expand it with and the rule it is
+// a match of, which for a secret written right after a shape rule's match is another shape rule.
+type redactUse struct {
+ pattern *regexp.Regexp
+ rule *redactionRule
+}
+
// startFor returns the leftmost place at or after pos where a match holding h can begin, or -1.
func (rule redactionRule) startFor(t string, h redactHit, pos int, boundary bool) int {
switch rule.start {
diff --git a/pkg/asymptoteobserve/redact_engine_test.go b/pkg/asymptoteobserve/redact_engine_test.go
index 7a0437ed..afeef1db 100644
--- a/pkg/asymptoteobserve/redact_engine_test.go
+++ b/pkg/asymptoteobserve/redact_engine_test.go
@@ -1,6 +1,7 @@
package asymptoteobserve
import (
+ "fmt"
mrand "math/rand"
"math/rand/v2"
"slices"
@@ -188,17 +189,87 @@ func TestRedactionWorkIsLinear(t *testing.T) {
"authorization headers": "authorization: bearer ,",
"cookie headers": "\ncookie: ",
"assignments": "token=abcdefgh12345678 ",
+ // Secrets written right after another one (redact_adjacent.go): chains of keys of fixed
+ // length, of tokens that run on into the next, of key ids that end only where the next
+ // key starts, of JWT first segments that never complete, and URLs, Bearer tokens, curl
+ // commands and replaced copies right after a key.
+ "Google keys back to back": "AIzaSyA-abcdefghijklmnopqrstuvwxyz01234",
+ "GitHub tokens back to back": "ghp_0123456789abcdefghijklmnopqrstuvwxyz",
+ "AWS key ids back to back": "AKIAIOSFODNN7EXAMPLE",
+ "npm tokens and Google keys": "npm_abcdefghijklmnopqrstuvwxyz0123456789AIzaSyA-abcdefghijklmnopqrstuvwxyz01234",
+ "tokens into JWT prefixes": "ghp_0123456789abcdefghijklmnopqrstuvwxyzeyJabc-",
+ "API keys into JWT prefixes": "sk-eyJabcdefghij",
+ "URL schemes after keys": "npm_abcdefghijklmnopqrstuvwxyz0123456789a://x:",
+ "Bearer tokens after keys": "npm_abcdefghijklmnopqrstuvwxyz0123456789Bearer ",
+ "curl commands after keys": "npm_abcdefghijklmnopqrstuvwxyz0123456789curl -b x ",
+ "keys after replaced copies": "token=abcdefgh12345678 abcdefgh12345678ghp_0123456789abcdefghijklmnopqrstuvwxyz ",
+ "key ids into unfinished keys": "AKIAIOSFODNN7EXAMPLEAIza",
+ "tokens into URL schemes": "ghp_0123456789abcdefghijklmnopqrstuvwxyzpostgres://u:p@h/",
+ "tokens into Bearer tokens": "hf_abcdefghijklmnopqrstuvwxyz01234567Bearer x ",
+ "keys into curl commands": "sk-abcdefghijklmnopqrstuvwxyz0123curl -u a:b ",
+ "keys into curl lines": "sk-abcdefghijklmnopqrstuvwxyz0123curl x ",
+ "key ids into URL schemes": "AKIAIOSFODNN7EXAMPLEa://x:y@",
+ "key ids into unfinished URLs": "AKIAIOSFODNN7EXAMPLEa:/",
+ // Values that are themselves labels, copies that start or end a match, and AWS secrets in
+ // URL passwords, which make redaction pass over the text again; tokens into copies.
+ "labels kept values hid": `token:seCret":token":0 `,
+ "labels hid three passes": `0secret=token :ghp_0123456789abcdefghijklmnopqrstsecret=token :seCret':token':@h' `,
+ "labels captured values hid": "token:abctoken\":`secret`:`token`:}token:--token \"token\" ",
+ "copies after flags": "-pAsswd --token -token:\"--token\" ",
+ "key ids before copies": "AKIAIOSFODNN7EXAMPLEAuthorization: Authorization: ",
+ "tokens into copies": "token=opaque-flag-value hf_abcdefghijklmnopqrstuvwxyz01234567opaque-flag-value ",
+ "AWS secrets in passwords": "https://u:a,AKIAIOSFODNN7EXAMPLE,wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY@h ",
+ // A URL password's run on (urlRunsOn) over AWS key ids and their secrets.
+ "URL runs over AWS keys": "A://token:,@AKIAIOSFODNN7EXAMPLE,wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY@",
+ "URL passwords over AWS keys": "A://token:,@0,AKIAIOSFODNN7EXAMPLE,wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY ",
+ // Tokens glued to `Bearer` with no blank after it, which run on to the end of the text.
+ "tokens glued to Bearer": "hf_abcdefghijklmnopqrstuvwxyz01234567Bearer.",
+ // Every other scan the chains make: open tails run on to the end, curl lines over blank
+ // lines, curl lines chained after keys, and URL schemes after tokens.
+ "API keys back to back": "sk-abcdefghijklmnopqrstuvwxyz0123",
+ "keys glued to curl over blank lines": "sk-abcdefghijklmnopqrstuvwxyz0123curl x\n \n\n",
+ "curl lines chained after keys": "npm_abcdefghijklmnopqrstuvwxyz0123456789curl x\n",
+ "tokens glued to URL scheme runs": "ghp_0123456789abcdefghijklmnopqrstuvwxyza.b",
}
+ texts := map[string]func(n int) string{}
for name, unit := range families {
+ texts[name] = func(n int) string { return repeatTo(unit, n) }
+ }
+ texts["one long flag"] = func(n int) string { return "-" + repeatTo("token", n) + " x" }
+ // Texts that are not one unit repeated: many places on one long line or run that all reach the
+ // same far end (a curl flag, a URL's `://`), and copies of assigned values that a token runs
+ // into.
+ texts["key ids glued to curl, one flag at the end"] = func(n int) string {
+ return repeatTo("AKIAIOSFODNN7EXAMPLEcurl ", n) + " -u a:b"
+ }
+ texts["keys glued to curl, one flag at the end"] = func(n int) string {
+ return repeatTo("sk-abcdefghijklmnopqrstuvwxyz0123curl ", n) + " -u a:b"
+ }
+ texts["key ids in a URL scheme run"] = func(n int) string { return repeatTo("AKIA.", n) + "b://" }
+ texts["key ids glued into a URL scheme"] = func(n int) string { return repeatTo("AKIAIOSFODNN7EXAMPLE", n) + "b://x:y@h" }
+ texts["key ids in URL scheme runs broken by non-ASCII"] = func(n int) string {
+ return repeatTo("AKIAIOSFODNN7EXAMPLEa\u00e9", n) + "b://x:y@h"
+ }
+ texts["key ids in a URL scheme run over Kelvin signs"] = func(n int) string {
+ return repeatTo("AKIAIOSFODNN7EXAMPLEa\u212a", n) + "b://x:y@h"
+ }
+ texts["a long value repeating a token's tail"] = func(n int) string {
+ a := strings.Repeat("a", n/3)
+ return "token=" + a + " hf_" + a + " " + strings.Repeat("x ", n/6)
+ }
+ texts["many values sharing their first bytes"] = func(n int) string {
+ var b strings.Builder
+ for i := 0; b.Len() < n/2; i++ {
+ fmt.Fprintf(&b, "token=aaaaaaaa%06d\n", i)
+ }
+ return b.String() + repeatTo("ghp_"+strings.Repeat("a", 40)+" ", n/2)
+ }
+ for name, text := range texts {
work := func(n int) int {
- text := strings.Repeat(unit, n/len(unit)+1)[:n]
- if name == "one long flag" {
- text = "-" + text + " x"
- }
total := 0
redactWork = func(bytes int) { total += bytes }
defer func() { redactWork = nil }()
- redactString(text, true)
+ redactString(text(n), true)
return total
}
previous := work(16 << 10)
@@ -212,3 +283,8 @@ func TestRedactionWorkIsLinear(t *testing.T) {
}
}
}
+
+// repeatTo is unit repeated to n bytes.
+func repeatTo(unit string, n int) string {
+ return strings.Repeat(unit, n/len(unit)+1)[:n]
+}