The cross-harness, self-improving memory layer for AI agents. Join our community: https://discord.com/invite/zdNChS2fBu
Go agent-ai-cliagent-harnessesagent-securitybrowser-extensionciclaude-codecloudcodexcursordetection-engineeringjevknowledge-basememorymobile-device-managementobservabilitysecuritysecurity-information-and-event-managementsecurity-toolstelemetrytraces
fix(privacy): redact secrets that directly follow another secret (#1153)
kyle-elliott-asymptoteb6439f20a9c1
diff --git a/pkg/asymptoteobserve/privacy.go b/pkg/asymptoteobserve/privacy.goindex dedc01d1..3ac8536e 100644--- a/pkg/asymptoteobserve/privacy.go+++ b/pkg/asymptoteobserve/privacy.go@@ -132,6 +132,12 @@ type redactionRule struct {// how many bytes before the needle a startNeedle match does. See redact_engine.go.start redactStartoffset int+ // shape marks a credential recognized by its own shape: a public type prefix, ${1}, and the+ // secret after it. Such a rule also redacts a secret written right after its match+ // (redact_adjacent.go).+ shape redactShapeKind+ // runsOn, when set, returns where the text a match replaces ends, at or past the match's end.+ runsOn func(t string, match []int) int}// pattern is the rule's compiled pattern.@@ -187,6 +193,13 @@ func curlRule(expr, replacement string) redactionRule {return r}+// shapeRule is a rule for a credential recognized by its own shape (redactionRule.shape).+func shapeRule(expr, replacement string, needles ...string) redactionRule {+ r := rule(expr, replacement, nil, needles...)+ r.shape = shapeToken+ return r+}+// lazyPattern compiles expr the first time it is called.//// The rules whose pattern names a credential (redactLabel, redactFlag) are written as struct@@ -421,30 +434,32 @@ var redactionRules = concatRules([]redactionRule{foldedRule(`(?im)((?:^\s*|["']\s*|(?:-H|--header)(?:=|\s+)["']?)(?:set-)?cookie["']?\s*:\s*["']?)([^\n"'\\]+)`, "${1}"+redactedValue, isRedactedValue, "cookie").from(startLine),// The password in a URL's userinfo: `postgres://user:pw@host`, `https://x-access-token:[email protected]`,// and with no user name, the way Redis URLs carry a password: `redis://:pw@host`.- foldedRule(`(?i)(\b[a-z][a-z0-9+.-]*://[^/\s:@"'<>]*:)([^@/\s"'<>]+)(@)`, "${1}"+redactedValue+"${3}", nil, "://").from(startScheme),+ foldedRule(`(?i)(\b[a-z][a-z0-9+.-]*://[^/\s:@"'<>]*:)([^@/\s"'<>]+)(@)`, "${1}"+redactedValue+"${3}", nil, "://").from(startScheme).runningOn(urlRunsOn),// Credentials recognizable by their own shape, wherever they appear. The type prefix is public// and stays (`ghp_[REDACTED]`, `AKIA[REDACTED]`), the way secret scanners report a finding: the// line still says a GitHub token or an AWS key was there, which is what detection rules match on- // in the log, and none of the credential is left.- rule(`\b(sk-)[A-Za-z0-9_-]{20,}`, "${1}"+redactedValue, nil, "sk-"), // Anthropic sk-ant-, OpenAI sk-proj- and sk-+ // in the log, and none of the credential is left. A secret written right after one of these is+ // redacted with it, keeping its own prefix: `ghp_[REDACTED]ghp_[REDACTED]` (redact_adjacent.go).+ shapeRule(`\b(sk-)[A-Za-z0-9_-]{20,}`, "${1}"+redactedValue, "sk-"), // Anthropic sk-ant-, OpenAI sk-proj- and sk-// An AWS secret access key has no shape of its own, but where it follows its key id within a few// separators (a credentials CSV, `aws configure` output, a table, a JSON array) it is the 40- // characters after them. This runs before the key id rule below replaces the id it keys on.- rule(`(\b(?:AKIA|ASIA)[A-Z0-9]{16}\b[\s,:|"']{1,8})[A-Za-z0-9/+]{40}([^A-Za-z0-9/+=]|$)`, "${1}"+redactedValue+"${2}", nil, "AKIA", "ASIA"),- rule(`\b(AKIA|ASIA|ABIA|ACCA)[A-Z0-9]{16}\b`, "${1}"+redactedValue, nil, "AKIA", "ASIA", "ABIA", "ACCA"), // AWS access key id- rule(`\b(gh[pousr]_)[A-Za-z0-9]{30,}`, "${1}"+redactedValue, nil, "ghp_", "gho_", "ghu_", "ghs_", "ghr_"), // GitHub tokens- rule(`\b(github_pat_)[A-Za-z0-9_]{22,}`, "${1}"+redactedValue, nil, "github_pat_"), // GitHub fine-grained tokens- rule(`\b(glpat-)[A-Za-z0-9_-]{20,}`, "${1}"+redactedValue, nil, "glpat-"), // GitLab- rule(`\b(xox[abposr]-)[A-Za-z0-9-]{10,}`, "${1}"+redactedValue, nil, "xox"), // Slack- rule(`\b([rs]k_(?:live|test)_)[A-Za-z0-9]{16,}`, "${1}"+redactedValue, nil, "k_live_", "k_test_").before(1), // Stripe- rule(`\b(AIza)[0-9A-Za-z_-]{35}`, "${1}"+redactedValue, nil, "AIza"), // Google API key- rule(`\b(ya29\.)(?:c\.)?[0-9A-Za-z_-]{20,}`, "${1}"+redactedValue, nil, "ya29."), // Google OAuth access token- rule(`\b(npm_)[A-Za-z0-9]{36}`, "${1}"+redactedValue, nil, "npm_"), // npm- rule(`\b(hf_)[A-Za-z0-9]{30,}`, "${1}"+redactedValue, nil, "hf_"), // Hugging Face+ // characters after them. The key id is replaced with it, as the key id rule below would: where+ // this rule redacts a secret right after another one, the key id rule has already run.+ shapeRule(redactAWSPair, "${1}"+redactedValue+"${2}"+redactedValue+"${3}", "AKIA", "ASIA"),+ shapeRule(`\b(AKIA|ASIA|ABIA|ACCA)[A-Z0-9]{16}\b`, "${1}"+redactedValue, "AKIA", "ASIA", "ABIA", "ACCA"), // AWS access key id+ shapeRule(`\b(gh[pousr]_)[A-Za-z0-9]{30,}`, "${1}"+redactedValue, "ghp_", "gho_", "ghu_", "ghs_", "ghr_"), // GitHub tokens+ shapeRule(`\b(github_pat_)[A-Za-z0-9_]{22,}`, "${1}"+redactedValue, "github_pat_"), // GitHub fine-grained tokens+ shapeRule(`\b(glpat-)[A-Za-z0-9_-]{20,}`, "${1}"+redactedValue, "glpat-"), // GitLab+ shapeRule(`\b(xox[abposr]-)[A-Za-z0-9-]{10,}`, "${1}"+redactedValue, "xox"), // Slack+ shapeRule(`\b([rs]k_(?:live|test)_)[A-Za-z0-9]{16,}`, "${1}"+redactedValue, "k_live_", "k_test_").before(1), // Stripe+ shapeRule(`\b(AIza)[0-9A-Za-z_-]{35}`, "${1}"+redactedValue, "AIza"), // Google API key+ shapeRule(`\b(ya29\.)(?:c\.)?[0-9A-Za-z_-]{20,}`, "${1}"+redactedValue, "ya29."), // Google OAuth access token+ shapeRule(`\b(npm_)[A-Za-z0-9]{36}`, "${1}"+redactedValue, "npm_"), // npm+ shapeRule(`\b(hf_)[A-Za-z0-9]{30,}`, "${1}"+redactedValue, "hf_"), // Hugging Face// Beacon's own keys and tokens: API keys, device and enforcement keys, MCP tokens and their refresh// tokens, and the CLI sign-in's access token, authorization code, client and exchange values.- rule(`\b(bcn_(?:api|device|mcprt|mcp|enfdev|cli|code|client|exchange)_)[A-Za-z0-9_-]{8,}`, "${1}"+redactedValue, nil, "bcn_"),- rule(`\b(eyJ)[A-Za-z0-9_-]{8,}\.eyJ[A-Za-z0-9_-]{8,}\.[A-Za-z0-9_-]{8,}`, "${1}"+redactedValue, nil, "eyJ"), // JWT+ shapeRule(`\b(bcn_(?:api|device|mcprt|mcp|enfdev|cli|code|client|exchange)_)[A-Za-z0-9_-]{8,}`, "${1}"+redactedValue, "bcn_"),+ shapeRule(`\b(eyJ)[A-Za-z0-9_-]{8,}\.eyJ[A-Za-z0-9_-]{8,}\.[A-Za-z0-9_-]{8,}`, "${1}"+redactedValue, "eyJ").segmented(), // JWT})func concatRules(groups ...[]redactionRule) []redactionRule {@@ -467,6 +482,20 @@ func (rule redactionRule) before(offset int) redactionRule {return rule}+// segmented marks a shape rule whose secret is a JWT: three dot-separated segments, the first two+// opening with `eyJ`, each of any length. redact_adjacent.go reads them itself to tell where one+// ends without reading on to the end of the text each time it asks.+func (rule redactionRule) segmented() redactionRule {+ rule.shape = shapeJWT+ return rule+}++// runningOn returns the rule with the text its matches replace running on where runsOn says.+func (rule redactionRule) runningOn(runsOn func(t string, match []int) int) redactionRule {+ rule.runsOn = runsOn+ return rule+}+// apply replaces every match of the rule in the gate's text, except one whose value keep passes,// and a repeating rule until the text stops changing.func (rule redactionRule) apply(text *redactGate, slot *redactSlot) string {@@ -482,20 +511,31 @@ func (rule redactionRule) apply(text *redactGate, slot *redactSlot) string {func (rule redactionRule) applyOnce(text *redactGate, slot *redactSlot) string {value := text.valuevar out []byte- last := 0- for match, pattern := range rule.matches(text, slot) {- if rule.keep != nil {- index := pattern.SubexpIndex("value")+ last, slashed := 0, false+ for match, use := range rule.matches(text, slot) {+ if match[0] < last {+ continue // inside a match that ran on (redactionRule.runsOn)+ }+ if keep := use.rule.keep; keep != nil {+ index := use.pattern.SubexpIndex("value")if index < 0 {index = 2}- if group := 2 * index; match[group] >= 0 && rule.keep(value[match[group]:match[group+1]]) {+ if group := 2 * index; match[group] >= 0 && keep(value[match[group]:match[group+1]]) {+ text.hidden = text.hidden || keptLabel(value, match[group+1])continue}}out = append(out, value[last:match[0]]...)- out = pattern.ExpandString(out, rule.replacement, value, match)- last = match[1]+ replacement, end := use.rule.replacing(value, match)+ out = use.pattern.ExpandString(out, replacement, value, match)+ last = end+ // An AWS secret may hold a `/`, where the URL rule, which runs before the shape rules,+ // stopped a password; once the secret is replaced, the password can run on to its `@`.+ slashed = slashed || slashedAWSSecret(use.rule.expr, value, match)+ }+ if slashed && strings.Contains(value, "://") {+ text.hidden = true}if out == nil {return value@@ -503,6 +543,128 @@ func (rule redactionRule) applyOnce(text *redactGate, slot *redactSlot) string {return string(append(out, value[last:]...))}+// replacing returns the replacement of match in t, and where the text it replaces ends: the+// match's end, unless the rule's match runs on (runsOn), when what the rule keeps after its value+// goes too.+func (rule *redactionRule) replacing(t string, match []int) (string, int) {+ if rule.runsOn != nil {+ if end := rule.runsOn(t, match); end >= 0 {+ return "${1}" + redactedValue, end+ }+ }+ return rule.replacement, match[1]+}++// urlRunsOn returns where the password of a URL match runs on to when the URL's user name is a+// secret-named label (`https://x-access-token:…@`), which the label rules redact too, or -1. The+// two stop at different bytes, a label's value at a quote, a comma or whitespace and a URL's+// password at an `@`, a slash, a quote or whitespace, and once one has replaced its part, the+// other's runs on over the marker; so each pass of the rules took one more piece of what follows,+// starting with the `@`. The value runs on as far as the two take it in turn, which is where+// every later pass would leave it.+//+// The shape rules run after this one, and one of them changes where either run stops: an AWS key+// id followed by its secret (awsPairAt), a secret recognized only after its key id, which holds+// `/` and `+`. A label's value that ends in the key id would leave the secret with no key id+// before it, so it takes the separators and the secret too; the password reads the pair as the+// markers and separators the shape rule leaves of it.+func urlRunsOn(t string, m []int) int {+ if !labelEndsAt(t[:m[3]]) {+ return -1+ }+ end := m[1]+ for {+ from := end+ // The label's value: the marker the URL rule wrote and what follows it. A key id it ends+ // in, before any separators it takes (`:`, `|`), takes its secret with it.+ for run := end; ; {+ for end < len(t) && !strings.ContainsRune("\"'`,", rune(t[end])) && !isRegexpSpace(t[end]) {+ end+++ }+ k := end+ for k > run && end-k < 8 && (t[k-1] == ':' || t[k-1] == '|') {+ k--+ }+ if k-run < awsKeyIDLen {+ break+ }+ _, secretEnd := awsPairAt(t, k-awsKeyIDLen)+ if secretEnd < 0 {+ break+ }+ end, run = secretEnd, secretEnd+ }+ // The URL's password: that value and what follows it up to an `@`.+ at := end+ for at < len(t) && t[at] != '@' && t[at] != '/' && !strings.ContainsRune("\"'<>", rune(t[at])) && !isRegexpSpace(t[at]) {+ if sep, secretEnd := awsPairAt(t, at); secretEnd >= 0 {+ // The key id and the secret become markers; the separators between them stay.+ if stop := strings.IndexAny(t[sep:secretEnd], "\"'"+regexpSpace); stop >= 0 {+ at = sep + stop+ break+ }+ at = secretEnd+ continue+ }+ at+++ }+ redactWorked(at - from)+ if at == len(t) || t[at] != '@' {+ return end+ }+ end = at + 1+ }+}++// redactAWSPair is an AWS access key id followed by its secret access key within a few separators:+// ${1} the key id's type prefix, ${2} the separators, ${3} what ends the secret.+const redactAWSPair = `(\b(?:AKIA|ASIA))[A-Z0-9]{16}(\b[\s,:|"']{1,8})[A-Za-z0-9/+]{40}([^A-Za-z0-9/+=]|$)`++// awsKeyIDLen is the length of an AWS access key id.+const awsKeyIDLen = 20++var redactAWSPairAt = lazyPattern(`^` + redactAWSPair)++// awsPairAt returns, where an AWS key id that the shape rule for a key id and its secret matches+// starts at s in t, where the separators after the key id start and where the secret ends;+// otherwise -1, -1.+func awsPairAt(t string, s int) (sep, secretEnd int) {+ if s > 0 && isWordByte(t[s-1]) || !strings.HasPrefix(t[s:], "AKIA") && !strings.HasPrefix(t[s:], "ASIA") {+ return -1, -1+ }+ pair := redactAWSPairAt().FindStringSubmatchIndex(t[s:])+ if pair == nil {+ redactWorked(min(len(t)-s, 80))+ return -1, -1+ }+ redactWorked(pair[1])+ return s + pair[4], s + pair[6]+}++// labelEndsAt reports whether s ends with a secret-named label and its separator, as the label+// rules read them: one of redactLabel's markers, an optional closing quote, and `:` or `=` with+// whitespace around. `deploykey:` is no label; `api_key:` is.+func labelEndsAt(s string) bool {+ i := len(s)+ for i > 0 && isRegexpSpace(s[i-1]) {+ i--+ }+ if i == 0 || s[i-1] != ':' && s[i-1] != '=' {+ return false+ }+ for i--; i > 0 && isRegexpSpace(s[i-1]); i-- {+ }+ if i > 0 && strings.IndexByte(redactQuotes, s[i-1]) >= 0 {+ if i--; i > 0 && s[i-1] == '\\' {+ i--+ }+ }+ return redactLabelEnd().MatchString(s[max(0, i-32):i])+}++// redactLabelEnd matches text that ends in one of redactLabel's markers.+var redactLabelEnd = lazyPattern(`(?i)(?:` + strings.TrimPrefix(redactLabel, `[A-Za-z0-9_.-]*?`) + `)$`)+// redactAssignedRule captures the value of a labeled assignment, quoted with ", ' or a backtick// (${1} to ${3}) or not (${4}), so a bare copy of it elsewhere in the same text is// removed too.@@ -545,44 +707,150 @@ func RedactString(value string) string {// label rules cost microseconds on a short string and tens of milliseconds on a tool output of a// few kilobytes. Almost no text holds a needle of most rules, and one pass over the text finds// every needle of every rule at once.+//+// One pass is enough except where a match hides another that only a later rule uncovers. A rule+// does not look inside a match whose value it keeps, nor the capture of assigned values inside a+// value it takes, and that value can itself be a label with its separator (keptLabel): of+// `token:seCret":token":0` the rule for a quoted label with a bare value keeps `seCret":token`+// (`token` names an auth scheme) and never sees `token":0`; the rule for unquoted labels then+// replaces `seCret`, the label that hid it, and only the next pass redacted `0`. Copies of+// assigned values, replaced after every rule has run, can be where a match starts or ends+// (copyHidesMatch); and the URL rule stops a password at a `/` in an AWS secret, which the shape+// rules replace after it. Such text is redacted again until it stops changing, as every later call+// would redact it. A pass can only uncover what the one before hid; texts the tests and fuzzing+// have tried took at most three passes that change them, and redactMaxPasses keeps the cost+// linear whatever the text.func redactString(value string, gated bool) string {+ for pass := 1; ; pass++ {+ out, hidden := redactPass(value, gated)+ if !hidden || out == value || pass == redactMaxPasses {+ return out+ }+ value = out+ }+}++// redactMaxPasses bounds the passes redactString makes over text where a match hid another.+const redactMaxPasses = 8++// redactPass is one pass of every rule over value, and whether a match may have hidden another+// (keptLabel, copyHidesMatch).+func redactPass(value string, gated bool) (string, bool) {engine := redactEngineFor()text := redactGate{value: value, enabled: gated, engine: engine}if text.holdsNothing() {// Most text: no rule's needle, so no set holds and no rule can match.- return value+ return value, false}// Remember assignment values before replacing their labeled occurrence so// repeated bare copies in the same retained prompt/output are also removed.// This is common in model reasoning that quotes `token=value` and later// refers to `value` by itself.var assigned map[string]struct{}+ var kept boolif text.holds(redactLabelNeedles) {- assignedRule := redactAssignedRule()- for match := range assignedRule.matches(&text, &engine.assigned) {- var copied strings.Builder- for g := 2; g+1 < len(match); g += 2 {- if match[g] >= 0 {- copied.WriteString(value[match[g]:match[g+1]])- }- }- if s := copied.String(); len(s) >= 8 && s != redactedValue && !isAuthScheme(s) {- if assigned == nil {- assigned = map[string]struct{}{}- }- assigned[s] = struct{}{}- }- }+ assigned, kept = captureAssigned(&text, engine, assigned)+ text.copies = byFirstBytes(assigned)}+ original := text.valuefor i, rule := range redactionRules {if !text.holds(rule.needles) {continue}text.set(rule.apply(&text, &engine.rules[i]))}- value = text.value- // Each copy is searched for and removed; none of it survives into the result.- return replaceAssigned(value, assigned)+ // A value the label rules keep part of can be captured whole and then changed by a later rule:+ // of `"secret": null}…x-access-token:ghs_…`, the rule for quoted labels keeps `null`, the+ // capture runs on to the end, and the rule for `x-access-token:` replaces the token inside it,+ // so no copy of what was captured is left to remove. What the value became is captured too, so+ // a second pass over the result finds nothing this one did not.+ if kept && text.value != original {+ assigned, _ = captureAssigned(&text, engine, assigned)+ }+ // Each copy is searched for and removed; none of it survives into the result. A secret right+ // after a replaced copy, or right after a replaced key, starts a word only now.+ replaced, glued := replaceAssigned(text.value, assigned)+ hidden := text.hidden || replaced != text.value && copyHidesMatch(assigned, glued)+ out, slashed := engine.redactAfterMarkers(replaced)+ return out, hidden || slashed && strings.Contains(out, "://")+}++// copyHidesMatch reports whether replacing copies of assigned values, which happens after every+// rule has run, may have made a match where the rules found none: where a copy that starts with a+// byte no word starts with was, which some rules' values cannot start with and their marker can;+// where a copy held a byte a rule's value stops at (`/`, `@`, a quote, a comma, a blank), which+// its marker does not; or where a word byte comes right before one (glued), which ends a word, or+// a redacted secret, only now. Of `-pAsswd --token -token:"--token ` the flag finds no value in+// `--token`, which starts with a dash, until the copy of the value `--token ` is replaced; of+// `AKIAIOSFODNN7EXAMPLEAuthorization: Authorization: ` the key id rule finds no end before the+// copy of `Authorization:` that the label holds. The next pass redacts them.+func copyHidesMatch(assigned map[string]struct{}, glued bool) bool {+ if glued {+ return true+ }+ for s := range assigned {+ if !isWordByte(s[0]) || strings.IndexFunc(s, stopsAValue) >= 0 {+ return true+ }+ }+ return false+}++// stopsAValue reports whether some rule's value may stop at r: any byte but a word byte, `.` and+// `-`.+func stopsAValue(r rune) bool {+ return r >= utf8.RuneSelf || !isWordByte(byte(r)) && r != '.' && r != '-'+}++// markersAfterWords counts the markers in t that a word byte comes right before.+func markersAfterWords(t string) int {+ n := 0+ for from := 0; ; {+ i := strings.Index(t[from:], redactedValue)+ if i < 0 {+ return n+ }+ if p := from + i; p > 0 && isWordByte(t[p-1]) {+ n+++ }+ from += i + len(redactedValue)+ }+}++// keptLabel reports whether a value that ends at end in t, one a rule keeps or the capture takes,+// is itself a secret-named label followed by its separator, as `token` is in `"secret":token":0`.+func keptLabel(t string, end int) bool {+ rest, ok := skipSeparator(t[end:])+ return ok && labelEndsAt(t[:len(t)-len(rest)])+}++// captureAssigned adds the value of every labeled assignment in the gate's text to assigned. kept+// reports whether one, of any length, was unquoted after a quoted label: the one way a value is+// captured that the label rules may keep part of (redactBare stops where the capture runs on), so+// that a later rule can change it, or make it long enough to remove.+func captureAssigned(text *redactGate, engine *redactEngine, assigned map[string]struct{}) (_ map[string]struct{}, kept bool) {+ value := text.value+ for match := range redactAssignedRule().matches(text, &engine.assigned) {+ // The value is in one group: each way of writing it is an alternative.+ g := 2+ for g+1 < len(match) && match[g] < 0 {+ g += 2+ }+ if g+1 >= len(match) {+ continue+ }+ kept = kept || g+2 == len(match) && strings.ContainsAny(value[match[0]:match[g]], redactQuotes)+ // A value that is itself a label hides the assignment it begins from the capture, as a kept+ // one does from the label rules.+ text.hidden = text.hidden || keptLabel(value, match[g+1])+ if s := value[match[g]:match[g+1]]; len(s) >= redactCopyMin && s != redactedValue && !isAuthScheme(s) {+ if assigned == nil {+ assigned = map[string]struct{}{}+ }+ assigned[s] = struct{}{}+ }+ }+ return assigned, kept}// replaceAssigned replaces every copy of each assigned value with redactedValue in one pass.@@ -590,9 +858,13 @@ func redactString(value string, gated bool) string {// grow with the text, so the cost was quadratic: Beacon Cloud redacts uploaded shared traces with// this function, where that let a small gzip body hold a core for minutes. The values are a set,// longest first, so where two start at the same byte the longer is removed whole.-func replaceAssigned(value string, assigned map[string]struct{}) string {+//+// It also reports whether a word byte comes right before a copy it replaced (copyHidesMatch). The+// copies are first replaced with a NUL, where the text holds none, so their places can be read;+// where it holds one, the markers that follow a word byte are counted before and after instead.+func replaceAssigned(value string, assigned map[string]struct{}) (string, bool) {if len(assigned) == 0 {- return value+ return value, false}values := slices.SortedFunc(maps.Keys(assigned), func(a, b string) int {if len(a) != len(b) {@@ -600,11 +872,51 @@ func replaceAssigned(value string, assigned map[string]struct{}) string {}return strings.Compare(a, b)})+ place := "\x00"+ if strings.IndexByte(value, 0) >= 0 {+ place = redactedValue+ }pairs := make([]string, 0, 2*len(values))for _, copied := range values {- pairs = append(pairs, copied, redactedValue)+ pairs = append(pairs, copied, place)+ }+ replaced := strings.NewReplacer(pairs...).Replace(value)+ if place == redactedValue {+ return replaced, markersAfterWords(replaced) > markersAfterWords(value)+ }+ glued := false+ for i := strings.IndexByte(replaced, 0); i >= 0 && !glued; {+ glued = i > 0 && isWordByte(replaced[i-1])+ j := strings.IndexByte(replaced[i+1:], 0)+ if j < 0 {+ break+ }+ i += 1 + j+ }+ return strings.ReplaceAll(replaced, place, redactedValue), glued+}++// redactCopyMin is the least length of an assigned value whose copies are removed.+const redactCopyMin = 8++// redactCopies are the assigned values by their first redactCopyMin bytes, so a place in the text+// is looked up once to tell whether a copy may start there, and the length of the longest+// (redactChain.overCopy).+type redactCopies struct {+ byFirst map[string][]string+ longest int+}++func byFirstBytes(assigned map[string]struct{}) *redactCopies {+ if len(assigned) == 0 {+ return nil+ }+ copies := &redactCopies{byFirst: map[string][]string{}}+ for s := range assigned {+ copies.byFirst[s[:redactCopyMin]] = append(copies.byFirst[s[:redactCopyMin]], s)+ copies.longest = max(copies.longest, len(s))}- return strings.NewReplacer(pairs...).Replace(value)+ return copies}// IsSecretKeyName reports whether a structured field's name has a word naming a credential, fordiff --git a/pkg/asymptoteobserve/privacy_test.go b/pkg/asymptoteobserve/privacy_test.goindex 533a6f36..53e7d7f4 100644--- a/pkg/asymptoteobserve/privacy_test.go+++ b/pkg/asymptoteobserve/privacy_test.go@@ -231,15 +231,7 @@ func TestRedactStringKeepsLabelsAndOrdinaryText(t *testing.T) {// Events are sanitized again on every read path, so a second pass must not change the first one's// output.func TestRedactStringIsIdempotent(t *testing.T) {- inputs := []string{- "config dump\napi_key=sk-abcdefghijklmnopqrstuvwxyz0123\nOPENAI sk-abcdefghijklmnopqrstuvwxyz0123",- `{"token": "opaque-session-value-9", "password": "hunter2hunter2"}`,- "Authorization: Bearer abc.def.ghi and --password hunter2hunter2",- "postgres://app:pa55w0rd@db/app",- "curl -u admin:hunter2hunter2 -H 'Cookie: sid=abcdef123456789' https://x",- "curl -b 'sessionid=abcdef123456789' --cookie sid=abcdef123456789 https://x",- }- for _, input := range inputs {+ for _, input := range redactIdempotenceInputs {once := RedactString(input)if twice := RedactString(once); twice != once {t.Fatalf("RedactString is not idempotent:\nonce %q\ntwice %q", once, twice)@@ -247,6 +239,81 @@ func TestRedactStringIsIdempotent(t *testing.T) {}}+// A label hidden inside a value a rule kept is redacted in the same call, as a second call would.+func TestRedactStringRedactsALabelAKeptValueHid(t *testing.T) {+ for input, want := range map[string]string{+ `token:seCret":token":0`: `token:[REDACTED]":token":"[REDACTED]"`,+ `--token "secret": "token": "abc"`: `--token "[REDACTED]": "token": "[REDACTED]"`,+ `--password seCret":token":abcdefg`: `--password [REDACTED]":token":"[REDACTED]"`,+ } {+ if got := RedactString(input); got != want {+ t.Errorf("RedactString(%q) = %q, want %q", input, got, want)+ }+ }+}++// redactIdempotenceInputs are texts one pass must leave nothing in for a second.+// FuzzRedactIsIdempotent starts from them too.+var redactIdempotenceInputs = []string{+ "config dump\napi_key=sk-abcdefghijklmnopqrstuvwxyz0123\nOPENAI sk-abcdefghijklmnopqrstuvwxyz0123",+ `{"token": "opaque-session-value-9", "password": "hunter2hunter2"}`,+ "Authorization: Bearer abc.def.ghi and --password hunter2hunter2",+ "postgres://app:pa55w0rd@db/app",+ "curl -u admin:hunter2hunter2 -H 'Cookie: sid=abcdef123456789' https://x",+ "curl -b 'sessionid=abcdef123456789' --cookie sid=abcdef123456789 https://x",+ // A URL whose user name is a secret-named label: the label rules and the URL rule stop at+ // different bytes, and each used to take one more piece per pass.+ "https://x-access-token:[email protected]_api_`@host",+ "A0000000000://token:,@",+ "A://token:,@0,@0,@",+ // A curl flag's blanks run over a newline to its value; the command follows a key.+ "npm_000000000000000000000000000000000000Curl -B\n0=",+ // The value captured after the quoted label is too short to remove until the URL rule+ // replaces the comma that ends it.+ `seCret":}A://:0,@`,+ // A kept value that is itself a label: `token`, an auth scheme, is kept as the value of+ // `seCret":` and hides `token":0` from the rule, until a later rule replaces `seCret` (here+ // the value of `token:`, a flag's value, a GitHub token's tail) and a second pass finds it.+ `token:seCret":token":0`,+ `token:seCret":token":abc`,+ `token:seCret':token':0`,+ `token:seCret":Token":0`,+ `--password seCret":token":0`,+ `--token "secret": "token": "abc"`,+ `ghp_0123456789abcdefghijklmnopqrstsecret":token":0`,+ // Two such labels in turn, and one that took three passes to settle.+ `token:seCret":token":seCret2": "token": "abc"`,+ `0secret=token :ghp_0123456789abcdefghijklmnopqrstsecret=token :seCret':token':@h'`,+ // The capture of assigned values takes `token` as the value of `secret` and so never reads+ // `token`:}token:--token` as an assignment; the label that hid it is replaced.+ "token:abctoken\":`secret`:`token`:}token:--token \"token\"",+ // A flag finds no value where a copy of an assigned value that starts with a dash follows it;+ // once the copy is replaced, a second pass redacted the marker and what follows it.+ `-pAsswd --token -token:"--token `,+ "--password --token ghp_0123456789abcdefghijklmnopqrsta=@h:token:'--token ",+ // An AWS key id that a copy of an assigned value follows ends a word only once the copy is+ // replaced, after every rule has run.+ "AKIAIOSFODNN7EXAMPLEAuthorization: Authorization: ",+ "@AKIAIOSFODNN7EXAMPLEx-access-token:x-access-token:",+ "AKIAIOSFODNN7EXAMPLEAKIAIOSFODNN7EXAMPLEAuthorization: Authorization: ",+ // The copy of `Authoriz` makes `ABIA[REDACTED]` of a label, which ends the key id before it.+ "ABIA0000000000000000ABIAAuthorizAtion:Authoriz",+ // The copy of `https://` holds the slash a URL's password stops at, and its marker does not.+ "A://token:https://,ghp_0123456789abcdefghijklmnopqrst==https://@h\n=",+ // A token runs on over that copy (overCopy), which takes its slash as replacing it would.+ "A://token:https://,ghp_0000000000000000000000000https://@",+ // A URL's password stops at the `/` of an AWS secret until the secret is replaced.+ "https://u:a,AKIAIOSFODNN7EXAMPLE,wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY@host",+ // The same where a token's item takes the key id and secret, and where they come after a+ // replaced copy (redactAfterMarkers).+ "https://u:pw,sk-abcAKIAIOSFODNN7EXAMPLE,wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY@ host",+ "token=abcdefgh12345678 https://u:pw,abcdefgh12345678AKIAIOSFODNN7EXAMPLE,wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY@ host",+ // The copy the key id ends before is followed by a marker already: what follows a word byte+ // is told by place, not by count.+ "token=abcdefgh12345678 AKIAIOSFODNN7EXAMPLEabcdefgh12345678[REDACTED]",+ "https://x-access-token:a@b,AKIAIOSFODNN7EXAMPLE,wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY@host",+}+// A field named for a credential is one, whatever its value looks like.func TestSanitizeMapRedactsValuesUnderSecretKeys(t *testing.T) {input := map[string]interface{}{diff --git a/pkg/asymptoteobserve/redact_adjacent.go b/pkg/asymptoteobserve/redact_adjacent.gonew file mode 100644index 00000000..a4a9f70a--- /dev/null+++ b/pkg/asymptoteobserve/redact_adjacent.go@@ -0,0 +1,1246 @@+package asymptoteobserve++import (+ "regexp"+ "regexp/syntax"+ "slices"+ "sort"+ "strconv"+ "strings"+ "unicode/utf8"+)++// This file is how redaction finds a secret written right after another one.+//+// The rules for credentials recognized by their own shape (privacy.go: `\b(ghp_)[A-Za-z0-9]{30,}`+// and the rest) open with \b, so they match a secret that starts a word. A secret written right+// after another one does not start a word, and a pass of the rules used to leave it:+//+// - A pattern of fixed length stops inside a run of word characters. Of two Google keys back to+// back, `AIza…AIza…`, the second has a letter before it. Once the first is replaced, `]` is+// before it, so a second pass redacted what the first left.+// - An open-ended pattern runs on into the next secret and stops at the first byte its class+// lacks. Of two GitHub tokens back to back, the first match takes the second's `ghp` and stops+// at its `_`, and the rest of the second token was left whole by every pass.+// - A pattern that ends in \b or in a byte that is no key character, the AWS key id's and the+// AWS secret's, does not match where the next secret follows it with nothing between them.+// - A rule that runs before the shape rules, for a URL's password, a Bearer token or a curl+// password, finds no word start after a key that has not been replaced yet; nor does any rule+// after a bare copy of an assigned value, which is replaced after every rule has run.+//+// So the end of a secret counts as the start of a word, and one pass redacts all of it:+//+// - When a shape rule matches, redactChain.run looks at what follows. If the match ran on into+// a secret that starts inside it and ends past it, the match is cut where that secret starts,+// or, too short to be cut there, runs on to that secret's end; if a secret of any shape rule+// starts right where the match ends, with no word boundary between them, it is redacted with+// it; and so on along the chain. Each secret keeps its own prefix: `ghp_[REDACTED]ghp_[REDACTED]`.+// - A pattern that fails only for the secret right after it matches with the text cut where+// that secret starts (redactChain.rescue).+// - After every rule has run and bare copies are replaced, the rules that open with \b and are+// not label rules are tried once more where a replacement ends right before a word byte+// (redactAfterMarkers).+//+// Text where no secret meets another redacts as it did. Every check reads a slice whose length+// is bounded by the pattern's own shape, or text that the chain then replaces, so redaction's+// work stays linear; TestRedactionWorkIsLinear counts it.++// redactShapeKind says whether a rule is a shape rule (redactionRule.shape), and which kind.+type redactShapeKind uint8++const (+ shapeNone redactShapeKind = iota+ shapeToken+ // shapeJWT: three dot-separated segments of any length, which redactChain reads itself.+ shapeJWT+)++// redactShape is what the adjacency checks know of one shape rule's pattern.+type redactShape struct {+ rule int+ needles uint64 // scanner needle bits+ offset int // the rule's offset: how many bytes before its needle a match starts+ // tail, when set, is the class of the open-ended repeat the pattern ends in, at least tailMin+ // long, after at most head bytes. Without it the pattern matches at most bound-1 bytes.+ tail *[256]bool+ tailMin int+ head int+ // bound is how long a slice from a match's start must be for the pattern to find in it what+ // it finds in the whole text: the same match, or, for an open-ended pattern, one that reaches+ // the slice's end where the whole match runs on (redactChain.shapeAt).+ bound int+ jwt bool+ // redacted matches the rule's replacement at the start of a text: its first group, at most+ // prefix bytes, and a marker.+ redacted *regexp.Regexp+ prefix int+ // trailing is the group the replacement ends with, or 0 when it ends in a marker.+ trailing int+}++// redactFollower is a rule that opens with \b and is neither a label rule nor a shape rule: a+// Bearer token, a curl password or cookie, a URL's password. redactAfterMarkers tries it at the+// start of a word that a replacement made.+type redactFollower struct {+ rule int+ needles uint64+ kind redactFollowerKind+ // charset is every byte the pattern can match, for followRun.+ charset [256]bool+ // lead, for followLine, is the pattern anchored at its needle with the lazy run after the+ // command word captured as `lead`: from any later place on the line up to that run's end, the+ // pattern ends where it ended from this one (lineFollowerAt).+ lead *regexp.Regexp+}++type redactFollowerKind uint8++const (+ // followRun: a word (its one needle), blanks, and a value: the match lies in the needle, the+ // blanks after it and the run of its charset without blanks after them (Bearer).+ followRun redactFollowerKind = iota+ // followScheme: a URL, which holds no `/` and no whitespace after its `://`.+ followScheme+ // followLine: a curl rule, which reads its whole command line and is applied to it until it+ // stops changing (redactLinesAfterMarkers).+ followLine+)++// redactAdjacency is what the adjacency checks know of the rules, built the first time a chain or+// a marker is looked at.+type redactAdjacency struct {+ shapes []redactShape // in rule order+ followers []redactFollower+ byRule []int // rule index -> index into shapes, or -1+}++func (e *redactEngine) adjacency() *redactAdjacency {+ e.adjacentOnce.Do(func() {+ a := &redactAdjacency{byRule: make([]int, len(redactionRules))}+ for i, rule := range redactionRules {+ a.byRule[i] = -1+ parsed, err := syntax.Parse(rule.expr, syntax.Perl)+ if err != nil {+ panic(err)+ }+ if rule.shape != shapeNone {+ a.byRule[i] = len(a.shapes)+ a.shapes = append(a.shapes, shapeOf(i, rule, parsed, e.setMasks[rule.needles.id]))+ continue+ }+ if !opensWithBoundary(rule.expr) || rule.start == startLabel {+ continue+ }+ if len(a.shapes) > 0 {+ panic("asymptoteobserve: a rule that opens with \\b must come before the shape rules: " + rule.expr)+ }+ f := redactFollower{rule: i, needles: e.setMasks[rule.needles.id]}+ switch {+ case rule.start == startScheme:+ f.kind = followScheme+ case rule.repeat:+ f.kind = followLine+ f.lead = leadPattern(rule)+ default:+ f.kind = followRun+ addCharset(&f.charset, parsed)+ for c := utf8.RuneSelf; c < 256; c++ {+ f.charset[c] = true+ }+ if f.charset['['] || f.charset['\n'] || len(rule.needles.needles) != 1 ||+ !strings.Contains(rule.expr, `\b`+rule.needles.needles[0]+`[ \t]+`) {+ panic("asymptoteobserve: a run follower has one needle, a blank after it, and cannot match '[' or a newline: " + rule.expr)+ }+ }+ a.followers = append(a.followers, f)+ }+ e.adjacent = a+ })+ return e.adjacent+}++// opensWithBoundary reports whether a pattern opens with \b, as redactSlot.prepare reads it.+func opensWithBoundary(expr string) bool {+ return slices.ContainsFunc(boundaryPrefixes, func(prefix string) bool { return strings.HasPrefix(expr, prefix) })+}++// leadPattern is a followLine rule's pattern anchored at its one needle, without the \b it opens+// with, and with the lazy run after the needle captured as `lead`.+func leadPattern(rule redactionRule) *regexp.Regexp {+ const run = `\bcurl\b[^\n]*?`+ if len(rule.needles.needles) != 1 || !strings.Contains(rule.expr, "("+run) {+ panic("asymptoteobserve: a line follower opens with `curl` and the lazy run of its line: " + rule.expr)+ }+ return regexp.MustCompile("^" + strings.Replace(rule.expr, "("+run, `(curl\b(?P<lead>[^\n]*?)`, 1))+}++// shapeOf reads a shape rule's pattern: a concatenation that ends, before any trailing assertion,+// either in an open-ended repeat of one class or in something of bounded length.+func shapeOf(index int, rule redactionRule, parsed *syntax.Regexp, needles uint64) redactShape {+ s := redactShape{rule: index, needles: needles, offset: rule.offset, jwt: rule.shape == shapeJWT}+ parts := []*syntax.Regexp{parsed}+ if parsed.Op == syntax.OpConcat {+ parts = parsed.Sub+ }+ for len(parts) > 0 && maxLength(parts[len(parts)-1]) == 0 {+ parts = parts[:len(parts)-1]+ }+ last := parts[len(parts)-1]+ if last.Op == syntax.OpRepeat && last.Max == -1 && last.Sub[0].Op == syntax.OpCharClass {+ s.tail = new([256]bool)+ addCharset(s.tail, last.Sub[0])+ s.tailMin = last.Min+ s.head = 0+ for _, part := range parts[:len(parts)-1] {+ if n := maxLength(part); n < 0 || s.head < 0 {+ s.head = -1+ } else {+ s.head += n+ }+ }+ s.bound = s.head + s.tailMin+ } else if n := maxLength(parsed); n >= 0 {+ s.bound = n + 1+ }+ group := firstGroup(parsed)+ if !strings.HasSuffix(rule.replacement, redactedValue) {+ i := strings.LastIndex(rule.replacement, "${")+ if i >= 0 && strings.HasSuffix(rule.replacement, "}") {+ s.trailing, _ = strconv.Atoi(rule.replacement[i+2 : len(rule.replacement)-1])+ }+ }+ if s.jwt != (s.head < 0) || s.tail == nil && s.bound == 0 || group == nil || maxLength(group) < 0 ||+ s.trailing == 0 && !strings.HasSuffix(rule.replacement, redactedValue) {+ panic("asymptoteobserve: a shape rule's pattern has a shape the adjacency checks do not know: " + rule.expr)+ }+ s.prefix = maxLength(group)+ s.redacted = regexp.MustCompile(`^(?:` + group.Sub[0].String() + `)` + regexp.QuoteMeta(redactedValue))+ return s+}++// firstGroup returns re's first capturing group, or nil.+func firstGroup(re *syntax.Regexp) *syntax.Regexp {+ if re.Op == syntax.OpCapture {+ return re+ }+ for _, sub := range re.Sub {+ if group := firstGroup(sub); group != nil {+ return group+ }+ }+ return nil+}++// maxLength returns the most bytes re can match, or -1 when that is unbounded.+func maxLength(re *syntax.Regexp) int {+ switch re.Op {+ case syntax.OpLiteral:+ n := 0+ for _, r := range re.Rune {+ n += utf8.RuneLen(r)+ }+ return n+ case syntax.OpCharClass:+ if len(re.Rune) > 0 && re.Rune[len(re.Rune)-1] < utf8.RuneSelf {+ return 1+ }+ return utf8.UTFMax+ case syntax.OpAnyChar, syntax.OpAnyCharNotNL:+ return utf8.UTFMax+ case syntax.OpCapture, syntax.OpQuest:+ return maxLength(re.Sub[0])+ case syntax.OpConcat, syntax.OpAlternate:+ n := 0+ for _, sub := range re.Sub {+ m := maxLength(sub)+ if m < 0 {+ return -1+ }+ if re.Op == syntax.OpConcat {+ n += m+ } else {+ n = max(n, m)+ }+ }+ return n+ case syntax.OpRepeat:+ if m := maxLength(re.Sub[0]); re.Max >= 0 && m >= 0 {+ return re.Max * m+ }+ return -1+ case syntax.OpStar, syntax.OpPlus:+ return -1+ }+ return 0 // an assertion or an empty match+}++// addCharset adds every ASCII byte re can match to set.+func addCharset(set *[256]bool, re *syntax.Regexp) {+ switch re.Op {+ case syntax.OpLiteral:+ for _, r := range re.Rune {+ if r < utf8.RuneSelf {+ set[r] = true+ if re.Flags&syntax.FoldCase != 0 {+ set[asciiLower(byte(r))] = true+ set[asciiUpper(byte(r))] = true+ }+ }+ }+ case syntax.OpCharClass:+ for i := 0; i+1 < len(re.Rune); i += 2 {+ for r := re.Rune[i]; r <= re.Rune[i+1] && r < utf8.RuneSelf; r++ {+ set[r] = true+ }+ }+ case syntax.OpAnyChar, syntax.OpAnyCharNotNL:+ for c := range utf8.RuneSelf {+ set[c] = c != '\n' || re.Op == syntax.OpAnyChar+ }+ }+ for _, sub := range re.Sub {+ addCharset(set, sub)+ }+}++func asciiUpper(c byte) byte {+ if 'a' <= c && c <= 'z' {+ return c - 'a' + 'A'+ }+ return c+}++// jwtSegment is the class of a JWT segment's bytes.+var jwtSegment = func() (set [256]bool) {+ for c := range 256 {+ b := byte(c)+ set[c] = isASCIILetter(b) || '0' <= b && b <= '9' || b == '_' || b == '-'+ }+ return set+}()++// redactChain finds the secrets written right after a shape rule's matches in one text.+type redactChain struct {+ engine *redactEngine+ adj *redactAdjacency+ t string+ hits []redactHit+ // The end of the run of JWT segment bytes from segFrom, and the end of a JWT whose first+ // segment ends at restAt (-1: none): each is read once however many places ask.+ segFrom, segEnd int+ restAt, restEnd int+ // What secretAt found where it was asked.+ secrets map[int]redactSecret+ // The end of the run of URL scheme bytes from schemeFrom, and the URL follower's match from+ // the run's first place asked (schemeAt; nil: none), which any letter of the run shares.+ schemeFrom, schemeTo int+ schemeAt int+ schemeMatch []int+ // The end of the line holding lineFrom; and, per followLine follower, the places on a line+ // where it finds nothing (from the earliest place where it found nothing to the end of that+ // place's line) and where it finds the match it found from an earlier place (from that place+ // to the end of its lead, and where the match ends).+ lineFrom, lineTo int+ lineFirst int // the end of lineFrom's own line, at or before lineTo+ lineFailed [][2]int+ lineFound [][3]int+ // The end of the run of a followRun follower's charset from runFrom, and per follower the+ // last place followerAt was asked about and its answer.+ runFrom, runTo int+ asked []redactAsked+ // The assigned values whose copies are removed after every rule has run (overCopy), what+ // running on over them may still read (copyBudget), and whether running on over one, or+ // over an AWS secret that holds a `/`, may hide a match from this pass, as replacing a copy+ // may (copyHidesMatch); matches reports it to the gate.+ copies *redactCopies+ copyBudget int+ hid bool+ slashed bool+}++// redactAsked is followerAt's answer for one follower at a place.+type redactAsked struct {+ at int+ m []int+}++func newRedactChain(engine *redactEngine, t string, hits []redactHit) *redactChain {+ return &redactChain{engine: engine, adj: engine.adjacency(), t: t, hits: hits, segFrom: -1, restAt: -1,+ schemeFrom: -1, schemeTo: -1, schemeAt: -1, lineFrom: -1, lineTo: -1, lineFirst: -1, runFrom: -1, runTo: -1,+ copyBudget: redactCopyWorkPerByte*len(t) + redactCopyWorkBase}+}++// overCopy may read redactCopyWorkPerByte bytes and copies per byte of the text, and+// redactCopyWorkBase more; past that it leaves items as they are, as main does.+const (+ redactCopyWorkPerByte = 8+ redactCopyWorkBase = 1 << 16+)++func redactWorked(n int) {+ if redactWork != nil {+ redactWork(n)+ }+}++// pattern is rule i's pattern anchored at the start of the text, without its opening \b.+func (c *redactChain) pattern(i int) *regexp.Regexp {+ slot := &c.engine.rules[i]+ slot.prepare(redactionRules[i])+ return slot.anchored+}++// find runs rule i's anchored pattern on t[q:end].+func (c *redactChain) find(i, q, end int) []int {+ redactWorked(end - q)+ m := c.pattern(i).FindStringSubmatchIndex(c.t[q:end])+ for j := range m {+ if m[j] >= 0 {+ m[j] += q+ }+ }+ return m+}++// search returns the first match of re, a shape rule's whole pattern, in t[pos:], as indexes into+// t. It reads up to the match's end.+func (c *redactChain) search(re *regexp.Regexp, pos int) []int {+ m := re.FindStringSubmatchIndex(c.t[pos:])+ if m == nil {+ redactWorked(len(c.t) - pos)+ return nil+ }+ redactWorked(m[1])+ for j := range m {+ if m[j] >= 0 {+ m[j] += pos+ }+ }+ return m+}++// hitsFrom returns the index of the first hit that starts at or after pos.+func hitsFrom(hits []redactHit, pos int) int {+ return sort.Search(len(hits), func(i int) bool { return hits[i].start >= pos })+}++// hasNeedleAt reports whether one of needles occurs at pos.+func (c *redactChain) hasNeedleAt(needles uint64, pos int) bool {+ for i := hitsFrom(c.hits, pos); i < len(c.hits) && c.hits[i].start == pos; i++ {+ if needles&(1<<c.hits[i].id) != 0 {+ return true+ }+ }+ return false+}++// shapeAt runs shape si's pattern at q, without the \b it opens with, on a slice that decides it:+// the match, and whether it runs on past the slice. Its needle must occur at q.+func (c *redactChain) shapeAt(si, q int) (m []int, open bool) {+ s := &c.adj.shapes[si]+ if s.jwt {+ if end := c.jwtEnd(q, q+3); end >= 0 {+ return c.find(s.rule, q, end), false+ }+ return nil, false+ }+ end := min(len(c.t), q+s.bound)+ m = c.find(s.rule, q, end)+ return m, m != nil && s.tail != nil && m[1] == end && end < len(c.t) && s.tail[c.t[end]]+}++// whole returns the match shapeAt found, run on to its end where it is open.+func (c *redactChain) whole(si, q int, m []int, open bool) []int {+ if !open {+ return m+ }+ s := &c.adj.shapes[si]+ end := m[1]+ for end < len(c.t) && s.tail[c.t[end]] {+ end+++ }+ redactWorked(end - m[1])+ return c.find(s.rule, q, end)+}++// rescue returns a match of shape si at q that ends where the next secret starts, or nil. A+// pattern of bounded length that ends in an assertion or a terminator, the AWS key id's \b and the+// AWS secret's byte that is no key character, fails where a secret follows with nothing between+// them; cut where that secret starts, the text ends right after it, and it matches. The next+// secret may already be replaced, by a rule that runs earlier. The pattern's own match at q, if+// any, must have been tried first.+func (c *redactChain) rescue(si, q int) []int {+ s := &c.adj.shapes[si]+ if s.tail != nil {+ return nil+ }+ for i := hitsFrom(c.hits, q+1); i < len(c.hits) && c.hits[i].start < q+s.bound+1; i++ {+ h := c.hits[i]+ for sj := range c.adj.shapes {+ x := h.start - c.adj.shapes[sj].offset+ if c.adj.shapes[sj].needles&(1<<h.id) == 0 || x <= q || x >= q+s.bound {+ continue+ }+ if m := c.find(s.rule, q, x); m != nil && m[1] == x {+ if _, next := c.secretAt(x); next != nil || c.redactedAt(x) || c.followerStarts(x) {+ return m+ }+ }+ }+ }+ // A URL, Bearer token or curl command right after it.+ for x := q + 1; x < min(len(c.t), q+s.bound); x++ {+ if isASCIILetter(c.t[x]) && c.followerStarts(x) {+ if m := c.find(s.rule, q, x); m != nil && m[1] == x {+ return m+ }+ }+ }+ return nil+}++// redactedAt reports whether a shape rule's replacement starts at x: its prefix and a marker.+func (c *redactChain) redactedAt(x int) bool {+ for si := range c.adj.shapes {+ s := &c.adj.shapes[si]+ if !c.hasNeedleAt(s.needles, x+s.offset) {+ continue+ }+ end := min(len(c.t), x+s.prefix+len(redactedValue))+ redactWorked(end - x)+ if s.redacted.MatchString(c.t[x:end]) {+ return true+ }+ }+ return false+}++// rescueBefore returns the first match rescue finds at a place of shape si from pos up to limit,+// where the pattern opens on a word boundary, or nil.+func (c *redactChain) rescueBefore(si, pos, limit int) []int {+ s := &c.adj.shapes[si]+ if s.tail != nil {+ return nil+ }+ for i := hitsFrom(c.hits, pos+s.offset); i < len(c.hits) && c.hits[i].start-s.offset < limit; i++ {+ q := c.hits[i].start - s.offset+ if s.needles&(1<<c.hits[i].id) == 0 || !wordBoundary(c.t, q) {+ continue+ }+ if m := c.rescue(si, q); m != nil {+ return m+ }+ }+ return nil+}++// secretAt returns the first shape rule, in rule order, whose pattern matches at p or is rescued+// there, and its match: the secret that starts at p when p counts as the start of a word. A rescue+// asks what starts where its match ends, so along a chain of key ids each place is asked about+// more than once; the answers are kept.+func (c *redactChain) secretAt(p int) (int, []int) {+ if found, ok := c.secrets[p]; ok {+ return found.shape, found.match+ }+ shape, match := -1, []int(nil)+ for si := range c.adj.shapes {+ s := &c.adj.shapes[si]+ if !c.hasNeedleAt(s.needles, p+s.offset) {+ continue+ }+ if m, open := c.shapeAt(si, p); m != nil {+ shape, match = si, c.whole(si, p, m, open)+ break+ }+ if m := c.rescue(si, p); m != nil {+ shape, match = si, m+ break+ }+ }+ if c.secrets == nil {+ c.secrets = map[int]redactSecret{}+ }+ c.secrets[p] = redactSecret{shape, match}+ return shape, match+}++// redactSecret is what secretAt found at a place.+type redactSecret struct {+ shape int+ match []int+}++// use is how shape si's matches are yielded: its anchored pattern and its rule.+func (c *redactChain) use(si int) redactUse {+ i := c.adj.shapes[si].rule+ return redactUse{c.pattern(i), &redactionRules[i]}+}++// run yields m, a match of shape si, and the secrets written right after it, and returns where+// the last of them ends, or false when yield asked to stop.+//+// A secret that starts inside the match and ends past it (crossing) is one the match ran on into.+// Where the match is still whole without the part from that secret on, it is cut there and the+// secret is the next item. Where it is not, the two are one item, run on to the second's end: of+// `sk-eyJhbGciOiJIUzI1NiJ9.eyJ…`, the key's pattern takes the JWT's first segment and stops at+// the dot, too short a key without it, and the item takes the whole JWT.+func (c *redactChain) run(si int, m []int, use redactUse, yield func([]int, redactUse) bool) (int, bool) {+ // item is what is yielded, with use and the groups of shape. si and m are the secret it ends+ // with, item itself unless merged. least is the first place item can be cut and stay whole.+ item, shape, merged := m, si, false+ least := c.least(si, m)+ for {+ q, sj, sm := -1, 0, []int(nil)+ if c.adj.shapes[shape].trailing == 0 && c.adj.shapes[si].trailing == 0 {+ q, sj, sm = c.crossing(si, m)+ }+ if q >= 0 {+ var cut []int+ if q >= least && merged {+ cut = slices.Clone(item)+ cut[1] = q+ } else if q >= least {+ if cut = c.find(c.adj.shapes[shape].rule, item[0], q); cut != nil && cut[1] != q {+ cut = nil+ }+ }+ if cut != nil {+ if !yield(cut, use) {+ return 0, false+ }+ item, shape, merged, use, least = sm, sj, false, c.use(sj), c.least(sj, sm)+ si, m = sj, sm+ continue+ }+ // The item keeps all of itself so far, and a cut can now fall only past that. What the+ // secret's own replacement would keep after it, it keeps.+ least, merged = item[1], true+ item = slices.Clone(item)+ item[1] = sm[1]+ if g := c.adj.shapes[sj].trailing; g > 0 && sm[2*g] >= 0 {+ item[1] = sm[2*g]+ }+ // The item is yielded with the first secret's rule: an AWS secret it takes is not seen+ // as one where the replacements are made (redactionRule.applyOnce).+ c.slashed = c.slashed || slashedAWSSecret(redactionRules[c.adj.shapes[sj].rule].expr, c.t, sm)+ si, m = sj, sm+ continue+ }+ // A URL, Bearer token or curl command whose first word the match took: cut the match+ // before it, and the follower is redacted where the marker ends (redactAfterMarkers); too+ // short to be cut there, the item runs on over the follower's match.+ if c.adj.shapes[shape].trailing == 0 {+ if q, fend := c.followerCrossing(m); q >= 0 {+ var cut []int+ if q >= least && merged {+ cut = slices.Clone(item)+ cut[1] = q+ } else if q >= least {+ if cut = c.find(c.adj.shapes[shape].rule, item[0], q); cut != nil && cut[1] != q {+ cut = nil+ }+ }+ if cut == nil {+ cut = slices.Clone(item)+ cut[1] = fend+ }+ return cut[1], yield(cut, use)+ }+ }+ if c.adj.shapes[shape].trailing == 0 {+ item = c.overCopy(item)+ }+ if !yield(item, use) {+ return 0, false+ }+ end := item[1]+ // A replacement that keeps what follows the secret does not end in a marker.+ if g := c.adj.shapes[shape].trailing; g > 0 && item[2*g] < item[2*g+1] ||+ end == len(c.t) || !isWordByte(c.t[end-1]) || !isWordByte(c.t[end]) {+ return end, true+ }+ sj, sm = c.secretAt(end)+ if sm == nil {+ return end, true+ }+ si, m, item, shape, merged, use, least = sj, sm, sm, sj, false, c.use(sj), c.least(sj, sm)+ }+}++// overCopy returns item run on to the end of a copy of an assigned value that starts inside it and+// ends past it, the item having taken the copy's first bytes; or item. The copy is removed after+// every rule has run, which finds it only whole: of `hf_…opaque-flag-value` with `opaque-flag-value`+// assigned, the token's pattern takes `opaque`, and `-flag-value` was left. Such a copy starts+// less than the longest copy's length before the item's end. What it reads, places, copies looked+// at and bytes compared, comes out of the chain's copyBudget, linear in the text; once that is+// spent, items are left as they are.+func (c *redactChain) overCopy(item []int) []int {+ if c.copies == nil || c.copyBudget <= 0 {+ return item+ }+ t, work := c.t, 0+ for x := max(item[0]+1, item[1]-c.copies.longest+1); x < item[1] && x+redactCopyMin <= len(t) && work < c.copyBudget; x++ {+ work+++ for _, v := range c.copies.byFirst[t[x:x+redactCopyMin]] {+ work+++ if x+len(v) <= item[1] {+ continue+ }+ work += len(v)+ if strings.HasPrefix(t[x:], v) {+ item = slices.Clone(item)+ item[1] = x + len(v)+ c.hid = true+ }+ }+ }+ c.copyBudget -= work+ redactWorked(work)+ return item+}++// slashedAWSSecret reports whether m, a match of the pattern expr, is one of the rule for an AWS+// key id and its secret whose secret holds a `/`, where the URL rule, which runs before the shape+// rules, stopped a password: once the secret is replaced, the password can run on to its `@`.+func slashedAWSSecret(expr, t string, m []int) bool {+ return expr == redactAWSPair && strings.IndexByte(t[m[5]:m[6]], '/') >= 0+}++// least returns the first place m, a match of shape si, can be cut and still be a whole match: as+// far into its open-ended tail as the tail's least length. A pattern without one cannot be cut.+func (c *redactChain) least(si int, m []int) int {+ s := &c.adj.shapes[si]+ if s.tail == nil {+ return len(c.t) + 1+ }+ tail := m[1]+ for tail > m[3] && s.tail[c.t[tail-1]] {+ tail--+ }+ redactWorked(m[1] - tail)+ return tail + s.tailMin+}++// crossing looks for a secret that starts inside m, a match of shape si, and ends past it: the+// match ran on into the next secret and stopped at a byte that secret holds, or the secret+// started inside a match of fixed length. It returns where the leftmost such secret starts, its+// shape and its match; or -1.+func (c *redactChain) crossing(si int, m []int) (int, int, []int) {+ t, start, end := c.t, m[0], m[1]+ if end >= len(t) {+ return -1, 0, nil+ }+ stop := t[end]+ best, bestShape := end, -1+ var bestMatch []int+ bestOpen := false+ for sj := range c.adj.shapes {+ s := &c.adj.shapes[sj]+ // run is where the run of the class that could carry s across end starts.+ run := end+ var class *[256]bool+ switch {+ case s.jwt && (jwtSegment[stop] || stop == '.'):+ class = &jwtSegment+ case s.tail != nil && s.tail[stop]:+ class = s.tail+ }+ if class != nil {+ for run > start+1 && class[t[run-1]] {+ run--+ }+ redactWorked(end - run)+ }+ lo := run+ switch {+ case s.jwt:+ if class == nil {+ continue+ }+ case s.tail == nil:+ lo = end - s.bound+ default:+ lo -= s.head+ }+ lo = max(lo, start+1)+ for i := hitsFrom(c.hits, lo+s.offset); i < len(c.hits); i++ {+ h := c.hits[i]+ q := h.start - s.offset+ if q >= best {+ break+ }+ if s.needles&(1<<h.id) == 0 || q < lo {+ continue+ }+ if s.jwt {+ // The leftmost `eyJ` in the run: a later one has a shorter first segment and the+ // same rest. Its first segment runs on from end.+ if jwt := c.jwtEnd(q, end); jwt > end {+ best, bestShape, bestMatch, bestOpen = q, sj, nil, false+ }+ break+ }+ sm, open := c.shapeAt(sj, q)+ if sm == nil {+ continue+ }+ // An open match whose tail runs on unbroken to end crosses it too.+ if sm[1] > end || open && class != nil && run <= sm[1] {+ best, bestShape, bestMatch, bestOpen = q, sj, sm, open+ break+ }+ }+ }+ if bestShape < 0 {+ return -1, 0, nil+ }+ if bestMatch == nil {+ bestMatch, bestOpen = c.shapeAt(bestShape, best)+ }+ return best, bestShape, c.whole(bestShape, best, bestMatch, bestOpen)+}++// jwtEnd returns where the JWT that starts at q ends, or -1. Its first segment holds only segment+// bytes from q+3 to from.+func (c *redactChain) jwtEnd(q, from int) int {+ first := c.segmentEnd(from)+ if first-(q+3) < 8 {+ return -1+ }+ if c.restAt == first {+ return c.restEnd+ }+ t, end := c.t, -1+ if strings.HasPrefix(t[first:], ".eyJ") {+ second := segmentEnd(t, first+4)+ if second-(first+4) >= 8 && second < len(t) && t[second] == '.' {+ if third := segmentEnd(t, second+1); third-(second+1) >= 8 {+ end = third+ }+ }+ }+ c.restAt, c.restEnd = first, end+ return end+}++// segmentEnd returns the end of the run of JWT segment bytes from from.+func (c *redactChain) segmentEnd(from int) int {+ if c.segFrom <= from && from <= c.segEnd {+ return c.segEnd+ }+ c.segFrom, c.segEnd = from, segmentEnd(c.t, from)+ return c.segEnd+}++func segmentEnd(t string, from int) int {+ end := from+ for end < len(t) && jwtSegment[t[end]] {+ end+++ }+ redactWorked(end - from)+ return end+}++// markerBefore returns the end of the first marker at or after from that a word byte follows, or+// -1.+func markerBeforeWord(t string, from int) int {+ for from < len(t) {+ i := strings.Index(t[from:], redactedValue)+ if i < 0 {+ return -1+ }+ p := from + i + len(redactedValue)+ if p < len(t) && isWordByte(t[p]) {+ return p+ }+ from = p+ }+ return -1+}++// redactAfterMarkers redacts what starts right after a replacement, which starts a word only in+// the redacted text: a URL's password, a Bearer token or a curl password right after a key, and+// any secret right after a replaced copy of an assigned value. Where a replacement ends right+// before a word byte, the followers and then the shape rules, in rule order, are tried there; the+// first that matches is applied, and a shape rule's chain after it. Each place is tried once, left+// to right, so a chain that alternates between them is redacted in one pass. The curl rules,+// which read a whole command line and are applied to it until it stops changing, are then+// applied to the line from the first such place where a curl command starts.+//+// It also reports whether it replaced an AWS secret that holds a `/` (slashedAWSSecret).+func (e *redactEngine) redactAfterMarkers(t string) (string, bool) {+ p := markerBeforeWord(t, 0)+ if p < 0 {+ return t, false+ }+ hits, _ := e.scanner.scan(t, nil)+ c := newRedactChain(e, t, hits)+ var out []byte+ last := 0+ emit := func(m []int, use redactUse) bool {+ out = append(out, t[last:m[0]]...)+ out = use.pattern.ExpandString(out, use.rule.replacement, t, m)+ last = m[1]+ c.slashed = c.slashed || slashedAWSSecret(use.rule.expr, t, m)+ return true+ }+ for ; p >= 0; p = markerBeforeWord(t, max(p, last)) {+ for p >= last && p < len(t) {+ if m, use := c.followerAt(p); m != nil {+ // The shape rules run after the followers, also over what a follower keeps of+ // its match, ${1}: a URL whose scheme starts with a key.+ replacement, end := use.rule.replacing(t, m)+ var items [][]int+ var uses []redactUse+ if si, sm := c.secretAt(p); sm != nil && strings.HasPrefix(replacement, "${1}") {+ if chained, _ := c.run(si, sm, c.use(si), func(m []int, use redactUse) bool {+ items, uses = append(items, m), append(uses, use)+ return true+ }); chained > m[3] {+ items = nil+ }+ }+ for i := range items {+ emit(items[i], uses[i])+ }+ if items != nil {+ out = append(out, t[last:m[3]]...)+ replacement = strings.TrimPrefix(replacement, "${1}")+ } else {+ out = append(out, t[last:m[0]]...)+ }+ out = use.pattern.ExpandString(out, replacement, t, m)+ last = end+ break+ }+ si, m := c.secretAt(p)+ if m == nil {+ break+ }+ end, _ := c.run(si, m, c.use(si), emit)+ // The chain checked for a shape rule at its end; a follower may start there.+ if end == len(t) || !isWordByte(t[end]) || len(out) == 0 || out[len(out)-1] != ']' {+ break+ }+ p = end+ }+ }+ if out != nil {+ t = string(append(out, t[last:]...))+ }+ return e.redactLinesAfterMarkers(t), c.slashed+}++// commandLineEnd returns the end of what a curl rule can match from p: its line; the next one,+// since the blank before a flag (`\s`) can be the newline, so the flag can start that line; and,+// since the blanks after a flag (`\s*`) can run over newlines, the next line that is not blank.+func commandLineEnd(t string, p int) int {+ i := strings.IndexByte(t[p:], '\n')+ if i < 0 {+ return len(t)+ }+ next := p + i + 1+ j := strings.IndexByte(t[next:], '\n')+ if j < 0 {+ return len(t)+ }+ next += j+ for next < len(t) && isRegexpSpace(t[next]) {+ next+++ }+ if k := strings.IndexByte(t[next:], '\n'); k >= 0 {+ return next + k+ }+ return len(t)+}++// redactLinesAfterMarkers applies the followLine rules, each as redaction applies it, to every+// line from the first place right after a marker where one of their needles starts.+func (e *redactEngine) redactLinesAfterMarkers(t string) string {+ adj := e.adjacency()+ var out []byte+ last := 0+ for p := markerBeforeWord(t, 0); p >= 0; p = markerBeforeWord(t, max(p, last)) {+ if p < last || !adj.startsLine(t, p) {+ continue+ }+ // A segment whose last line holds another curl command runs on to that command's value.+ end := commandLineEnd(t, p)+ for end < len(t) {+ ls := strings.LastIndexByte(t[:end], '\n') + 1+ if ls <= p || !strings.Contains(strings.ToLower(t[ls:end]), "curl") {+ break+ }+ end = commandLineEnd(t, ls)+ }+ line := t[p:end]+ for _, f := range adj.followers {+ if f.kind != followLine {+ continue+ }+ text := redactGate{value: line, enabled: true, engine: e}+ if rule := redactionRules[f.rule]; text.holds(rule.needles) {+ line = rule.apply(&text, &e.rules[f.rule])+ }+ }+ out = append(append(out, t[last:p]...), line...)+ last = end+ }+ if out == nil {+ return t+ }+ return string(append(out, t[last:]...))+}++// startsLine reports whether one of the followLine rules' needles starts at p.+func (a *redactAdjacency) startsLine(t string, p int) bool {+ for _, f := range a.followers {+ if f.kind != followLine {+ continue+ }+ for _, needle := range redactionRules[f.rule].needles.needles {+ if len(t)-p >= len(needle) && strings.EqualFold(t[p:p+len(needle)], needle) {+ return true+ }+ }+ }+ return false+}++// followerStarts reports whether a follower matches at x: a URL, a Bearer token or a curl command.+func (c *redactChain) followerStarts(x int) bool {+ if m, _ := c.followerAt(x); m != nil {+ return true+ }+ return c.lineFollowerAt(x) != nil+}++// followerCrossing returns where a follower starts whose first word m took, the match running on+// over a URL's scheme, `Bearer` or `curl` and stopping at the byte after it, and where the+// follower's match ends; or -1.+func (c *redactChain) followerCrossing(m []int) (int, int) {+ t, start, e := c.t, m[0], m[1]+ if e >= len(t) || e-1 <= start || c.followerStarts(e) {+ return -1, 0+ }+ // The whole scheme: where the match stopped inside it, at `.`, `-` or `+`, the URL rule found a+ // scheme of its own after that byte.+ if isASCIILetter(t[e-1]) && strings.HasPrefix(t[e:], "://") {+ if fm, _ := c.followerAt(e - 1); fm != nil && fm[1] > e {+ return e - 1, fm[1]+ }+ }+ for _, f := range c.adj.followers {+ if f.kind == followScheme {+ continue+ }+ for _, needle := range redactionRules[f.rule].needles.needles {+ q := e - len(needle)+ if q <= start || !c.hasNeedleAt(f.needles, q) {+ continue+ }+ if f.kind == followLine {+ if fm := c.lineFollowerAt(q); fm != nil {+ return q, fm[1]+ }+ } else if fm, _ := c.followerAt(q); fm != nil && fm[1] > e {+ return q, fm[1]+ }+ }+ }+ return -1, 0+}++// schemeEnd returns the end of the run of URL scheme bytes from p: exactly what the URL pattern's+// case-insensitive `[a-z0-9+.-]*` takes, the ASCII class and the two other characters that fold+// into it, `K` (U+212A, k) and `ſ` (U+017F, s). schemeFollowerAt shares one answer across the run,+// so a byte the pattern stops at must end it.+func (c *redactChain) schemeEnd(p int) int {+ if c.schemeFrom <= p && p <= c.schemeTo {+ return c.schemeTo+ }+ t, end := c.t, p+ for end < len(t) {+ switch {+ case schemeClass[t[end]]:+ end+++ continue+ case strings.HasPrefix(t[end:], "\u212a"):+ end += len("\u212a")+ continue+ case strings.HasPrefix(t[end:], "\u017f"):+ end += len("\u017f")+ continue+ }+ break+ }+ redactWorked(end - p)+ c.schemeFrom, c.schemeTo = p, end+ return end+}++// lineFollowerAt returns the match of the first followLine rule that matches at q, or nil (only+// its start and end: the callers read no group). It hands the pattern slices that double until+// the match ends inside one or one reaches the line's end. A place on a line where the needle+// ends a word and the rule finds nothing means no later place on the line finds anything (from+// the earlier `curl`, the pattern's lazy run takes the bytes up to the later one); and where it+// finds a match, a later place up to the end of that lazy run finds one that ends at the same+// byte, the flags between having failed from the earlier place too. So each line is read a+// bounded number of times however many places on it are asked about.+func (c *redactChain) lineFollowerAt(q int) []int {+ t := c.t+ if c.lineFailed == nil {+ c.lineFailed = make([][2]int, len(c.adj.followers))+ c.lineFound = make([][3]int, len(c.adj.followers))+ for k := range c.lineFailed {+ c.lineFailed[k] = [2]int{-1, -1}+ c.lineFound[k] = [3]int{-1, -1, -1}+ }+ }+ for k, f := range c.adj.followers {+ if f.kind != followLine || !c.hasNeedleAt(f.needles, q) {+ continue+ }+ // The pattern opens with the needle and \b: where a word byte follows the needle it finds+ // nothing, which says nothing of later places.+ word := len(redactionRules[f.rule].needles.needles[0])+ if q+word < len(t) && isWordByte(t[q+word]) {+ continue+ }+ failed, found := &c.lineFailed[k], &c.lineFound[k]+ if failed[0] <= q && q <= failed[1] {+ continue+ }+ if found[0] <= q && q <= found[1] {+ return []int{q, found[2]}+ }+ if q < c.lineFrom || q > c.lineFirst {+ c.lineFrom, c.lineTo = q, commandLineEnd(t, q)+ c.lineFirst = c.lineTo+ if i := strings.IndexByte(t[q:c.lineTo], '\n'); i >= 0 {+ c.lineFirst = q + i+ }+ redactWorked(c.lineTo - q)+ }+ for n := 256; ; n *= 2 {+ end := min(c.lineTo, q+n)+ redactWorked(end - q)+ m := f.lead.FindStringSubmatchIndex(t[q:end])+ if m != nil && q+m[1] < end || end == c.lineTo {+ if m != nil {+ lead := q + m[2*f.lead.SubexpIndex("lead")+1]+ *found = [3]int{q, lead - word, q + m[1]}+ return []int{q, q + m[1]}+ }+ break+ }+ }+ // The places are not asked in order: each shape's rescue asks from its own start.+ if failed[1] == c.lineFirst && failed[0] >= 0 {+ failed[0] = min(failed[0], q)+ } else {+ *failed = [2]int{q, c.lineFirst}+ }+ }+ return nil+}++// followerAt returns the match of the first followRun or followScheme rule, in rule order, that+// matches at p and keeps no value of it, and how to expand it. Each answer is kept for the place+// asked last, and a URL's for every letter of its scheme's run: the places of a run share it.+func (c *redactChain) followerAt(p int) ([]int, redactUse) {+ t := c.t+ if c.asked == nil {+ c.asked = make([]redactAsked, len(c.adj.followers))+ for k := range c.asked {+ c.asked[k].at = -1+ }+ }+ for k := range c.adj.followers {+ f := &c.adj.followers[k]+ var m []int+ switch {+ case f.kind == followLine:+ continue+ case c.asked[k].at == p:+ m = c.asked[k].m+ case f.kind == followScheme:+ if p >= len(t) || !isASCIILetter(t[p]) {+ continue+ }+ m = c.schemeFollowerAt(f, p)+ c.asked[k] = redactAsked{p, m}+ default:+ if !c.hasNeedleAt(f.needles, p) {+ continue+ }+ // The word, the blanks after it, and a run of the charset without blanks. A run+ // follower's pattern needs a blank after its word (adjacency checks it).+ end := p + len(redactionRules[f.rule].needles.needles[0])+ if end >= len(t) || t[end] != ' ' && t[end] != '\t' {+ c.asked[k] = redactAsked{p, nil}+ continue+ }+ for end < len(t) && (t[end] == ' ' || t[end] == '\t') {+ end+++ }+ m = c.find(f.rule, p, min(len(t), c.runEnd(f, end)+1))+ c.asked[k] = redactAsked{p, m}+ }+ if m == nil {+ continue+ }+ rule, re := &redactionRules[f.rule], c.pattern(f.rule)+ if rule.keep != nil {+ index := re.SubexpIndex("value")+ if index < 0 {+ index = 2+ }+ if g := 2 * index; m[g] >= 0 && rule.keep(t[m[g]:m[g+1]]) {+ continue+ }+ }+ return m, redactUse{re, rule}+ }+ return nil, redactUse{}+}++// schemeFollowerAt returns the URL follower's match at p, a letter, or nil. From any letter of a+// run of scheme bytes the pattern reads the run to its `://` and the same text after it, so the+// match from the first letter asked serves every other: only its start moves.+func (c *redactChain) schemeFollowerAt(f *redactFollower, p int) []int {+ t := c.t+ r := c.schemeEnd(p)+ if !strings.HasPrefix(t[r:], "://") {+ return nil+ }+ if c.schemeFrom <= c.schemeAt && c.schemeAt <= r {+ if c.schemeMatch == nil {+ return nil+ }+ m := slices.Clone(c.schemeMatch)+ m[0], m[2] = p, p+ return m+ }+ end := r + 3+ for end < len(t) && t[end] != '/' && !isRegexpSpace(t[end]) {+ end+++ }+ redactWorked(end - r)+ c.schemeAt, c.schemeMatch = p, c.find(f.rule, p, end)+ return c.schemeMatch+}++// runEnd returns the end of the run of follower f's charset, without blanks, from s.+func (c *redactChain) runEnd(f *redactFollower, s int) int {+ if c.runFrom <= s && s <= c.runTo {+ return c.runTo+ }+ end := s+ for end < len(c.t) && f.charset[c.t[end]] && c.t[end] != ' ' && c.t[end] != '\t' {+ end+++ }+ redactWorked(end - s)+ c.runFrom, c.runTo = s, end+ return end+}diff --git a/pkg/asymptoteobserve/redact_adjacent_test.go b/pkg/asymptoteobserve/redact_adjacent_test.gonew file mode 100644index 00000000..46d745f6--- /dev/null+++ b/pkg/asymptoteobserve/redact_adjacent_test.go@@ -0,0 +1,243 @@+package asymptoteobserve++import (+ "math/rand/v2"+ "regexp"+ "strings"+ "testing"+)++const (+ adjacentGitHub = "ghp_0123456789abcdefghijklmnopqrstuvwxyz"+ adjacentGoogle = "AIzaSyA-abcdefghijklmnopqrstuvwxyz01234"+ adjacentNPM = "npm_abcdefghijklmnopqrstuvwxyz0123456789"+ adjacentJWT = "eyJhbGciOiJIUzI1NiJ9.eyJzdWIiOiIxMjM0NTY3ODkwIn0.abcdefghijk"+)++// adjacentCases are secrets written right after another secret, or right after text a pass+// replaces, each in the shape it is written in. A pass of the rules used to leave the second+// secret, or part of it.+var adjacentCases = []struct{ name, input, want string }{+ {"two Google keys", adjacentGoogle + adjacentGoogle, "AIza[REDACTED]AIza[REDACTED]"},+ {"two npm tokens", adjacentNPM + adjacentNPM, "npm_[REDACTED]npm_[REDACTED]"},+ // The first token's pattern takes the second's `ghp` and stops at its `_`.+ {"two GitHub tokens", adjacentGitHub + adjacentGitHub, "ghp_[REDACTED]ghp_[REDACTED]"},+ {"two short GitHub tokens", "ghp_abcdefghijklmnopqrstuvwxyz0123ghp_abcdefghijklmnopqrstuvwxyz0123", "ghp_[REDACTED]ghp_[REDACTED]"},+ {"two Hugging Face tokens", "hf_abcdefghijklmnopqrstuvwxyz01234567hf_abcdefghijklmnopqrstuvwxyz01234567", "hf_[REDACTED]hf_[REDACTED]"},+ {"two Stripe keys", "sk_live_abcdefghijklmnopsk_live_abcdefghijklmnop", "sk_live_[REDACTED]sk_live_[REDACTED]"},+ {"a Slack token into a GitHub token", "xoxb-1234567890-abc" + adjacentGitHub, "xoxb-[REDACTED]ghp_[REDACTED]"},+ {"a GitHub token into a Google key", adjacentGitHub + adjacentGoogle, "ghp_[REDACTED]AIza[REDACTED]"},+ {"a GitHub token into a JWT", adjacentGitHub + adjacentJWT, "ghp_[REDACTED]eyJ[REDACTED]"},+ {"a JWT into a JWT", adjacentJWT + adjacentJWT, "eyJ[REDACTED]eyJ[REDACTED]"},+ {"an API key into an OAuth token", "sk-abcdefghijklmnopqrst-ya29.abcdefghijklmnopqrstuvwxyz", "sk-[REDACTED]ya29.[REDACTED]"},+ // The key's pattern takes the JWT's first segment and stops at its dot, too short a key+ // without it: the two are redacted as one.+ {"an API key into a JWT it cannot be cut before", "sk-" + adjacentJWT, "sk-[REDACTED]"},+ {"curl cookies after an npm token", adjacentNPM + "curl -b 'a=1' -b 'b=2'", "npm_[REDACTED]curl -b '[REDACTED]' -b '[REDACTED]'"},+ // A token's pattern runs on over a URL's scheme, `Bearer` or `curl`: the token is cut before+ // it, and the follower is redacted.+ {"a URL password after a GitHub token", adjacentGitHub + "postgres://user:pw@host/db", "ghp_[REDACTED]s://user:[REDACTED]@host/db"},+ {"a URL password after a JWT", adjacentJWT + "postgres://user:pw@host/db", "eyJ[REDACTED]s://user:[REDACTED]@host/db"},+ {"a Bearer token after a Hugging Face token", "hf_abcdefghijklmnopqrstuvwxyz01234567Bearer abcdefghijk", "hf_[REDACTED]Bearer [REDACTED]"},+ {"a Bearer token after a Slack token", "xoxb-1234567890-abcBearer abcdefghijk", "xoxb-[REDACTED]Bearer [REDACTED]"},+ {"a curl password after an API key", "sk-abcdefghijklmnopqrstuvwxyz0123curl -u admin:hunter2 https://x", "sk-[REDACTED]curl -u admin:[REDACTED] https://x"},+ {"a curl password after a GitLab token", "glpat-abcdefghijklmnopqrstuvcurl -u admin:hunter2", "glpat-[REDACTED]curl -u admin:[REDACTED]"},+ {"a URL password after an AWS key id", "AKIAIOSFODNN7EXAMPLEpostgres://user:pw@host/db", "AKIA[REDACTED]postgres://user:[REDACTED]@host/db"},+ {"a Bearer token after an AWS key id", "AKIAIOSFODNN7EXAMPLEBearer abcdefghijk", "AKIA[REDACTED]Bearer [REDACTED]"},+ {"a curl password after an AWS key id", "AKIAIOSFODNN7EXAMPLEcurl -u admin:hunter2", "AKIA[REDACTED]curl -u admin:[REDACTED]"},+ // A URL's scheme run ends at a byte its pattern stops at (`é`), and goes on over one that+ // folds into it (`K`, Kelvin): what is asked before the first is not the answer after it.+ {"a URL password after an AWS key id, past a non-ASCII letter", "AKIAIOSFODNN7EXAMPLEa\u00e9AKIAIOSFODNN7EXAMPLEpostgres://u:hunter2@ h", "AKIAIOSFODNN7EXAMPLEa\u00e9AKIA[REDACTED]postgres://u:[REDACTED]@ h"},+ {"a URL password after AWS key ids, over a Kelvin sign", "AKIAIOSFODNN7EXAMPLEa\u212aAKIAIOSFODNN7EXAMPLEpostgres://u:hunter2@ h", "AKIA[REDACTED]a\u212aAKIA[REDACTED]postgres://u:[REDACTED]@ h"},+ // When the key id rule asks whether a curl command follows the key id, the rule for a key id+ // and its secret has already asked about the line from its later `curl`, which has none.+ {"a curl password after a Google key and an AWS key id", adjacentGoogle + "AKIAIOSFODNN7EXAMPLEcurl -u a:curl -u a:", "AIza[REDACTED]AKIA[REDACTED]curl -u a:[REDACTED] -u a:"},+ // A `curl` a word byte follows (`curl_x`) is no command, and says nothing of a later one.+ {"a curl password after a token, past a glued curl_", adjacentGitHub + "curl_x " + adjacentGitHub + "curl -u admin:hunter2", "ghp_[REDACTED]_x ghp_[REDACTED]curl -u admin:[REDACTED]"},+ // The flag starts the line after the command, and its value is on the line after that.+ {"a curl flag alone on the line after a key", adjacentNPM + "curl x\n-u\na:hunter2", "npm_[REDACTED]curl x\n-u\na:[REDACTED]"},+ {"a Google key, then a GitHub token", adjacentGoogle + adjacentGitHub, "AIza[REDACTED]ghp_[REDACTED]"},+ {"npm tokens and Google keys in turn", adjacentNPM + adjacentGoogle + adjacentNPM + adjacentGoogle, "npm_[REDACTED]AIza[REDACTED]npm_[REDACTED]AIza[REDACTED]"},+ // The key id's pattern ends in \b, which a token right after it takes away.+ {"an AWS key id, then a GitHub token", "AKIAIOSFODNN7EXAMPLE" + adjacentGitHub, "AKIA[REDACTED]ghp_[REDACTED]"},+ {"an AWS key id, then a Google key", "AKIAIOSFODNN7EXAMPLE" + adjacentGoogle, "AKIA[REDACTED]AIza[REDACTED]"},+ {"an AWS secret, then a GitHub token", "AKIAIOSFODNN7EXAMPLE,wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY" + adjacentGitHub, "AKIA[REDACTED],[REDACTED]ghp_[REDACTED]"},+ {"a GitHub token, then an AWS key id and secret", adjacentGitHub + "AKIAIOSFODNN7EXAMPLE,wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY\n", "ghp_[REDACTED]AKIA[REDACTED],[REDACTED]\n"},+ // The rules for a URL's password, a Bearer token and a curl password run before the shape+ // rules, while the key before them is still there.+ {"a URL password after an npm token", adjacentNPM + "postgres://user:pw@host/db", "npm_[REDACTED]postgres://user:[REDACTED]@host/db"},+ {"a Bearer token after an npm token", adjacentNPM + "Bearer abcdefghijk", "npm_[REDACTED]Bearer [REDACTED]"},+ {"a curl password after an npm token", adjacentNPM + "curl -u admin:hunter2 https://x", "npm_[REDACTED]curl -u admin:[REDACTED] https://x"},+ {"curl passwords after npm tokens", adjacentNPM + "curl -u a:b1 " + adjacentNPM + "curl -u c:d2", "npm_[REDACTED]curl -u a:[REDACTED] npm_[REDACTED]curl -u c:[REDACTED]"},+ {"a URL password after a Google key, then a token", adjacentGoogle + "https://x:pw@h/" + adjacentNPM, "AIza[REDACTED]https://x:[REDACTED]@h/npm_[REDACTED]"},+ // A bare copy of an assigned value is replaced after every rule has run.+ {"a GitHub token after a copy of an assigned value", "token=abcdefgh12345678 abcdefgh12345678" + adjacentGitHub, "token=[REDACTED] [REDACTED]ghp_[REDACTED]"},+ {"a URL password after a copy of an assigned value", "token=abcdefgh12345678 abcdefgh12345678redis://:pw@h", "token=[REDACTED] [REDACTED]redis://:[REDACTED]@h"},+ // The quoted-label rule keeps `null`, the capture of the value runs on past it, and the rule+ // for `x-access-token:` then replaces the token inside what was captured.+ {"a value changed after it was captured", `"secret": null}bcn_code_--https://x-access-token:[email protected]/o/r`, `"secret": [REDACTED]`},+ // A token's pattern takes the first bytes of a copy of an assigned value written right after+ // it, and the copy, removed only whole, was left from there on: the item runs on to its end.+ {"a copy of an assigned value after a token", "hf_abcdefghijklmnopqrstuvwxyz01234567opaque-flag-value tool login --token=opaque-flag-value", "hf_[REDACTED] tool login --token=[REDACTED]"},+}++func TestRedactStringRedactsSecretsWrittenRightAfterAnother(t *testing.T) {+ for _, tc := range adjacentCases {+ if got := RedactString(tc.input); got != tc.want {+ t.Errorf("%s: RedactString(%q) = %q, want %q", tc.name, tc.input, got, tc.want)+ }+ if got := redactString(tc.input, false); got != tc.want {+ t.Errorf("%s: with every rule's whole pattern, %q, want %q", tc.name, got, tc.want)+ }+ withBudget(t, 0, -1, func() {+ if got := RedactString(tc.input); got != tc.want {+ t.Errorf("%s: from safe starts, %q, want %q", tc.name, got, tc.want)+ }+ })+ }+}++// Text where a secret only ends next to other text, or is followed by something shaped like one+// that is not, redacts as before.+func TestRedactStringKeepsWhatFollowsASecretWhenItIsNone(t *testing.T) {+ for input, want := range map[string]string{+ adjacentGoogle + "xyz": "AIza[REDACTED]xyz",+ adjacentNPM + "postgres": "npm_[REDACTED]postgres",+ adjacentNPM + "ghp_short": "npm_[REDACTED]ghp_short",+ "AKIAIOSFODNN7EXAMPLExoxz": "AKIAIOSFODNN7EXAMPLExoxz",+ "AKIAIOSFODNN7EXAMPLEghp_short": "AKIAIOSFODNN7EXAMPLEghp_short",+ adjacentGitHub + "_suffix": "ghp_[REDACTED]_suffix",+ "x" + adjacentGitHub: "x" + adjacentGitHub,+ "token=abcdefgh12345678 abcdefgh12345678xyz": "token=[REDACTED] [REDACTED]xyz",+ "[REDACTED]Bearer": "[REDACTED]Bearer",+ // A URL whose user name only ends in a word a label holds keeps its host and path; one+ // that is a label (api_key) is redacted as far as the label rules take it.+ "https://deploykey:pw@host/repo": "https://deploykey:[REDACTED]@host/repo",+ "https://key:pw@host/repo": "https://key:[REDACTED]@host/repo",+ "https://keys:pw@host/repo": "https://keys:[REDACTED]@host/repo",+ } {+ if got := RedactString(input); got != want {+ t.Errorf("RedactString(%q) = %q, want %q", input, got, want)+ }+ }+}++// A URL's password that runs on as the label rules read it (urlRunsOn) reads an AWS key id and+// its secret as the shape rule for the pair leaves them: the run that takes the key id takes the+// secret too, and the pair does not stop the password where the secret's `/` would.+func TestRedactStringRunsAURLPasswordOnOverAnAWSKeyAndSecret(t *testing.T) {+ const pair = "AKIAIOSFODNN7EXAMPLE,wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY"+ for input, want := range map[string]string{+ "A://token:,@0,@0,@" + pair + "*S-1": "A://token:[REDACTED]",+ "A://token:,@AKIAIOSFODNN7EXAMPLE:,wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY x": "A://token:[REDACTED] x",+ "A://token:,@0," + pair + "@h": "A://token:[REDACTED]",+ "A://token:,@0,AKIAIOSFODNN7EXAMPLE wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY@h": "A://token:[REDACTED],AKIA[REDACTED] [REDACTED]@h",+ } {+ got := RedactString(input)+ if got != want {+ t.Errorf("RedactString(%q) = %q, want %q", input, got, want)+ }+ if again := RedactString(got); again != got {+ t.Errorf("RedactString(%q) = %q, and again %q", input, got, again)+ }+ }+}++// The run schemeFollowerAt shares one answer across is exactly what the URL pattern's+// case-insensitive scheme class takes, non-ASCII characters that fold into it among it.+func TestSchemeEndIsTheURLPatternsRun(t *testing.T) {+ run := regexp.MustCompile(`^(?i)[a-z0-9+.-]*`)+ pieces := []string{"a", "Z", "0", "+", ".", "-", "k", "s", "\u212a", "\u017f", "\u00e9", "\u212b", ":", "/", " ", "\xff", "\xe2\x84"}+ r := rand.New(rand.NewPCG(7, 7))+ for range 20000 {+ var b strings.Builder+ for n := r.IntN(12); n >= 0; n-- {+ b.WriteString(pieces[r.IntN(len(pieces))])+ }+ text := b.String()+ c := newRedactChain(redactEngineFor(), text, nil)+ if got, want := c.schemeEnd(0), len(run.FindString(text)); got != want {+ t.Fatalf("schemeEnd(%q) = %d, the pattern's run ends at %d", text, got, want)+ }+ }+}++// One pass leaves nothing for a second one: events are sanitized again on every read path, and a+// derived field that redacts text a writer already redacted must find nothing new.+func FuzzRedactIsIdempotent(f *testing.F) {+ for _, tc := range adjacentCases {+ f.Add(tc.input)+ }+ for _, input := range redactIdempotenceInputs {+ f.Add(input)+ }+ r := rand.New(rand.NewPCG(5, 5))+ for range 64 {+ f.Add(randomAdjacentInput(r))+ }+ f.Fuzz(func(t *testing.T, input string) {+ once := RedactString(input)+ if twice := RedactString(once); twice != once {+ t.Fatalf("RedactString is not idempotent on %q:\nonce %q\ntwice %q", input, once, twice)+ }+ if all := redactString(input, false); all != once {+ t.Fatalf("redaction of %q = %q, every rule's whole pattern gives %q", input, once, all)+ }+ })+}++// randomAdjacentInput is randomRedactInput's fragments with whole secrets among them, so they+// often meet.+func randomAdjacentInput(r *rand.Rand) string {+ var b strings.Builder+ for n := r.IntN(8); n >= 0; n-- {+ if r.IntN(3) == 0 {+ b.WriteString(adjacentSecrets[r.IntN(len(adjacentSecrets))])+ } else {+ b.WriteString(redactGateFragments[r.IntN(len(redactGateFragments))])+ }+ }+ return b.String()+}++var adjacentSecrets = []string{+ adjacentGitHub, adjacentGoogle, adjacentNPM, adjacentJWT,+ "ghs_abcdefghijklmnopqrstuvwxyz0123", "hf_abcdefghijklmnopqrstuvwxyz01234567",+ "sk-abcdefghijklmnopqrstuvwxyz0123", "sk-ant-api03-abcdefghijklmnopqrstuvwxyz", "AKIAIOSFODNN7EXAMPLE",+ "AKIAIOSFODNN7EXAMPLE,wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY", "github_pat_abcdefghijklmnopqrstuvwxyz",+ "glpat-abcdefghijklmnopqrstuv", "xoxb-1234567890-abc", "sk_live_abcdefghijklmnop", "ya29.abcdefghijklmnopqrstuvwxyz",+ "bcn_api_abcdefgh", "postgres://user:pw@host/db", "https://x-access-token:[email protected]", "token=abcdefgh12345678",+ "Bearer abcdefghijk", "curl -u admin:hunter2 ", "abcdefgh12345678",+}++// Idempotence and the gate's equivalence over many random texts where secrets meet, beyond what+// the fuzz corpus holds, every fourth also from safe starts.+func TestRedactIsIdempotentWhereSecretsMeet(t *testing.T) {+ backToBack := regexp.MustCompile(`\[REDACTED\][A-Za-z0-9_.-]{1,16}\[REDACTED\]`)+ r := rand.New(rand.NewPCG(6, 6))+ met := 0+ for i := range 10000 {+ input := randomAdjacentInput(r)+ once := RedactString(input)+ if twice := RedactString(once); twice != once {+ t.Fatalf("RedactString is not idempotent on %q:\nonce %q\ntwice %q", input, once, twice)+ }+ if all := redactString(input, false); all != once {+ t.Fatalf("redaction of %q = %q, every rule's whole pattern gives %q", input, once, all)+ }+ if i%4 == 0 {+ withBudget(t, 0, -1, func() {+ if got := RedactString(input); got != once {+ t.Fatalf("redaction of %q from safe starts = %q, want %q", input, got, once)+ }+ })+ }+ if backToBack.MatchString(once) {+ met+++ }+ }+ // The texts have to put secrets next to each other for the check to mean anything.+ if met < 600 {+ t.Fatalf("only %d texts redacted a secret right after another", met)+ }+}diff --git a/pkg/asymptoteobserve/redact_engine.go b/pkg/asymptoteobserve/redact_engine.goindex fd4e5cfe..6c835bc9 100644--- a/pkg/asymptoteobserve/redact_engine.go+++ b/pkg/asymptoteobserve/redact_engine.go@@ -35,6 +35,9 @@ import (// case-insensitive rules run their whole patterns, as they always did. redactString(value, false)// runs every rule's whole pattern on any text, and TestRedactGateNeverChangesAResult and// FuzzRedactGate hold the two to the same result.+//+// A shape rule's matches also take in the secrets written right after them (redact_adjacent.go),+// the same way whichever way the matches are found.// redactStart says where, around an occurrence of one of a rule's needles that passes the needle// set's precededBy and followedBy (and those of the sets it is within), a match of the rule's@@ -70,15 +73,20 @@ type redactSlot struct {// The assertions the whole pattern makes, which decide where it can be run on a suffix of the// text with the same result (safeStart): \b or \B, (?m)^, and ^ or \A.wordAssert, lineStart, textStart bool+ // index is the rule's index in redactionRules, or -1.+ index int}+// boundaryPrefixes are the ways a pattern opens with \b.+var boundaryPrefixes = []string{`(?i)(\b`, `(?i)\b`, `(\b`, `\b`}+// prepare compiles the rule's pattern anchored at the start of the text, without the \b it opens// with, if any: the anchored pattern cannot see the byte before its start, so the boundary is// checked against the whole text instead.func (s *redactSlot) prepare(rule redactionRule) {s.once.Do(func() {expr := rule.expr- for _, prefix := range []string{`(?i)(\b`, `(?i)\b`, `(\b`, `\b`} {+ for _, prefix := range boundaryPrefixes {if strings.HasPrefix(expr, prefix) {s.boundary = trueexpr = prefix[:len(prefix)-2] + expr[len(prefix):]@@ -157,10 +165,17 @@ type redactEngine struct {setMasks [64]uint64 // redactNeedles id -> the needles of the set, as scanner needle bitsrules []redactSlotassigned redactSlot+ // adjacent is what redact_adjacent.go knows of the rules, built the first time it is needed.+ adjacentOnce sync.Once+ adjacent *redactAdjacency}var redactEngineFor = sync.OnceValue(func() *redactEngine {e := &redactEngine{rules: make([]redactSlot, len(redactionRules))}+ for i := range e.rules {+ e.rules[i].index = i+ }+ e.assigned.index = -1ids := map[redactNeedle]int{}var needles []redactNeedlevar register func(set *redactNeedles)@@ -219,6 +234,13 @@ type redactGate struct {// known and present are bit sets over redactNeedles ids: which sets were looked for in this// version of the text, and which were found.known, present uint64+ // hidden is set when a match may hide another from this pass: a value a rule keeps, or the+ // capture takes, that is itself a label and its separator (keptLabel), inside a match that+ // neither looks into; a copy a token ran on over, as replacing a copy may (copyHidesMatch); or a+ // replaced AWS secret's `/`, where a URL's password stopped.+ hidden bool+ // copies are the assigned values whose copies are removed after every rule has run.+ copies *redactCopies}func (g *redactGate) set(value string) {@@ -325,23 +347,69 @@ func (set *redactNeedles) passes(engine *redactEngine, t string, h redactHit, wi// When the places would hand the pattern more text than the budget allows (redactAnchoredPerByte),// the whole pattern runs once from a safe start before the first of them instead. No match starts// before the first place, and from a safe start the pattern finds what it finds in the whole text.-func (rule redactionRule) matches(text *redactGate, slot *redactSlot) iter.Seq2[[]int, *regexp.Regexp] {- return func(yield func([]int, *regexp.Regexp) bool) {+//+// A shape rule's match is followed by the secrets written right after it (redactChain.run), each+// yielded with the rule it is a match of, and the next match is looked for from where they end,+// in both ways of finding it. Text with none gets exactly FindAll's matches.+func (rule redactionRule) matches(text *redactGate, slot *redactSlot) iter.Seq2[[]int, redactUse] {+ return func(yield func([]int, redactUse) bool) {t := text.value+ var chain *redactChain+ if rule.shape != shapeNone {+ text.scan()+ slot.prepare(rule)+ chain = newRedactChain(text.e(), t, text.hits)+ chain.copies = text.copies+ defer func() {+ text.hidden = text.hidden || chain.hid || chain.slashed && strings.Contains(t, "://")+ }()+ }whole := func(from int) {- if redactWork != nil {- redactWork(len(t) - from)- }re := rule.pattern()- for _, m := range re.FindAllStringSubmatchIndex(t[from:], -1) {- for i := range m {- if m[i] >= 0 {- m[i] += from+ use := redactUse{re, &rule}+ if chain == nil {+ redactWorked(len(t) - from)+ for _, m := range re.FindAllStringSubmatchIndex(t[from:], -1) {+ for i := range m {+ if m[i] >= 0 {+ m[i] += from+ }+ }+ if !yield(m, use) {+ return}}- if !yield(m, re) {+ return+ }+ // FindAll's loop, from where each chain ends. The pattern run on t[pos:] takes pos for+ // the start of a word; where t does not, the search goes on from the next byte. The+ // first match from aheadFrom on (nil: none) stays the first from any later pos up to+ // its start, so a rescued match before it does not make the search read on to it again.+ si := chain.adj.byRule[slot.index]+ var ahead []int+ aheadFrom := -1+ for pos := from; pos < len(t); {+ if aheadFrom < 0 || ahead != nil && ahead[0] < pos {+ ahead, aheadFrom = chain.search(re, pos), pos+ if ahead != nil && ahead[0] == pos && pos > from && slot.boundary && !wordBoundary(t, pos) {+ ahead = chain.search(re, pos+1)+ }+ }+ m, limit := ahead, len(t)+ if m != nil {+ limit = m[0]+ }+ if rescued := chain.rescueBefore(si, pos, limit); rescued != nil {+ m = rescued+ }+ if m == nil {return}+ end, ok := chain.run(si, m, use, yield)+ if !ok {+ return+ }+ pos = end}}if !text.enabled || rule.needles.folded && text.unfoldable() {@@ -400,23 +468,42 @@ func (rule redactionRule) matches(text *redactGate, slot *redactSlot) iter.Seq2[redactWork(len(t) - q)}m := slot.anchored.FindStringSubmatchIndex(t[q:])+ for i := range m {+ if m[i] >= 0 {+ m[i] += q+ }+ }+ if m == nil && chain != nil {+ m = chain.rescue(chain.adj.byRule[slot.index], q)+ }if m == nil {failed = qcontinue}- for i := range m {- if m[i] >= 0 {- m[i] += q+ use := redactUse{slot.anchored, &rule}+ if chain != nil {+ end, ok := chain.run(chain.adj.byRule[slot.index], m, use, yield)+ if !ok {+ return}+ pos = end+ continue}pos = m[1]- if !yield(m, slot.anchored) {+ if !yield(m, use) {return}}}}+// redactUse is what matches yields with a match: the pattern to expand it with and the rule it is+// a match of, which for a secret written right after a shape rule's match is another shape rule.+type redactUse struct {+ pattern *regexp.Regexp+ rule *redactionRule+}+// startFor returns the leftmost place at or after pos where a match holding h can begin, or -1.func (rule redactionRule) startFor(t string, h redactHit, pos int, boundary bool) int {switch rule.start {diff --git a/pkg/asymptoteobserve/redact_engine_test.go b/pkg/asymptoteobserve/redact_engine_test.goindex 7a0437ed..afeef1db 100644--- a/pkg/asymptoteobserve/redact_engine_test.go+++ b/pkg/asymptoteobserve/redact_engine_test.go@@ -1,6 +1,7 @@package asymptoteobserveimport (+ "fmt"mrand "math/rand""math/rand/v2""slices"@@ -188,17 +189,87 @@ func TestRedactionWorkIsLinear(t *testing.T) {"authorization headers": "authorization: bearer ,","cookie headers": "\ncookie: ","assignments": "token=abcdefgh12345678 ",+ // Secrets written right after another one (redact_adjacent.go): chains of keys of fixed+ // length, of tokens that run on into the next, of key ids that end only where the next+ // key starts, of JWT first segments that never complete, and URLs, Bearer tokens, curl+ // commands and replaced copies right after a key.+ "Google keys back to back": "AIzaSyA-abcdefghijklmnopqrstuvwxyz01234",+ "GitHub tokens back to back": "ghp_0123456789abcdefghijklmnopqrstuvwxyz",+ "AWS key ids back to back": "AKIAIOSFODNN7EXAMPLE",+ "npm tokens and Google keys": "npm_abcdefghijklmnopqrstuvwxyz0123456789AIzaSyA-abcdefghijklmnopqrstuvwxyz01234",+ "tokens into JWT prefixes": "ghp_0123456789abcdefghijklmnopqrstuvwxyzeyJabc-",+ "API keys into JWT prefixes": "sk-eyJabcdefghij",+ "URL schemes after keys": "npm_abcdefghijklmnopqrstuvwxyz0123456789a://x:",+ "Bearer tokens after keys": "npm_abcdefghijklmnopqrstuvwxyz0123456789Bearer ",+ "curl commands after keys": "npm_abcdefghijklmnopqrstuvwxyz0123456789curl -b x ",+ "keys after replaced copies": "token=abcdefgh12345678 abcdefgh12345678ghp_0123456789abcdefghijklmnopqrstuvwxyz ",+ "key ids into unfinished keys": "AKIAIOSFODNN7EXAMPLEAIza",+ "tokens into URL schemes": "ghp_0123456789abcdefghijklmnopqrstuvwxyzpostgres://u:p@h/",+ "tokens into Bearer tokens": "hf_abcdefghijklmnopqrstuvwxyz01234567Bearer x ",+ "keys into curl commands": "sk-abcdefghijklmnopqrstuvwxyz0123curl -u a:b ",+ "keys into curl lines": "sk-abcdefghijklmnopqrstuvwxyz0123curl x ",+ "key ids into URL schemes": "AKIAIOSFODNN7EXAMPLEa://x:y@",+ "key ids into unfinished URLs": "AKIAIOSFODNN7EXAMPLEa:/",+ // Values that are themselves labels, copies that start or end a match, and AWS secrets in+ // URL passwords, which make redaction pass over the text again; tokens into copies.+ "labels kept values hid": `token:seCret":token":0 `,+ "labels hid three passes": `0secret=token :ghp_0123456789abcdefghijklmnopqrstsecret=token :seCret':token':@h' `,+ "labels captured values hid": "token:abctoken\":`secret`:`token`:}token:--token \"token\" ",+ "copies after flags": "-pAsswd --token -token:\"--token\" ",+ "key ids before copies": "AKIAIOSFODNN7EXAMPLEAuthorization: Authorization: ",+ "tokens into copies": "token=opaque-flag-value hf_abcdefghijklmnopqrstuvwxyz01234567opaque-flag-value ",+ "AWS secrets in passwords": "https://u:a,AKIAIOSFODNN7EXAMPLE,wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY@h ",+ // A URL password's run on (urlRunsOn) over AWS key ids and their secrets.+ "URL runs over AWS keys": "A://token:,@AKIAIOSFODNN7EXAMPLE,wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY@",+ "URL passwords over AWS keys": "A://token:,@0,AKIAIOSFODNN7EXAMPLE,wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY ",+ // Tokens glued to `Bearer` with no blank after it, which run on to the end of the text.+ "tokens glued to Bearer": "hf_abcdefghijklmnopqrstuvwxyz01234567Bearer.",+ // Every other scan the chains make: open tails run on to the end, curl lines over blank+ // lines, curl lines chained after keys, and URL schemes after tokens.+ "API keys back to back": "sk-abcdefghijklmnopqrstuvwxyz0123",+ "keys glued to curl over blank lines": "sk-abcdefghijklmnopqrstuvwxyz0123curl x\n \n\n",+ "curl lines chained after keys": "npm_abcdefghijklmnopqrstuvwxyz0123456789curl x\n",+ "tokens glued to URL scheme runs": "ghp_0123456789abcdefghijklmnopqrstuvwxyza.b",}+ texts := map[string]func(n int) string{}for name, unit := range families {+ texts[name] = func(n int) string { return repeatTo(unit, n) }+ }+ texts["one long flag"] = func(n int) string { return "-" + repeatTo("token", n) + " x" }+ // Texts that are not one unit repeated: many places on one long line or run that all reach the+ // same far end (a curl flag, a URL's `://`), and copies of assigned values that a token runs+ // into.+ texts["key ids glued to curl, one flag at the end"] = func(n int) string {+ return repeatTo("AKIAIOSFODNN7EXAMPLEcurl ", n) + " -u a:b"+ }+ texts["keys glued to curl, one flag at the end"] = func(n int) string {+ return repeatTo("sk-abcdefghijklmnopqrstuvwxyz0123curl ", n) + " -u a:b"+ }+ texts["key ids in a URL scheme run"] = func(n int) string { return repeatTo("AKIA.", n) + "b://" }+ texts["key ids glued into a URL scheme"] = func(n int) string { return repeatTo("AKIAIOSFODNN7EXAMPLE", n) + "b://x:y@h" }+ texts["key ids in URL scheme runs broken by non-ASCII"] = func(n int) string {+ return repeatTo("AKIAIOSFODNN7EXAMPLEa\u00e9", n) + "b://x:y@h"+ }+ texts["key ids in a URL scheme run over Kelvin signs"] = func(n int) string {+ return repeatTo("AKIAIOSFODNN7EXAMPLEa\u212a", n) + "b://x:y@h"+ }+ texts["a long value repeating a token's tail"] = func(n int) string {+ a := strings.Repeat("a", n/3)+ return "token=" + a + " hf_" + a + " " + strings.Repeat("x ", n/6)+ }+ texts["many values sharing their first bytes"] = func(n int) string {+ var b strings.Builder+ for i := 0; b.Len() < n/2; i++ {+ fmt.Fprintf(&b, "token=aaaaaaaa%06d\n", i)+ }+ return b.String() + repeatTo("ghp_"+strings.Repeat("a", 40)+" ", n/2)+ }+ for name, text := range texts {work := func(n int) int {- text := strings.Repeat(unit, n/len(unit)+1)[:n]- if name == "one long flag" {- text = "-" + text + " x"- }total := 0redactWork = func(bytes int) { total += bytes }defer func() { redactWork = nil }()- redactString(text, true)+ redactString(text(n), true)return total}previous := work(16 << 10)@@ -212,3 +283,8 @@ func TestRedactionWorkIsLinear(t *testing.T) {}}}++// repeatTo is unit repeated to n bytes.+func repeatTo(unit string, n int) string {+ return strings.Repeat(unit, n/len(unit)+1)[:n]+}