Skip to content

The cross-harness, self-improving memory layer for AI agents. Join our community: https://discord.com/invite/zdNChS2fBu

perf(redaction): find every rule's needles in one pass and run patterns only where they can match (#1143)

kyle-elliott-asymptote722e58277758
diff --git a/pkg/asymptoteobserve/perf_ratchet_test.go b/pkg/asymptoteobserve/perf_ratchet_test.go
index daa09eb5..a716b27a 100644
--- a/pkg/asymptoteobserve/perf_ratchet_test.go
+++ b/pkg/asymptoteobserve/perf_ratchet_test.go
@@ -17,9 +17,10 @@ import (
// gain cannot be given back unnoticed.
// initAllocCeiling bounds the allocations this package's initialization makes, test files
-// included (409 when it was set). Compiling the redaction patterns at init made it 11,508: a
-// package-level regexp.MustCompile belongs behind sync.OnceValue.
-const initAllocCeiling = 500
+// included (409 when it was set at 500; 337 when the redaction rules stopped building a
+// sync.OnceValue each at init, and it was lowered to 420). Compiling the redaction patterns at
+// init made it 11,508: a package-level regexp.MustCompile belongs behind sync.OnceValue.
+const initAllocCeiling = 420
func TestPackageInitStaysCheap(t *testing.T) {
if os.Getenv("BEACON_INIT_TRACE_CHILD") != "" {
diff --git a/pkg/asymptoteobserve/privacy.go b/pkg/asymptoteobserve/privacy.go
index 7184c44e..4d05f4ce 100644
--- a/pkg/asymptoteobserve/privacy.go
+++ b/pkg/asymptoteobserve/privacy.go
@@ -72,9 +72,9 @@ const (
// argument is a constant naming a credential (the pattern spells out "password") as returning that
// credential, and these rules hold patterns, not secrets.
func quotedRules(kind quotedKind, keep func(string) bool) []redactionRule {
- prefix, words, separator, within := `\b`+redactLabel+redactSeparator, redactLabelWords, skipSeparator, redactLabelNeedles
+ prefix, words, separator, within, start := `\b`+redactLabel+redactSeparator, redactLabelWords, skipSeparator, redactLabelNeedles, startLabel
if kind == quotedFlag {
- prefix, words, separator, within = redactFlag, redactFlagWords, skipFlagSeparator, redactFlagNeedles
+ prefix, words, separator, within, start = redactFlag, redactFlagWords, skipFlagSeparator, redactFlagNeedles, startFlag
}
rules := make([]redactionRule, 0, len(quotedValues))
for _, quoted := range quotedValues {
@@ -87,7 +87,10 @@ func quotedRules(kind quotedKind, keep func(string) bool) []redactionRule {
rest, ok := separator(rest)
return ok && strings.HasPrefix(rest, open)
}
- rules = append(rules, lazyRule(`(?i)(`+prefix+quoted[0]+`)(?P<value>`+quoted[1]+`)`, "${1}"+redactedValue, keep, needles))
+ rules = append(rules, redactionRule{
+ expr: `(?i)(` + prefix + quoted[0] + `)(?P<value>` + quoted[1] + `)`, replacement: "${1}" + redactedValue,
+ keep: keep, needles: needles, start: start,
+ })
}
return rules
}
@@ -110,7 +113,10 @@ const redactFlag = `(?:^|\s)` + redactFlagName + `(?:=|\s+)`
// redactionRule is one redaction. ${1} and ${3}, when the pattern has them, are kept around the
// value.
type redactionRule struct {
- pattern func() *regexp.Regexp
+ // expr is the pattern. It is compiled the first time text gets past the needles: a hook is a
+ // process per event, and most events never need most rules, so compiling all of them up front
+ // cost every hook run a few milliseconds before it had read its input.
+ expr string
replacement string
// keep, when set, leaves a match alone if its value (the group named value) passes.
keep func(value string) bool
@@ -121,13 +127,22 @@ type redactionRule struct {
// matches from the command to the value, so a second such value on the line lies inside the
// first match and is reached only by another pass.
repeat bool
+ // start says where, around an occurrence of one of its needles, a match can begin, and offset
+ // how many bytes before the needle a startNeedle match does. See redact_engine.go.
+ start redactStart
+ offset int
+}
+
+// pattern is the rule's compiled pattern.
+func (rule redactionRule) pattern() *regexp.Regexp {
+ return compiledPattern(rule.expr)
}
// redactNeedles is a set of literals a pattern cannot match without. folded needles are lower
-// case and compared against the lower-cased text, for a case-insensitive pattern. followedBy, when
-// set, must also hold for the text right after the needle. within, when set, is a broader set the
-// text must also hold, searched first: it is what keeps a family of narrow sets to one search on
-// text that holds none of them.
+// case and found whatever the case of the text's ASCII letters, for a case-insensitive pattern.
+// followedBy, when set, must also hold for the text right after the needle, and precededBy for the
+// text before it; both read the text as it is, so they must not depend on the case of its letters.
+// within, when set, is a broader set the text must also hold.
type redactNeedles struct {
id uint
folded bool
@@ -151,15 +166,16 @@ func newRedactNeedles(folded bool, needles ...string) *redactNeedles {
return set
}
-// rule is a case-sensitive rule that cannot match text holding none of needles.
+// rule is a case-sensitive rule that cannot match text holding none of needles. A match begins
+// where its needle does.
func rule(expr, replacement string, keep func(string) bool, needles ...string) redactionRule {
- return lazyRule(expr, replacement, keep, newRedactNeedles(false, needles...))
+ return redactionRule{expr: expr, replacement: replacement, keep: keep, needles: newRedactNeedles(false, needles...)}
}
// foldedRule is a case-insensitive rule that cannot match text whose lower-cased form holds none
-// of needles, which must be lower case.
+// of needles, which must be lower case. A match begins where its needle does.
func foldedRule(expr, replacement string, keep func(string) bool, needles ...string) redactionRule {
- return lazyRule(expr, replacement, keep, newRedactNeedles(true, needles...))
+ return redactionRule{expr: expr, replacement: replacement, keep: keep, needles: newRedactNeedles(true, needles...)}
}
// curlRule is a foldedRule scoped to a curl command line. A flag curl accepts more than once (two
@@ -170,18 +186,11 @@ func curlRule(expr, replacement string) redactionRule {
return r
}
-// lazyRule compiles its pattern the first time text gets past the needles. A hook is a process
-// per event, and most events never need most rules: compiling all of them up front cost every
-// hook run a few milliseconds before it had read its input.
-func lazyRule(expr, replacement string, keep func(string) bool, needles *redactNeedles) redactionRule {
- return redactionRule{pattern: lazyPattern(expr), replacement: replacement, keep: keep, needles: needles}
-}
-
// lazyPattern compiles expr the first time it is called.
//
// The rules whose pattern names a credential (redactLabel, redactFlag) are written as struct
-// literals around it rather than through lazyRule: static analysis reads a call whose argument is a
-// constant spelling out "password" as returning that password, and would then treat every string
+// literals rather than through rule or foldedRule: static analysis reads a call whose argument is
+// a constant spelling out "password" as returning that password, and would then treat every string
// redaction returns, and every hash of one, as a credential.
func lazyPattern(expr string) func() *regexp.Regexp {
return sync.OnceValue(func() *regexp.Regexp { return regexp.MustCompile(expr) })
@@ -266,16 +275,20 @@ func startsFlag(before string) bool {
// flagStart finds the dash that opens the flag the text before a flag word ends in: the start of
// the run of letters, digits and dashes at its end. A pattern's `--?[a-z0-9-]*` can only begin
// where that maximal run does, because what it requires before the dash (whitespace, a quote) is
-// not part of the run.
+// not part of the run. The patterns are case-insensitive, so the run takes capitals too: the
+// needle scan reads the text as it is, not lower-cased.
func flagStart(before string) (int, bool) {
start := len(before)
for start > 0 {
c := before[start-1]
- if !(c >= 'a' && c <= 'z' || c >= '0' && c <= '9' || c == '-') {
+ if !(c >= 'a' && c <= 'z' || c >= 'A' && c <= 'Z' || c >= '0' && c <= '9' || c == '-') {
break
}
start--
}
+ if redactWork != nil {
+ redactWork(len(before) - start)
+ }
if start == len(before) || before[start] != '-' {
return 0, false
}
@@ -376,22 +389,22 @@ var redactionRules = concatRules([]redactionRule{
// parses where something decodes it again (string-encoded tool arguments do); null, true, false
// and nested objects or arrays are left for their own fields' rules.
{
- pattern: lazyPattern(`(?i)(\b` + redactLabel + `(?P<q>` + redactOpenQuote + `)\s*[:=]\s*)(?P<value>` + redactBare + `)`),
- replacement: "${1}${q}" + redactedValue + "${q}", keep: isLiteralOrAuthScheme, needles: redactQuotedLabelNeedles,
+ expr: `(?i)(\b` + redactLabel + `(?P<q>` + redactOpenQuote + `)\s*[:=]\s*)(?P<value>` + redactBare + `)`,
+ replacement: "${1}${q}" + redactedValue + "${q}", keep: isLiteralOrAuthScheme, needles: redactQuotedLabelNeedles, start: startLabel,
},
{
- pattern: lazyPattern(`(?i)(\b` + redactLabel + `\s*[:=]\s*)(?P<value>` + redactUnquoted + `)`),
- replacement: "${1}" + redactedValue, keep: isAuthScheme, needles: redactLabelNeedles,
+ expr: `(?i)(\b` + redactLabel + `\s*[:=]\s*)(?P<value>` + redactUnquoted + `)`,
+ replacement: "${1}" + redactedValue, keep: isAuthScheme, needles: redactLabelNeedles, start: startLabel,
},
// A command-line flag: `--password x`, `--token=x`, `--api-key x`, and quoted, the way agents
// usually write arguments: `--password "x y"`, `--token='x'`.
}, quotedRules(quotedFlag, nil), []redactionRule{
- {pattern: lazyPattern(`(?i)(` + redactFlag + `)([^\s"'-][^\s"']*)`), replacement: "${1}" + redactedValue, needles: redactFlagNeedles},
+ {expr: `(?i)(` + redactFlag + `)([^\s"'-][^\s"']*)`, replacement: "${1}" + redactedValue, needles: redactFlagNeedles, start: startFlag},
// A flag and its value as two quoted items of an argument list, the way an MCP server's args
// are written in JSON, TOML and YAML: `"args": ["--api-key", "x"]`.
{
- pattern: lazyPattern(`(?i)(["']` + redactFlagName + `["']\s*,\s*["'])(?P<value>(?:[^"'\n\\]|\\[^\n])+)`),
- replacement: "${1}" + redactedValue, keep: isRedactedValue, needles: redactListedFlagNeedles,
+ expr: `(?i)(["']` + redactFlagName + `["']\s*,\s*["'])(?P<value>(?:[^"'\n\\]|\\[^\n])+)`,
+ replacement: "${1}" + redactedValue, keep: isRedactedValue, needles: redactListedFlagNeedles, start: startListedFlag,
},
// curl's basic-auth credentials: `-u user:pass`, `-uuser:pass`, `--user user:pass` and
// `--proxy-user user:pass`. The user name and the colon stay. A bare `-u user` (curl prompts for
@@ -404,10 +417,10 @@ var redactionRules = concatRules([]redactionRule{
// A Cookie or Set-Cookie header: the whole value is a session credential, as the field-name rule
// in SanitizeMap already treats a structured `cookie` field. Only where a header or key can
// start (a line, a quote, curl's -H), so prose that says "cookie:" keeps its sentence.
- foldedRule(`(?im)((?:^\s*|["']\s*|(?:-H|--header)(?:=|\s+)["']?)(?:set-)?cookie["']?\s*:\s*["']?)([^\n"'\\]+)`, "${1}"+redactedValue, isRedactedValue, "cookie"),
+ foldedRule(`(?im)((?:^\s*|["']\s*|(?:-H|--header)(?:=|\s+)["']?)(?:set-)?cookie["']?\s*:\s*["']?)([^\n"'\\]+)`, "${1}"+redactedValue, isRedactedValue, "cookie").from(startLine),
// The password in a URL's userinfo: `postgres://user:pw@host`, `https://x-access-token:[email protected]`,
// and with no user name, the way Redis URLs carry a password: `redis://:pw@host`.
- foldedRule(`(?i)(\b[a-z][a-z0-9+.-]*://[^/\s:@"'<>]*:)([^@/\s"'<>]+)(@)`, "${1}"+redactedValue+"${3}", nil, "://"),
+ foldedRule(`(?i)(\b[a-z][a-z0-9+.-]*://[^/\s:@"'<>]*:)([^@/\s"'<>]+)(@)`, "${1}"+redactedValue+"${3}", nil, "://").from(startScheme),
// Credentials recognizable by their own shape, wherever they appear. The type prefix is public
// and stays (`ghp_[REDACTED]`, `AKIA[REDACTED]`), the way secret scanners report a finding: the
// line still says a GitHub token or an AWS key was there, which is what detection rules match on
@@ -417,16 +430,16 @@ var redactionRules = concatRules([]redactionRule{
// separators (a credentials CSV, `aws configure` output, a table, a JSON array) it is the 40
// characters after them. This runs before the key id rule below replaces the id it keys on.
rule(`(\b(?:AKIA|ASIA)[A-Z0-9]{16}\b[\s,:|"']{1,8})[A-Za-z0-9/+]{40}([^A-Za-z0-9/+=]|$)`, "${1}"+redactedValue+"${2}", nil, "AKIA", "ASIA"),
- rule(`\b(AKIA|ASIA|ABIA|ACCA)[A-Z0-9]{16}\b`, "${1}"+redactedValue, nil, "AKIA", "ASIA", "ABIA", "ACCA"), // AWS access key id
- rule(`\b(gh[pousr]_)[A-Za-z0-9]{30,}`, "${1}"+redactedValue, nil, "ghp_", "gho_", "ghu_", "ghs_", "ghr_"), // GitHub tokens
- rule(`\b(github_pat_)[A-Za-z0-9_]{22,}`, "${1}"+redactedValue, nil, "github_pat_"), // GitHub fine-grained tokens
- rule(`\b(glpat-)[A-Za-z0-9_-]{20,}`, "${1}"+redactedValue, nil, "glpat-"), // GitLab
- rule(`\b(xox[abposr]-)[A-Za-z0-9-]{10,}`, "${1}"+redactedValue, nil, "xox"), // Slack
- rule(`\b([rs]k_(?:live|test)_)[A-Za-z0-9]{16,}`, "${1}"+redactedValue, nil, "k_live_", "k_test_"), // Stripe
- rule(`\b(AIza)[0-9A-Za-z_-]{35}`, "${1}"+redactedValue, nil, "AIza"), // Google API key
- rule(`\b(ya29\.)(?:c\.)?[0-9A-Za-z_-]{20,}`, "${1}"+redactedValue, nil, "ya29."), // Google OAuth access token
- rule(`\b(npm_)[A-Za-z0-9]{36}`, "${1}"+redactedValue, nil, "npm_"), // npm
- rule(`\b(hf_)[A-Za-z0-9]{30,}`, "${1}"+redactedValue, nil, "hf_"), // Hugging Face
+ rule(`\b(AKIA|ASIA|ABIA|ACCA)[A-Z0-9]{16}\b`, "${1}"+redactedValue, nil, "AKIA", "ASIA", "ABIA", "ACCA"), // AWS access key id
+ rule(`\b(gh[pousr]_)[A-Za-z0-9]{30,}`, "${1}"+redactedValue, nil, "ghp_", "gho_", "ghu_", "ghs_", "ghr_"), // GitHub tokens
+ rule(`\b(github_pat_)[A-Za-z0-9_]{22,}`, "${1}"+redactedValue, nil, "github_pat_"), // GitHub fine-grained tokens
+ rule(`\b(glpat-)[A-Za-z0-9_-]{20,}`, "${1}"+redactedValue, nil, "glpat-"), // GitLab
+ rule(`\b(xox[abposr]-)[A-Za-z0-9-]{10,}`, "${1}"+redactedValue, nil, "xox"), // Slack
+ rule(`\b([rs]k_(?:live|test)_)[A-Za-z0-9]{16,}`, "${1}"+redactedValue, nil, "k_live_", "k_test_").before(1), // Stripe
+ rule(`\b(AIza)[0-9A-Za-z_-]{35}`, "${1}"+redactedValue, nil, "AIza"), // Google API key
+ rule(`\b(ya29\.)(?:c\.)?[0-9A-Za-z_-]{20,}`, "${1}"+redactedValue, nil, "ya29."), // Google OAuth access token
+ rule(`\b(npm_)[A-Za-z0-9]{36}`, "${1}"+redactedValue, nil, "npm_"), // npm
+ rule(`\b(hf_)[A-Za-z0-9]{30,}`, "${1}"+redactedValue, nil, "hf_"), // Hugging Face
// Beacon's own keys and tokens: API keys, device and enforcement keys, MCP tokens and their refresh
// tokens, and the CLI sign-in's access token, authorization code, client and exchange values.
rule(`\b(bcn_(?:api|device|mcprt|mcp|enfdev|cli|code|client|exchange)_)[A-Za-z0-9_-]{8,}`, "${1}"+redactedValue, nil, "bcn_"),
@@ -441,26 +454,43 @@ func concatRules(groups ...[]redactionRule) []redactionRule {
return all
}
-func (rule redactionRule) apply(value string) string {
- pattern := rule.pattern()
- if rule.keep == nil {
- out := pattern.ReplaceAllString(value, rule.replacement)
- // Each pass redacts at least one more value, and a redacted value matches no further.
- for pass := 0; rule.repeat && out != value && pass < 16; pass++ {
- value, out = out, pattern.ReplaceAllString(out, rule.replacement)
- }
- return out
- }
- index := pattern.SubexpIndex("value")
- if index < 0 {
- index = 2
+// from returns the rule with its matches beginning where start says rather than at a needle.
+func (rule redactionRule) from(start redactStart) redactionRule {
+ rule.start = start
+ return rule
+}
+
+// before returns the rule with its matches beginning offset bytes before a needle.
+func (rule redactionRule) before(offset int) redactionRule {
+ rule.offset = offset
+ return rule
+}
+
+// apply replaces every match of the rule in the gate's text, except one whose value keep passes,
+// and a repeating rule until the text stops changing.
+func (rule redactionRule) apply(text *redactGate, slot *redactSlot) string {
+ out := rule.applyOnce(text, slot)
+ // Each pass redacts at least one more value, and a redacted value matches no further.
+ for pass := 0; rule.repeat && out != text.value && pass < 16; pass++ {
+ text.set(out)
+ out = rule.applyOnce(text, slot)
}
- group := 2 * index
+ return out
+}
+
+func (rule redactionRule) applyOnce(text *redactGate, slot *redactSlot) string {
+ value := text.value
var out []byte
last := 0
- for _, match := range pattern.FindAllStringSubmatchIndex(value, -1) {
- if match[group] >= 0 && rule.keep(value[match[group]:match[group+1]]) {
- continue
+ for match, pattern := range rule.matches(text, slot) {
+ if rule.keep != nil {
+ index := pattern.SubexpIndex("value")
+ if index < 0 {
+ index = 2
+ }
+ if group := 2 * index; match[group] >= 0 && rule.keep(value[match[group]:match[group+1]]) {
+ continue
+ }
}
out = append(out, value[last:match[0]]...)
out = pattern.ExpandString(out, rule.replacement, value, match)
@@ -472,11 +502,14 @@ func (rule redactionRule) apply(value string) string {
return string(append(out, value[last:]...))
}
-// redactAssignedPattern captures the value of a labeled assignment, quoted with ", ' or a backtick
+// redactAssignedRule captures the value of a labeled assignment, quoted with ", ' or a backtick
// (${1} to ${3}) or not (${4}), so a bare copy of it elsewhere in the same text is
// removed too.
-var redactAssignedPattern = sync.OnceValue(func() *regexp.Regexp {
- return regexp.MustCompile(`(?i)\b` + redactLabel + redactSeparator + `(?:` + quotedAlternatives() + `|(` + redactUnquoted + `))`)
+var redactAssignedRule = sync.OnceValue(func() redactionRule {
+ return redactionRule{
+ expr: `(?i)\b` + redactLabel + redactSeparator + `(?:` + quotedAlternatives() + `|(` + redactUnquoted + `))`,
+ needles: redactLabelNeedles, start: startLabel,
+ }
})
// quotedAlternatives is every quotedValues entry as one capturing alternative.
@@ -502,34 +535,49 @@ func RedactString(value string) string {
return redactString(value, true)
}
-// redactString is RedactString; gated false runs every rule whatever the text holds, which the
-// tests use to prove the gate never changes a result.
+// redactString is RedactString; gated false runs every rule's whole pattern over the whole text
+// whatever the text holds, which the tests use to prove the gate and the needle-anchored matching
+// (redact_engine.go) never change a result.
//
// The gate is what keeps redaction affordable on the hook path, where it runs over every string of
// every event before the agent may continue. Go's regexp engine has no DFA, so the case-insensitive
// label rules cost microseconds on a short string and tens of milliseconds on a tool output of a
-// few kilobytes. Almost no text holds a needle of most rules, and a substring search for one is
-// orders of magnitude cheaper than the pattern it stands in front of.
+// few kilobytes. Almost no text holds a needle of most rules, and one pass over the text finds
+// every needle of every rule at once.
func redactString(value string, gated bool) string {
- text := redactGate{value: value, enabled: gated}
+ engine := redactEngineFor()
+ text := redactGate{value: value, enabled: gated, engine: engine}
+ if text.holdsNothing() {
+ // Most text: no rule's needle, so no set holds and no rule can match.
+ return value
+ }
// Remember assignment values before replacing their labeled occurrence so
// repeated bare copies in the same retained prompt/output are also removed.
// This is common in model reasoning that quotes `token=value` and later
// refers to `value` by itself.
- assigned := map[string]struct{}{}
+ var assigned map[string]struct{}
if text.holds(redactLabelNeedles) {
- for _, match := range redactAssignedPattern().FindAllStringSubmatch(value, -1) {
- copied := strings.Join(match[1:], "")
- if len(copied) >= 8 && copied != redactedValue && !isAuthScheme(copied) {
- assigned[copied] = struct{}{}
+ assignedRule := redactAssignedRule()
+ for match := range assignedRule.matches(&text, &engine.assigned) {
+ var copied strings.Builder
+ for g := 2; g+1 < len(match); g += 2 {
+ if match[g] >= 0 {
+ copied.WriteString(value[match[g]:match[g+1]])
+ }
+ }
+ if s := copied.String(); len(s) >= 8 && s != redactedValue && !isAuthScheme(s) {
+ if assigned == nil {
+ assigned = map[string]struct{}{}
+ }
+ assigned[s] = struct{}{}
}
}
}
- for _, rule := range redactionRules {
+ for i, rule := range redactionRules {
if !text.holds(rule.needles) {
continue
}
- text.set(rule.apply(text.value))
+ text.set(rule.apply(&text, &engine.rules[i]))
}
value = text.value
// Each copy is searched for and removed; none of it survives into the result.
@@ -611,70 +659,6 @@ func IsSecretFlag(arg string) bool {
var redactSecretFlagPattern = lazyPattern(`(?i)^` + redactFlagName + `$`)
-// redactGate answers whether text holds any of a set of needles, lower-casing it and searching for
-// each set at most once per version of the text.
-type redactGate struct {
- value string
- enabled bool
- lower string
- lowered bool
- // known and present are bit sets over redactNeedles ids: which sets were searched for in this
- // version of the text, and which were found.
- known, present uint64
- // unfoldable is set when the text holds a character Go's case folding equates with an ASCII
- // letter (U+017F folds to s, U+212A to k): a case-insensitive pattern can match "ſecret", which
- // lower-casing does not turn into "secret", so folded rules always run on such text.
- unfoldable bool
-}
-
-func (g *redactGate) set(value string) {
- if value != g.value {
- g.value, g.lowered, g.known, g.present = value, false, 0, 0
- }
-}
-
-func (g *redactGate) holds(set *redactNeedles) bool {
- if !g.enabled {
- return true
- }
- if set.within != nil && !g.holds(set.within) {
- return false
- }
- bit := uint64(1) << set.id
- if g.known&bit != 0 {
- return g.present&bit != 0
- }
- g.known |= bit
- haystack := g.value
- if set.folded {
- if !g.lowered {
- g.lower, g.lowered = strings.ToLower(g.value), true
- g.unfoldable = strings.ContainsAny(g.value, "\u017f\u212a")
- }
- if g.unfoldable {
- g.present |= bit
- return true
- }
- haystack = g.lower
- }
- for _, needle := range set.needles {
- for from := 0; ; {
- i := strings.Index(haystack[from:], needle)
- if i < 0 {
- break
- }
- start, end := from+i, from+i+len(needle)
- // The next search starts one byte on, not at end: an occurrence may overlap this one.
- from = start + 1
- if (set.precededBy == nil || set.precededBy(haystack[:start])) && (set.followedBy == nil || set.followedBy(haystack[end:])) {
- g.present |= bit
- return true
- }
- }
- }
- return false
-}
-
// secretKeyName reports whether a structured field's name says its value is a credential, and
// whether the only word that says so is "tokens". The name is split into words at punctuation and
// at camelCase, and a word naming a credential marks it wherever it stands: `GITHUB_TOKEN`,
diff --git a/pkg/asymptoteobserve/privacy_bench_test.go b/pkg/asymptoteobserve/privacy_bench_test.go
new file mode 100644
index 00000000..68e722a3
--- /dev/null
+++ b/pkg/asymptoteobserve/privacy_bench_test.go
@@ -0,0 +1,84 @@
+package asymptoteobserve_test
+
+import (
+ "fmt"
+ "strings"
+ "testing"
+
+ obs "github.com/asymptote-labs/agent-beacon/pkg/asymptoteobserve"
+)
+
+// The benchmarks in this file time redaction on the text a hook redacts most: a file read's result,
+// a system prompt, a shell command, and text whose every line holds a secret.
+
+var benchGoLines = []string{
+ "package cache", "", "import (", "\t\"container/list\"", "\t\"sync\"", "\t\"time\"", ")",
+ "// LRU is a fixed-size cache that evicts the least recently used entry.",
+ "type LRU[K comparable, V any] struct {", "\tmu sync.Mutex", "\tcap int", "\tll *list.List",
+ "\titems map[K]*list.Element", "}", "func (c *LRU[K, V]) Get(k K) (V, bool) {", "\tc.mu.Lock()",
+ "\tdefer c.mu.Unlock()", "\tif e, ok := c.items[k]; ok {", "\t\tc.ll.MoveToFront(e)",
+ "\t\treturn e.Value.(*entry[K, V]).v, true", "\t}", "\tvar zero V", "\treturn zero, false", "}",
+ "\t// Expired entries are dropped lazily on the next Put.", "\t\tif err := c.flush(ctx); err != nil {",
+ "\t\t\treturn fmt.Errorf(\"flush: %w\", err)", "\t\t}",
+}
+
+// benchFileRead is n bytes of `cat -n` output of Go source, as a Read tool returns it.
+func benchFileRead(n int) string {
+ var b strings.Builder
+ for i := 0; b.Len() < n; i++ {
+ fmt.Fprintf(&b, "%6d\t%s\n", i+1, benchGoLines[i%len(benchGoLines)])
+ }
+ return b.String()[:n]
+}
+
+// benchSystemPrompt is n bytes of a coding agent's system prompt.
+func benchSystemPrompt(n int) string {
+ paragraph := "You are an interactive CLI tool that helps users with software engineering tasks. " +
+ "Use the instructions below and the tools available to you to assist the user. When you run a " +
+ "non-trivial bash command, explain what it does and why. Never introduce code that exposes or " +
+ "logs secrets and keys, and never commit secrets or keys to the repository.\n\n"
+ return strings.Repeat(paragraph, n/len(paragraph)+1)[:n]
+}
+
+var benchTexts = []struct {
+ name, text string
+ limit int
+}{
+ {"file.read_3KB", benchFileRead(3000), obs.DefaultRawStringLimit},
+ {"system_prompt_12KB", benchSystemPrompt(12000), obs.DefaultStringLimit},
+ {"command", "cd /home/user/project && go test ./... -count=1 2>&1 | tail -50", obs.DefaultStringLimit},
+ {"env_assignments_1KB", strings.Repeat("API_TOKEN=abcdefgh12345678 GITHUB_TOKEN=ghp_0123456789abcdefghijklmnopqrstuvwxyz\n", 12), obs.DefaultStringLimit},
+}
+
+func BenchmarkCleanString(b *testing.B) {
+ for _, bc := range benchTexts {
+ b.Run(bc.name, func(b *testing.B) {
+ b.ReportAllocs()
+ for b.Loop() {
+ obs.CleanString(bc.text, bc.limit, true)
+ }
+ })
+ }
+}
+
+// BenchmarkRedactAdversarial redacts 16 KB texts built to make rules try many places, each of
+// which a pattern could read on from to the end of the line: many curl commands on one line, a
+// flag name that is one long run of credential words, JWT-shaped fragments that never complete,
+// and assignments without values.
+func BenchmarkRedactAdversarial(b *testing.B) {
+ repeat := func(unit string) string { return strings.Repeat(unit, 16<<10/len(unit)+1)[:16<<10] }
+ for _, bc := range []struct{ name, text string }{
+ {"curl_words_one_line", repeat("curl -H 'a: b' -b ")},
+ {"long_flag_run", "-" + repeat("token") + " x"},
+ {"jwt_prefixes", repeat("-eyJabcdefghij")},
+ {"labels_without_values", repeat("key=,")},
+ } {
+ b.Run(bc.name, func(b *testing.B) {
+ b.SetBytes(int64(len(bc.text)))
+ b.ReportAllocs()
+ for b.Loop() {
+ obs.RedactString(bc.text)
+ }
+ })
+ }
+}
diff --git a/pkg/asymptoteobserve/redact_engine.go b/pkg/asymptoteobserve/redact_engine.go
new file mode 100644
index 00000000..fd4e5cfe
--- /dev/null
+++ b/pkg/asymptoteobserve/redact_engine.go
@@ -0,0 +1,703 @@
+package asymptoteobserve
+
+import (
+ "iter"
+ "math/bits"
+ "regexp"
+ "regexp/syntax"
+ "slices"
+ "strings"
+ "sync"
+ "unicode/utf8"
+)
+
+// This file is how redaction finds the matches of its rules (privacy.go). The rules, their order,
+// their patterns and their replacements are unchanged; what changed is how a rule's matches are
+// found, without changing which:
+//
+// - The gate used to lower-case the text and run a substring search per needle set. On a few
+// kilobytes of tool output or source code that search was nearly all of redaction's cost. One
+// pass of an Aho-Corasick automaton over the text, folding ASCII case as it reads, now finds
+// every occurrence of every needle of every rule (needleScanner).
+// - A rule's pattern used to run over the whole text once the gate let it through. Go's regexp
+// has no DFA, so that cost tens of nanoseconds a byte for the case-insensitive label rules. A
+// match always holds one of the rule's needles, at a place the rule's start names, so the
+// pattern now runs anchored at those places only, which finds exactly the matches the whole
+// pattern does (see redactionRule.matches).
+// - Redaction stays linear in the text whatever the text holds. A pattern anchored at one place
+// can read on to the end of its line or run, so many places on one long line could cost a
+// quadratic amount; past a budget a rule runs its whole pattern once instead, from a start
+// where it finds the same matches (redactSlot.safeStart). TestRedactionWorkIsLinear counts the
+// work on texts built for the worst case.
+//
+// Text holding a character Go's case folding equates with an ASCII letter (U+017F folds to s,
+// U+212A to k) can match a case-insensitive pattern where no needle occurs, so on such text the
+// case-insensitive rules run their whole patterns, as they always did. redactString(value, false)
+// runs every rule's whole pattern on any text, and TestRedactGateNeverChangesAResult and
+// FuzzRedactGate hold the two to the same result.
+
+// redactStart says where, around an occurrence of one of a rule's needles that passes the needle
+// set's precededBy and followedBy (and those of the sets it is within), a match of the rule's
+// pattern can begin.
+type redactStart uint8
+
+const (
+ // startNeedle: the rule's offset bytes before the needle, at a word boundary when the pattern
+ // opens with \b.
+ startNeedle redactStart = iota
+ // startLabel: at a word boundary in the run of [A-Za-z0-9_.-] that ends at the needle, where
+ // redactLabel's lazy `[A-Za-z0-9_.-]*?` begins. The pattern opens with \b.
+ startLabel
+ // startFlag: at the whitespace before the flag that ends at the needle, or at the start of the
+ // text where the flag does: redactFlag's `(?:^|\s)`.
+ startFlag
+ // startListedFlag: at the quote before the flag that ends at the needle.
+ startListedFlag
+ // startScheme: at a letter on a word boundary in the run of [A-Za-z0-9+.-] before the needle,
+ // a URL's `\b[a-z][a-z0-9+.-]*` scheme. The pattern opens with \b.
+ startScheme
+ // startLine: anywhere on the line of the needle or the blank lines before it. The pattern runs
+ // whole from the start of that line, where its (?m)^ and \b see what they see in the whole
+ // text.
+ startLine
+)
+
+// redactSlot is what one rule compiles, on first use.
+type redactSlot struct {
+ once sync.Once
+ anchored *regexp.Regexp
+ boundary bool
+ // The assertions the whole pattern makes, which decide where it can be run on a suffix of the
+ // text with the same result (safeStart): \b or \B, (?m)^, and ^ or \A.
+ wordAssert, lineStart, textStart bool
+}
+
+// prepare compiles the rule's pattern anchored at the start of the text, without the \b it opens
+// with, if any: the anchored pattern cannot see the byte before its start, so the boundary is
+// checked against the whole text instead.
+func (s *redactSlot) prepare(rule redactionRule) {
+ s.once.Do(func() {
+ expr := rule.expr
+ for _, prefix := range []string{`(?i)(\b`, `(?i)\b`, `(\b`, `\b`} {
+ if strings.HasPrefix(expr, prefix) {
+ s.boundary = true
+ expr = prefix[:len(prefix)-2] + expr[len(prefix):]
+ break
+ }
+ }
+ if (rule.start == startLabel || rule.start == startScheme) && !s.boundary {
+ panic("asymptoteobserve: a label or scheme redaction rule must open with \\b: " + rule.expr)
+ }
+ s.anchored = regexp.MustCompile(`^` + expr)
+ parsed, err := syntax.Parse(rule.expr, syntax.Perl)
+ if err != nil {
+ panic(err)
+ }
+ var walk func(re *syntax.Regexp)
+ walk = func(re *syntax.Regexp) {
+ switch re.Op {
+ case syntax.OpWordBoundary, syntax.OpNoWordBoundary:
+ s.wordAssert = true
+ case syntax.OpBeginLine:
+ s.lineStart = true
+ case syntax.OpBeginText:
+ s.textStart = true
+ }
+ for _, sub := range re.Sub {
+ walk(sub)
+ }
+ }
+ walk(parsed)
+ })
+}
+
+// safeStart returns the latest position at or before q where the rule's whole pattern can run on
+// the rest of the text and find what it finds there in the whole text: a position whose assertions
+// read the same with the text before it cut off. It is the start of the text, or a position after
+// an ASCII byte (so a character boundary however the text before it decodes) that is not a word
+// character when the pattern tests word boundaries, and a newline when it tests line starts; a
+// pattern that tests the start of the text can only run from it.
+func (s *redactSlot) safeStart(t string, q int) int {
+ if s.textStart {
+ return 0
+ }
+ from := q
+ for ; from > 0; from-- {
+ c := t[from-1]
+ if c < utf8.RuneSelf && (!s.wordAssert || !isWordByte(c)) && (!s.lineStart || c == '\n') {
+ break
+ }
+ }
+ if redactWork != nil {
+ redactWork(q - from)
+ }
+ return from
+}
+
+// Anchored matching tries one place per occurrence of a rule's needles, and a pattern anchored at
+// one place can read on to the end of the text before it fails: a curl rule reads its whole line.
+// A text with many occurrences could make that quadratic. So before trying any, a rule adds up the
+// text it would hand the pattern, from each place to the end, and when that passes
+// redactAnchoredPerByte times the text's length plus redactAnchoredBase, it runs its whole pattern
+// once instead, from a safe start before the first place. Either way the matches are the same, and
+// the regexp engine reads at most that budget plus the text once: linear in the text, whatever it
+// holds. TestRedactionWorkIsLinear holds this.
+var (
+ redactAnchoredPerByte = 8
+ redactAnchoredBase = 16 << 10
+ // redactWork, when set (by tests), is told the length of every text redaction hands a pattern
+ // and of every run it scans back over to find where a match starts.
+ redactWork func(n int)
+)
+
+// redactEngine is built the first time anything is redacted: building it at package init would
+// cost every hook run its allocations whether it redacts or not (TestPackageInitStaysCheap).
+type redactEngine struct {
+ scanner needleScanner
+ setMasks [64]uint64 // redactNeedles id -> the needles of the set, as scanner needle bits
+ rules []redactSlot
+ assigned redactSlot
+}
+
+var redactEngineFor = sync.OnceValue(func() *redactEngine {
+ e := &redactEngine{rules: make([]redactSlot, len(redactionRules))}
+ ids := map[redactNeedle]int{}
+ var needles []redactNeedle
+ var register func(set *redactNeedles)
+ register = func(set *redactNeedles) {
+ for ; set != nil; set = set.within {
+ for _, text := range set.needles {
+ n := redactNeedle{text: text, folded: set.folded}
+ id, ok := ids[n]
+ if !ok {
+ id = len(needles)
+ ids[n] = id
+ needles = append(needles, n)
+ }
+ e.setMasks[set.id] |= 1 << id
+ }
+ }
+ }
+ for _, rule := range redactionRules {
+ register(rule.needles)
+ }
+ register(redactAssignedRule().needles)
+ // Not needles of any rule: they mark text a case-insensitive pattern can match without one.
+ for _, text := range []string{"\u017f", "\u212a"} {
+ e.scanner.unfoldable |= 1 << len(needles)
+ needles = append(needles, redactNeedle{text: text})
+ }
+ if len(needles) > 64 {
+ panic("asymptoteobserve: redaction tracks at most 64 needles")
+ }
+ e.scanner.build(needles)
+ return e
+})
+
+// compiledPatterns holds every redaction pattern compiled so far, by expression.
+var compiledPatterns sync.Map
+
+// compiledPattern compiles expr the first time it is asked for.
+func compiledPattern(expr string) *regexp.Regexp {
+ if re, ok := compiledPatterns.Load(expr); ok {
+ return re.(*regexp.Regexp)
+ }
+ re, _ := compiledPatterns.LoadOrStore(expr, regexp.MustCompile(expr))
+ return re.(*regexp.Regexp)
+}
+
+// redactGate answers whether text holds any of a set of needles: an occurrence of one that
+// passes the set's precededBy and followedBy, in a text that holds the set it is within. The
+// occurrences are found once per version of the text.
+type redactGate struct {
+ value string
+ enabled bool
+ engine *redactEngine
+ scanned bool
+ hits []redactHit
+ found uint64 // scanner needle bits
+ // known and present are bit sets over redactNeedles ids: which sets were looked for in this
+ // version of the text, and which were found.
+ known, present uint64
+}
+
+func (g *redactGate) set(value string) {
+ if value != g.value {
+ g.value, g.scanned, g.known, g.present = value, false, 0, 0
+ }
+}
+
+func (g *redactGate) e() *redactEngine {
+ if g.engine == nil {
+ g.engine = redactEngineFor()
+ }
+ return g.engine
+}
+
+func (g *redactGate) scan() {
+ if !g.scanned {
+ g.hits, g.found = g.e().scanner.scan(g.value, g.hits[:0])
+ g.scanned = true
+ }
+}
+
+// unfoldable reports whether the text holds U+017F or U+212A, which a case-insensitive pattern
+// equates with s and k: such a pattern can match "ſecret", which holds no needle "secret".
+func (g *redactGate) unfoldable() bool {
+ g.scan()
+ return g.found&g.e().scanner.unfoldable != 0
+}
+
+// holdsNothing reports whether the text holds no needle at all, and so no set: no rule can match
+// it.
+func (g *redactGate) holdsNothing() bool {
+ g.scan()
+ return g.enabled && g.found == 0
+}
+
+func (g *redactGate) holds(set *redactNeedles) bool {
+ if !g.enabled {
+ return true
+ }
+ if set.within != nil && !g.holds(set.within) {
+ return false
+ }
+ bit := uint64(1) << set.id
+ if g.known&bit != 0 {
+ return g.present&bit != 0
+ }
+ g.known |= bit
+ if set.folded && g.unfoldable() {
+ g.present |= bit
+ return true
+ }
+ g.scan()
+ mask := g.e().setMasks[set.id]
+ if g.found&mask == 0 {
+ return false
+ }
+ for _, h := range g.hits {
+ if mask&(1<<h.id) != 0 && set.passes(g.e(), g.value, h, false) {
+ g.present |= bit
+ return true
+ }
+ }
+ return false
+}
+
+// passes reports whether the occurrence h passes the set's followedBy and precededBy and, with
+// within, those of the sets it is within that hold its needle too. For a rule's own set with
+// within, that is a condition every match holding the occurrence meets: a narrower set's needle
+// ends the same label or flag as the broader set's does.
+//
+// followedBy is asked first. It reads only the separator after the needle, and it passes only a
+// needle that ends its run of letters, digits and dashes, so precededBy, which reads that run back
+// from the needle, reads each run once however many needles it holds.
+func (set *redactNeedles) passes(engine *redactEngine, t string, h redactHit, within bool) bool {
+ masks := &engine.setMasks
+ for s := set; s != nil; s = s.within {
+ if masks[s.id]&(1<<h.id) != 0 && (s.followedBy != nil && !s.followedBy(t[h.end:]) || s.precededBy != nil && !s.precededBy(t[:h.start])) {
+ return false
+ }
+ if !within {
+ break
+ }
+ }
+ return true
+}
+
+// matches yields what FindAllStringSubmatchIndex(text, -1) of the rule's pattern returns, in
+// order and with the same indexes, each with a pattern ExpandString can expand it with.
+//
+// FindAll returns, after the end of the previous match, the match that starts leftmost, the one the
+// pattern prefers among those starting there. Every match holds an occurrence of one of the rule's
+// needles that passes its set's conditions, and begins at a place the rule's start names around
+// it. The pattern anchored at such a place (with an opening \b checked against the whole text,
+// which the anchored pattern cannot see before its start) finds exactly the match the whole
+// pattern prefers among those starting there. So trying the places in order and taking the first
+// that matches is what FindAll does, without running the pattern over the text between them.
+//
+// Where a match can begin anywhere in a run before the needle (startLabel, startScheme), only the
+// leftmost place in the run is tried. Whatever a match from a later place in the run takes, a
+// match from the earlier one can take too, its lazy or starred run of the same characters taking
+// the ones between; so when the earlier place does not match, no place in the run does.
+//
+// When the places would hand the pattern more text than the budget allows (redactAnchoredPerByte),
+// the whole pattern runs once from a safe start before the first of them instead. No match starts
+// before the first place, and from a safe start the pattern finds what it finds in the whole text.
+func (rule redactionRule) matches(text *redactGate, slot *redactSlot) iter.Seq2[[]int, *regexp.Regexp] {
+ return func(yield func([]int, *regexp.Regexp) bool) {
+ t := text.value
+ whole := func(from int) {
+ if redactWork != nil {
+ redactWork(len(t) - from)
+ }
+ re := rule.pattern()
+ for _, m := range re.FindAllStringSubmatchIndex(t[from:], -1) {
+ for i := range m {
+ if m[i] >= 0 {
+ m[i] += from
+ }
+ }
+ if !yield(m, re) {
+ return
+ }
+ }
+ }
+ if !text.enabled || rule.needles.folded && text.unfoldable() {
+ whole(0)
+ return
+ }
+ text.scan()
+ engine := text.e()
+ mask := engine.setMasks[rule.needles.id]
+ if text.found&mask == 0 {
+ return
+ }
+ if rule.start == startLine {
+ if from, ok := lineStart(engine, t, text.hits, mask, rule.needles); ok {
+ whole(from)
+ }
+ return
+ }
+ slot.prepare(rule)
+ // What the anchored attempts would hand the pattern, at most: each place, to the end.
+ first, handed, last := -1, 0, -1
+ budget := redactAnchoredPerByte*len(t) + redactAnchoredBase
+ for _, h := range text.hits {
+ if mask&(1<<h.id) == 0 || !rule.needles.passes(engine, t, h, true) {
+ continue
+ }
+ q := rule.startFor(t, h, 0, slot.boundary)
+ if q < 0 || q == last {
+ continue
+ }
+ if first < 0 || q < first {
+ first = q
+ }
+ last = q
+ if handed += len(t) - q; handed > budget {
+ break
+ }
+ }
+ if first < 0 {
+ return
+ }
+ if handed > budget {
+ whole(slot.safeStart(t, first))
+ return
+ }
+ pos, failed := 0, -1
+ for _, h := range text.hits {
+ if mask&(1<<h.id) == 0 || h.start < pos || !rule.needles.passes(engine, t, h, true) {
+ continue
+ }
+ q := rule.startFor(t, h, pos, slot.boundary)
+ if q < 0 || q == failed {
+ continue
+ }
+ if redactWork != nil {
+ redactWork(len(t) - q)
+ }
+ m := slot.anchored.FindStringSubmatchIndex(t[q:])
+ if m == nil {
+ failed = q
+ continue
+ }
+ for i := range m {
+ if m[i] >= 0 {
+ m[i] += q
+ }
+ }
+ pos = m[1]
+ if !yield(m, slot.anchored) {
+ return
+ }
+ }
+ }
+}
+
+// startFor returns the leftmost place at or after pos where a match holding h can begin, or -1.
+func (rule redactionRule) startFor(t string, h redactHit, pos int, boundary bool) int {
+ switch rule.start {
+ case startNeedle:
+ q := h.start - rule.offset
+ if q < pos || boundary && !wordBoundary(t, q) {
+ return -1
+ }
+ return q
+ case startLabel, startScheme:
+ class, last := &labelClass, h.start
+ if rule.start == startScheme {
+ class, last = &schemeClass, h.start-1
+ }
+ run := h.start
+ for run > 0 && class[t[run-1]] {
+ run--
+ }
+ if redactWork != nil {
+ redactWork(h.start - run)
+ }
+ for q := max(run, pos); q <= last; q++ {
+ if wordBoundary(t, q) && (rule.start == startLabel || isASCIILetter(t[q])) {
+ return q
+ }
+ }
+ return -1
+ case startFlag, startListedFlag:
+ // The needle set's precededBy has checked that what comes before the flag is whitespace,
+ // the start of the text, or (listed) a quote.
+ q, _ := flagStart(t[:h.start])
+ if q > 0 {
+ q--
+ }
+ if q < pos {
+ return -1
+ }
+ return q
+ }
+ panic("asymptoteobserve: no start for a redaction rule")
+}
+
+// lineStart returns the start of a line at or before every place a startLine match holding one of
+// the needles can begin: before the needle, an optional `set-`, a quote, whitespace across lines,
+// and a few bytes (`=`, `--header`) more.
+func lineStart(engine *redactEngine, t string, hits []redactHit, mask uint64, set *redactNeedles) (int, bool) {
+ from, ok := len(t), false
+ for _, h := range hits {
+ if mask&(1<<h.id) == 0 || !set.passes(engine, t, h, true) {
+ continue
+ }
+ b := h.start
+ if b >= 4 && strings.EqualFold(t[b-4:b], "set-") {
+ b -= 4
+ }
+ if b > 0 && strings.IndexByte(redactQuotes, t[b-1]) >= 0 {
+ b--
+ }
+ for b > 0 && isRegexpSpace(t[b-1]) {
+ b--
+ }
+ from, ok = min(from, max(0, b-10)), true
+ }
+ for from > 0 && t[from-1] != '\n' {
+ from--
+ }
+ return from, ok
+}
+
+func isRegexpSpace(c byte) bool {
+ return c == ' ' || c == '\t' || c == '\n' || c == '\f' || c == '\r'
+}
+
+// isWordByte is \b's notion of a word character; a byte of a multi-byte character, or an invalid
+// one, decodes to a rune that is not one.
+func isWordByte(c byte) bool {
+ return 'a' <= c && c <= 'z' || 'A' <= c && c <= 'Z' || '0' <= c && c <= '9' || c == '_'
+}
+
+// wordBoundary reports whether \b holds at i < len(t).
+func wordBoundary(t string, i int) bool {
+ return (i > 0 && isWordByte(t[i-1])) != isWordByte(t[i])
+}
+
+func isASCIILetter(c byte) bool {
+ return 'a' <= c && c <= 'z' || 'A' <= c && c <= 'Z'
+}
+
+// labelClass and schemeClass are the runs a startLabel and a startScheme match begin in, with
+// capitals: the patterns are case-insensitive.
+var labelClass, schemeClass = func() (label, scheme [256]bool) {
+ for c := range 256 {
+ b := byte(c)
+ alnum := isASCIILetter(b) || '0' <= b && b <= '9'
+ label[c] = alnum || b == '_' || b == '.' || b == '-'
+ scheme[c] = alnum || b == '+' || b == '.' || b == '-'
+ }
+ return label, scheme
+}()
+
+// ---- the needle scanner ----
+
+// redactNeedle is one literal some rule's pattern cannot match without; folded needles are lower
+// case and found whatever the case of the text's ASCII letters.
+type redactNeedle struct {
+ text string
+ folded bool
+}
+
+// redactHit is one occurrence of a needle.
+type redactHit struct {
+ start, end int
+ id int
+}
+
+// needleScanner is an Aho-Corasick automaton over every needle, reading the text with its ASCII
+// letters lower-cased: one pass finds every occurrence of every needle, overlapping ones included.
+//
+// A state is the offset of its row in delta, so a step is one add and one load. The states where
+// a needle ends are numbered last, from firstOut, so a step that finds none costs one comparison.
+type needleScanner struct {
+ needles []redactNeedle
+ class [256]uint8 // byte -> input class; 0 for bytes no needle holds
+ delta []uint32 // state + class -> next state
+ out []uint64 // (state-firstOut)/nc -> the needles ending there
+ nc int // number of classes, the length of a row
+ firstOut int
+ longest int // the longest needle's length
+ unfoldable uint64 // the needles that are U+017F and U+212A
+}
+
+func asciiLower(c byte) byte {
+ if 'A' <= c && c <= 'Z' {
+ return c + 'a' - 'A'
+ }
+ return c
+}
+
+func (sc *needleScanner) build(needles []redactNeedle) {
+ sc.needles = needles
+ var classOf [256]uint8
+ n := 1
+ for _, nd := range needles {
+ sc.longest = max(sc.longest, len(nd.text))
+ for i := range len(nd.text) {
+ if c := asciiLower(nd.text[i]); classOf[c] == 0 {
+ classOf[c] = uint8(n)
+ n++
+ }
+ }
+ }
+ sc.nc = n
+ for b := range 256 {
+ sc.class[b] = classOf[asciiLower(byte(b))]
+ }
+ // The trie.
+ next := [][]int{make([]int, n)}
+ out := []uint64{0}
+ for id, nd := range needles {
+ s := 0
+ for i := range len(nd.text) {
+ c := sc.class[nd.text[i]]
+ if next[s][c] == 0 {
+ next = append(next, make([]int, n))
+ out = append(out, 0)
+ next[s][c] = len(next) - 1
+ }
+ s = next[s][c]
+ }
+ out[s] |= 1 << id
+ }
+ // Failure links, breadth first, folded into a complete transition table. Class 0 never
+ // continues a needle, so it always returns to the root.
+ fail := make([]int, len(next))
+ delta := make([]int, len(next)*n)
+ var queue []int
+ for c := 1; c < n; c++ {
+ if t := next[0][c]; t != 0 {
+ queue = append(queue, t)
+ }
+ delta[c] = next[0][c]
+ }
+ for len(queue) > 0 {
+ s := queue[0]
+ queue = queue[1:]
+ out[s] |= out[fail[s]]
+ for c := 1; c < n; c++ {
+ if t := next[s][c]; t != 0 {
+ fail[t] = delta[fail[s]*n+c]
+ delta[s*n+c] = t
+ queue = append(queue, t)
+ } else {
+ delta[s*n+c] = delta[fail[s]*n+c]
+ }
+ }
+ }
+ // Renumber: the root first, then the states no needle ends in, then those some do.
+ order := make([]int, 0, len(next))
+ for s := range next {
+ if out[s] == 0 {
+ order = append(order, s)
+ }
+ }
+ sc.firstOut = len(order) * n
+ for s := range next {
+ if out[s] != 0 {
+ order = append(order, s)
+ sc.out = append(sc.out, out[s])
+ }
+ }
+ row := make([]int, len(next))
+ for i, s := range order {
+ row[s] = i * n
+ }
+ sc.delta = make([]uint32, len(next)*n)
+ for s := range next {
+ for c := range n {
+ sc.delta[row[s]+c] = uint32(row[delta[s*n+c]])
+ }
+ }
+}
+
+// scanLanes is how many places of a long text are read at once. One reading is a chain of
+// dependent loads; independent ones overlap.
+const scanLanes = 4
+
+// scan appends every needle occurrence in t to hits, ordered by start, and returns them with the
+// set of needles found.
+func (sc *needleScanner) scan(t string, hits []redactHit) ([]redactHit, uint64) {
+ var found uint64
+ delta, class, first := sc.delta, &sc.class, uint32(sc.firstOut)
+ s, from := uint32(0), 0
+ if seg := len(t) / scanLanes; seg >= 64 {
+ // Lane k reports the occurrences that end in (k*seg, (k+1)*seg]. It starts reading early
+ // enough for the longest needle to end at its first byte, and lane 0 reads that much past
+ // its end; what ends outside its part, a lane does not report.
+ overlap := sc.longest - 1
+ t0 := t[:seg+overlap]
+ t1 := t[seg-overlap : 2*seg]
+ t2 := t[2*seg-overlap : 3*seg]
+ t3 := t[3*seg-overlap : 4*seg]
+ var s0, s1, s2, s3 uint32
+ for j := 0; j < len(t0) && j < len(t1) && j < len(t2) && j < len(t3); j++ {
+ s0 = delta[s0+uint32(class[t0[j]])]
+ s1 = delta[s1+uint32(class[t1[j]])]
+ s2 = delta[s2+uint32(class[t2[j]])]
+ s3 = delta[s3+uint32(class[t3[j]])]
+ if s0 >= first || s1 >= first || s2 >= first || s3 >= first {
+ for k, st := range [scanLanes]uint32{s0, s1, s2, s3} {
+ if end := k*seg - overlap*min(k, 1) + j + 1; st >= first && end > k*seg && end <= (k+1)*seg {
+ hits = sc.record(t, st, end, hits, &found)
+ }
+ }
+ }
+ }
+ // The last lane goes on to the end.
+ s, from = s3, scanLanes*seg
+ }
+ for i := from; i < len(t); i++ {
+ if s = delta[s+uint32(class[t[i]])]; s >= first {
+ hits = sc.record(t, s, i+1, hits, &found)
+ }
+ }
+ // The automaton finds occurrences by where they end, lane by lane; the rules want them by
+ // where they start.
+ slices.SortStableFunc(hits, func(a, b redactHit) int { return a.start - b.start })
+ return hits, found
+}
+
+// record appends the occurrences ending at end, where the automaton is in state s. A needle that
+// is not folded must match the text byte for byte.
+func (sc *needleScanner) record(t string, s uint32, end int, hits []redactHit, found *uint64) []redactHit {
+ for m := sc.out[(int(s)-sc.firstOut)/sc.nc]; m != 0; m &= m - 1 {
+ id := bits.TrailingZeros64(m)
+ nd := &sc.needles[id]
+ start := end - len(nd.text)
+ if !nd.folded && t[start:end] != nd.text {
+ continue
+ }
+ *found |= 1 << id
+ hits = append(hits, redactHit{start: start, end: end, id: id})
+ }
+ return hits
+}
diff --git a/pkg/asymptoteobserve/redact_engine_test.go b/pkg/asymptoteobserve/redact_engine_test.go
new file mode 100644
index 00000000..7a0437ed
--- /dev/null
+++ b/pkg/asymptoteobserve/redact_engine_test.go
@@ -0,0 +1,214 @@
+package asymptoteobserve
+
+import (
+ mrand "math/rand"
+ "math/rand/v2"
+ "slices"
+ "strings"
+ "testing"
+)
+
+// The needle scanner must report every occurrence of every needle, overlapping ones included,
+// folded needles whatever the case of the text's ASCII letters, on text short enough to be read in
+// one lane and long enough to be read in several, with occurrences across the lanes' edges.
+func TestNeedleScannerFindsEveryOccurrence(t *testing.T) {
+ sc := &redactEngineFor().scanner
+ want := func(text string) []redactHit {
+ var hits []redactHit
+ for start := range len(text) {
+ for id, nd := range sc.needles {
+ end := start + len(nd.text)
+ if end > len(text) {
+ continue
+ }
+ if nd.folded && strings.EqualFold(text[start:end], nd.text) && isASCII(text[start:end]) || !nd.folded && text[start:end] == nd.text {
+ hits = append(hits, redactHit{start: start, end: end, id: id})
+ }
+ }
+ }
+ return hits
+ }
+ pieces := []string{" ", "\n", "x", "_", "-", "=", "://", "\u017f", "\u212a", "é", "\xff"}
+ for _, nd := range sc.needles {
+ pieces = append(pieces, nd.text, strings.ToUpper(nd.text), nd.text[:len(nd.text)/2])
+ }
+ r := rand.New(rand.NewPCG(1, 2))
+ for i := range 3000 {
+ var b strings.Builder
+ n := 1 + r.IntN(8)
+ if i%3 == 0 {
+ n = 60 + r.IntN(200)
+ }
+ for range n {
+ b.WriteString(pieces[r.IntN(len(pieces))])
+ }
+ text := b.String()
+ got, found := sc.scan(text, nil)
+ expected := want(text)
+ key := func(a, b redactHit) int {
+ if a.start != b.start {
+ return a.start - b.start
+ }
+ return a.id - b.id
+ }
+ slices.SortFunc(got, key)
+ slices.SortFunc(expected, key)
+ if !slices.Equal(got, expected) {
+ t.Fatalf("scan(%q)\n = %v\nwant %v", text, got, expected)
+ }
+ var mask uint64
+ for _, h := range expected {
+ mask |= 1 << h.id
+ }
+ if found != mask {
+ t.Fatalf("scan(%q) found %b, want %b", text, found, mask)
+ }
+ }
+}
+
+func isASCII(s string) bool {
+ for i := range len(s) {
+ if s[i] >= 0x80 {
+ return false
+ }
+ }
+ return true
+}
+
+// Every rule opens where its start says it does: running a rule's whole pattern over text built
+// around its needles, running it anchored where its start names, and running it whole from the safe
+// start before the first of those places (what it does past its budget), find the same matches.
+// TestRedactGateNeverChangesAResult checks the same through RedactString; this names the rule.
+func TestEveryRuleMatchesAsItsWholePattern(t *testing.T) {
+ engine := redactEngineFor()
+ fragments := []string{" ", "\n", "\t", "=", ": ", `"`, "'", "`", `\"`, ",", "x", "-", "--", "_", ".", "@", "/",
+ "abcdefghijklmnopqrstuvwxyz0123456789ABCDEF", "[REDACTED]", "eyJ", "set-", "-H ", "curl ", "-u ", "-b ",
+ "token=abcdefgh1234", `"api_key": "x y"`, "'secret': 42", "--password hunter2", `["--api-key", "k"]`,
+ "curl -u admin:pw https://x", "curl -b 'sid=1' x", "Cookie: sid=1", "postgres://u:p@h", "redis://:p@h",
+ "Bearer abc.def", "authorization: basic x", "ghp_abcdefghijklmnopqrstuvwxyz0123", "AKIAIOSFODNN7EXAMPLE",
+ "sk-abcdefghijklmnopqrstuvwxyz", "sk_live_abcdefghijklmnop", "ya29.abcdefghijklmnopqrstuvwxyz",
+ "eyJabcdefghij.eyJabcdefghij.abcdefghij", "-----BEGIN RSA PRIVATE KEY-----\nMIIEowIBAAKCAQEAabcdefghijklmnop\n"}
+ for _, set := range []*redactNeedles{redactLabelNeedles, redactFlagNeedles} {
+ fragments = append(fragments, set.needles...)
+ }
+ for _, rule := range redactionRules {
+ fragments = append(fragments, rule.needles.needles...)
+ }
+ r := rand.New(rand.NewPCG(3, 4))
+ matched := 0
+ for i, rule := range redactionRules {
+ // The fragments the rule's pattern matches on their own, and its needles, are drawn often.
+ near := slices.Clone(rule.needles.needles)
+ for _, f := range fragments {
+ if rule.pattern().MatchString(f) {
+ near = append(near, f)
+ }
+ }
+ for range 400 {
+ var b strings.Builder
+ for range 1 + r.IntN(10) {
+ f := fragments[r.IntN(len(fragments))]
+ if r.IntN(2) == 0 {
+ f = near[r.IntN(len(near))]
+ }
+ b.WriteString(f)
+ }
+ text := b.String()
+ collect := func(text *redactGate) (out [][]int) {
+ for m := range rule.matches(text, &engine.rules[i]) {
+ out = append(out, m)
+ }
+ return out
+ }
+ want := collect(&redactGate{value: text, enabled: false, engine: engine})
+ got := collect(&redactGate{value: text, enabled: true, engine: engine})
+ if !slices.EqualFunc(got, want, slices.Equal[[]int]) {
+ t.Fatalf("rule %d (%s) on %q: anchored matches %v, whole pattern %v", i, rule.expr, text, got, want)
+ }
+ withBudget(t, 0, -1, func() {
+ got = collect(&redactGate{value: text, enabled: true, engine: engine})
+ })
+ if !slices.EqualFunc(got, want, slices.Equal[[]int]) {
+ t.Fatalf("rule %d (%s) on %q: from a safe start %v, whole pattern %v", i, rule.expr, text, got, want)
+ }
+ matched += len(want)
+ }
+ }
+ // The texts have to reach the patterns for the comparison to mean anything.
+ if matched < 3000 {
+ t.Fatalf("only %d matches across every rule", matched)
+ }
+}
+
+// withBudget runs f with the anchored-matching budget set to perByte times the text plus base: 0
+// and -1 make every rule run its whole pattern from a safe start.
+func withBudget(t *testing.T, perByte, base int, f func()) {
+ t.Helper()
+ savedPerByte, savedBase := redactAnchoredPerByte, redactAnchoredBase
+ redactAnchoredPerByte, redactAnchoredBase = perByte, base
+ defer func() { redactAnchoredPerByte, redactAnchoredBase = savedPerByte, savedBase }()
+ f()
+}
+
+// Redaction from safe starts gives what every rule's whole pattern gives, on the gate test's own
+// random inputs: the safe start is exact for every rule, whatever comes before its first place.
+// (randomRedactInput takes the legacy math/rand generator the gate test uses.)
+func TestRedactFromSafeStartsNeverChangesAResult(t *testing.T) {
+ r := mrand.New(mrand.NewSource(7))
+ withBudget(t, 0, -1, func() {
+ for range 5000 {
+ input := randomRedactInput(r)
+ if gated, all := redactString(input, true), redactString(input, false); gated != all {
+ t.Fatalf("redaction from safe starts of %q = %q, every rule gives %q", input, gated, all)
+ }
+ }
+ })
+}
+
+// Redaction's work is linear in the text whatever the text holds. Each text below makes some rule
+// try many places, each of which a pattern could read on from to the end of a long line or run:
+// many curl commands on one line, JWT-shaped and private-key-shaped fragments that never complete,
+// assignments and flags without values, long runs of label words, URL schemes. The work counted
+// is every byte redaction hands a pattern, and every byte it scans back over to find where a match
+// starts: an upper bound on what the regexp engine and the needle checks read. Doubling the text
+// must not more than double it, give or take, at every size.
+func TestRedactionWorkIsLinear(t *testing.T) {
+ families := map[string]string{
+ "curl words on one line": "curl x ",
+ "curl flags on one line": "curl -H 'a: b' -b ",
+ "JWT prefixes": "-eyJabcdefghij",
+ "private key headers": "-----BEGIN PRIVATE KEY-----Proc-Type: x ",
+ "labels without values": "key=,",
+ "quoted labels without values": `"token": ,`,
+ "flags without values": " --token ''",
+ "listed flags": `"--token", `,
+ "one long flag": "token",
+ "one long label": "key",
+ "URL schemes": "a://",
+ "authorization headers": "authorization: bearer ,",
+ "cookie headers": "\ncookie: ",
+ "assignments": "token=abcdefgh12345678 ",
+ }
+ for name, unit := range families {
+ work := func(n int) int {
+ text := strings.Repeat(unit, n/len(unit)+1)[:n]
+ if name == "one long flag" {
+ text = "-" + text + " x"
+ }
+ total := 0
+ redactWork = func(bytes int) { total += bytes }
+ defer func() { redactWork = nil }()
+ redactString(text, true)
+ return total
+ }
+ previous := work(16 << 10)
+ for n := 32 << 10; n <= 256<<10; n *= 2 {
+ w := work(n)
+ if w > 5*previous/2+4096 {
+ t.Errorf("%s: work %d on %d bytes, %d on half as many: not linear", name, w, n, previous)
+ break
+ }
+ previous = w
+ }
+ }
+}