The cross-harness, self-improving memory layer for AI agents. Join our community: https://discord.com/invite/zdNChS2fBu
Go agent-ai-cliagent-harnessesagent-securitybrowser-extensionciclaude-codecloudcodexcursordetection-engineeringjevknowledge-basememorymobile-device-managementobservabilitysecuritysecurity-information-and-event-managementsecurity-toolstelemetrytraces
perf(redaction): find every rule's needles in one pass and run patterns only where they can match (#1143)
kyle-elliott-asymptote722e58277758
diff --git a/pkg/asymptoteobserve/perf_ratchet_test.go b/pkg/asymptoteobserve/perf_ratchet_test.goindex daa09eb5..a716b27a 100644--- a/pkg/asymptoteobserve/perf_ratchet_test.go+++ b/pkg/asymptoteobserve/perf_ratchet_test.go@@ -17,9 +17,10 @@ import (// gain cannot be given back unnoticed.// initAllocCeiling bounds the allocations this package's initialization makes, test files-// included (409 when it was set). Compiling the redaction patterns at init made it 11,508: a-// package-level regexp.MustCompile belongs behind sync.OnceValue.-const initAllocCeiling = 500+// included (409 when it was set at 500; 337 when the redaction rules stopped building a+// sync.OnceValue each at init, and it was lowered to 420). Compiling the redaction patterns at+// init made it 11,508: a package-level regexp.MustCompile belongs behind sync.OnceValue.+const initAllocCeiling = 420func TestPackageInitStaysCheap(t *testing.T) {if os.Getenv("BEACON_INIT_TRACE_CHILD") != "" {diff --git a/pkg/asymptoteobserve/privacy.go b/pkg/asymptoteobserve/privacy.goindex 7184c44e..4d05f4ce 100644--- a/pkg/asymptoteobserve/privacy.go+++ b/pkg/asymptoteobserve/privacy.go@@ -72,9 +72,9 @@ const (// argument is a constant naming a credential (the pattern spells out "password") as returning that// credential, and these rules hold patterns, not secrets.func quotedRules(kind quotedKind, keep func(string) bool) []redactionRule {- prefix, words, separator, within := `\b`+redactLabel+redactSeparator, redactLabelWords, skipSeparator, redactLabelNeedles+ prefix, words, separator, within, start := `\b`+redactLabel+redactSeparator, redactLabelWords, skipSeparator, redactLabelNeedles, startLabelif kind == quotedFlag {- prefix, words, separator, within = redactFlag, redactFlagWords, skipFlagSeparator, redactFlagNeedles+ prefix, words, separator, within, start = redactFlag, redactFlagWords, skipFlagSeparator, redactFlagNeedles, startFlag}rules := make([]redactionRule, 0, len(quotedValues))for _, quoted := range quotedValues {@@ -87,7 +87,10 @@ func quotedRules(kind quotedKind, keep func(string) bool) []redactionRule {rest, ok := separator(rest)return ok && strings.HasPrefix(rest, open)}- rules = append(rules, lazyRule(`(?i)(`+prefix+quoted[0]+`)(?P<value>`+quoted[1]+`)`, "${1}"+redactedValue, keep, needles))+ rules = append(rules, redactionRule{+ expr: `(?i)(` + prefix + quoted[0] + `)(?P<value>` + quoted[1] + `)`, replacement: "${1}" + redactedValue,+ keep: keep, needles: needles, start: start,+ })}return rules}@@ -110,7 +113,10 @@ const redactFlag = `(?:^|\s)` + redactFlagName + `(?:=|\s+)`// redactionRule is one redaction. ${1} and ${3}, when the pattern has them, are kept around the// value.type redactionRule struct {- pattern func() *regexp.Regexp+ // expr is the pattern. It is compiled the first time text gets past the needles: a hook is a+ // process per event, and most events never need most rules, so compiling all of them up front+ // cost every hook run a few milliseconds before it had read its input.+ expr stringreplacement string// keep, when set, leaves a match alone if its value (the group named value) passes.keep func(value string) bool@@ -121,13 +127,22 @@ type redactionRule struct {// matches from the command to the value, so a second such value on the line lies inside the// first match and is reached only by another pass.repeat bool+ // start says where, around an occurrence of one of its needles, a match can begin, and offset+ // how many bytes before the needle a startNeedle match does. See redact_engine.go.+ start redactStart+ offset int+}++// pattern is the rule's compiled pattern.+func (rule redactionRule) pattern() *regexp.Regexp {+ return compiledPattern(rule.expr)}// redactNeedles is a set of literals a pattern cannot match without. folded needles are lower-// case and compared against the lower-cased text, for a case-insensitive pattern. followedBy, when-// set, must also hold for the text right after the needle. within, when set, is a broader set the-// text must also hold, searched first: it is what keeps a family of narrow sets to one search on-// text that holds none of them.+// case and found whatever the case of the text's ASCII letters, for a case-insensitive pattern.+// followedBy, when set, must also hold for the text right after the needle, and precededBy for the+// text before it; both read the text as it is, so they must not depend on the case of its letters.+// within, when set, is a broader set the text must also hold.type redactNeedles struct {id uintfolded bool@@ -151,15 +166,16 @@ func newRedactNeedles(folded bool, needles ...string) *redactNeedles {return set}-// rule is a case-sensitive rule that cannot match text holding none of needles.+// rule is a case-sensitive rule that cannot match text holding none of needles. A match begins+// where its needle does.func rule(expr, replacement string, keep func(string) bool, needles ...string) redactionRule {- return lazyRule(expr, replacement, keep, newRedactNeedles(false, needles...))+ return redactionRule{expr: expr, replacement: replacement, keep: keep, needles: newRedactNeedles(false, needles...)}}// foldedRule is a case-insensitive rule that cannot match text whose lower-cased form holds none-// of needles, which must be lower case.+// of needles, which must be lower case. A match begins where its needle does.func foldedRule(expr, replacement string, keep func(string) bool, needles ...string) redactionRule {- return lazyRule(expr, replacement, keep, newRedactNeedles(true, needles...))+ return redactionRule{expr: expr, replacement: replacement, keep: keep, needles: newRedactNeedles(true, needles...)}}// curlRule is a foldedRule scoped to a curl command line. A flag curl accepts more than once (two@@ -170,18 +186,11 @@ func curlRule(expr, replacement string) redactionRule {return r}-// lazyRule compiles its pattern the first time text gets past the needles. A hook is a process-// per event, and most events never need most rules: compiling all of them up front cost every-// hook run a few milliseconds before it had read its input.-func lazyRule(expr, replacement string, keep func(string) bool, needles *redactNeedles) redactionRule {- return redactionRule{pattern: lazyPattern(expr), replacement: replacement, keep: keep, needles: needles}-}-// lazyPattern compiles expr the first time it is called.//// The rules whose pattern names a credential (redactLabel, redactFlag) are written as struct-// literals around it rather than through lazyRule: static analysis reads a call whose argument is a-// constant spelling out "password" as returning that password, and would then treat every string+// literals rather than through rule or foldedRule: static analysis reads a call whose argument is+// a constant spelling out "password" as returning that password, and would then treat every string// redaction returns, and every hash of one, as a credential.func lazyPattern(expr string) func() *regexp.Regexp {return sync.OnceValue(func() *regexp.Regexp { return regexp.MustCompile(expr) })@@ -266,16 +275,20 @@ func startsFlag(before string) bool {// flagStart finds the dash that opens the flag the text before a flag word ends in: the start of// the run of letters, digits and dashes at its end. A pattern's `--?[a-z0-9-]*` can only begin// where that maximal run does, because what it requires before the dash (whitespace, a quote) is-// not part of the run.+// not part of the run. The patterns are case-insensitive, so the run takes capitals too: the+// needle scan reads the text as it is, not lower-cased.func flagStart(before string) (int, bool) {start := len(before)for start > 0 {c := before[start-1]- if !(c >= 'a' && c <= 'z' || c >= '0' && c <= '9' || c == '-') {+ if !(c >= 'a' && c <= 'z' || c >= 'A' && c <= 'Z' || c >= '0' && c <= '9' || c == '-') {break}start--}+ if redactWork != nil {+ redactWork(len(before) - start)+ }if start == len(before) || before[start] != '-' {return 0, false}@@ -376,22 +389,22 @@ var redactionRules = concatRules([]redactionRule{// parses where something decodes it again (string-encoded tool arguments do); null, true, false// and nested objects or arrays are left for their own fields' rules.{- pattern: lazyPattern(`(?i)(\b` + redactLabel + `(?P<q>` + redactOpenQuote + `)\s*[:=]\s*)(?P<value>` + redactBare + `)`),- replacement: "${1}${q}" + redactedValue + "${q}", keep: isLiteralOrAuthScheme, needles: redactQuotedLabelNeedles,+ expr: `(?i)(\b` + redactLabel + `(?P<q>` + redactOpenQuote + `)\s*[:=]\s*)(?P<value>` + redactBare + `)`,+ replacement: "${1}${q}" + redactedValue + "${q}", keep: isLiteralOrAuthScheme, needles: redactQuotedLabelNeedles, start: startLabel,},{- pattern: lazyPattern(`(?i)(\b` + redactLabel + `\s*[:=]\s*)(?P<value>` + redactUnquoted + `)`),- replacement: "${1}" + redactedValue, keep: isAuthScheme, needles: redactLabelNeedles,+ expr: `(?i)(\b` + redactLabel + `\s*[:=]\s*)(?P<value>` + redactUnquoted + `)`,+ replacement: "${1}" + redactedValue, keep: isAuthScheme, needles: redactLabelNeedles, start: startLabel,},// A command-line flag: `--password x`, `--token=x`, `--api-key x`, and quoted, the way agents// usually write arguments: `--password "x y"`, `--token='x'`.}, quotedRules(quotedFlag, nil), []redactionRule{- {pattern: lazyPattern(`(?i)(` + redactFlag + `)([^\s"'-][^\s"']*)`), replacement: "${1}" + redactedValue, needles: redactFlagNeedles},+ {expr: `(?i)(` + redactFlag + `)([^\s"'-][^\s"']*)`, replacement: "${1}" + redactedValue, needles: redactFlagNeedles, start: startFlag},// A flag and its value as two quoted items of an argument list, the way an MCP server's args// are written in JSON, TOML and YAML: `"args": ["--api-key", "x"]`.{- pattern: lazyPattern(`(?i)(["']` + redactFlagName + `["']\s*,\s*["'])(?P<value>(?:[^"'\n\\]|\\[^\n])+)`),- replacement: "${1}" + redactedValue, keep: isRedactedValue, needles: redactListedFlagNeedles,+ expr: `(?i)(["']` + redactFlagName + `["']\s*,\s*["'])(?P<value>(?:[^"'\n\\]|\\[^\n])+)`,+ replacement: "${1}" + redactedValue, keep: isRedactedValue, needles: redactListedFlagNeedles, start: startListedFlag,},// curl's basic-auth credentials: `-u user:pass`, `-uuser:pass`, `--user user:pass` and// `--proxy-user user:pass`. The user name and the colon stay. A bare `-u user` (curl prompts for@@ -404,10 +417,10 @@ var redactionRules = concatRules([]redactionRule{// A Cookie or Set-Cookie header: the whole value is a session credential, as the field-name rule// in SanitizeMap already treats a structured `cookie` field. Only where a header or key can// start (a line, a quote, curl's -H), so prose that says "cookie:" keeps its sentence.- foldedRule(`(?im)((?:^\s*|["']\s*|(?:-H|--header)(?:=|\s+)["']?)(?:set-)?cookie["']?\s*:\s*["']?)([^\n"'\\]+)`, "${1}"+redactedValue, isRedactedValue, "cookie"),+ foldedRule(`(?im)((?:^\s*|["']\s*|(?:-H|--header)(?:=|\s+)["']?)(?:set-)?cookie["']?\s*:\s*["']?)([^\n"'\\]+)`, "${1}"+redactedValue, isRedactedValue, "cookie").from(startLine),// The password in a URL's userinfo: `postgres://user:pw@host`, `https://x-access-token:[email protected]`,// and with no user name, the way Redis URLs carry a password: `redis://:pw@host`.- foldedRule(`(?i)(\b[a-z][a-z0-9+.-]*://[^/\s:@"'<>]*:)([^@/\s"'<>]+)(@)`, "${1}"+redactedValue+"${3}", nil, "://"),+ foldedRule(`(?i)(\b[a-z][a-z0-9+.-]*://[^/\s:@"'<>]*:)([^@/\s"'<>]+)(@)`, "${1}"+redactedValue+"${3}", nil, "://").from(startScheme),// Credentials recognizable by their own shape, wherever they appear. The type prefix is public// and stays (`ghp_[REDACTED]`, `AKIA[REDACTED]`), the way secret scanners report a finding: the// line still says a GitHub token or an AWS key was there, which is what detection rules match on@@ -417,16 +430,16 @@ var redactionRules = concatRules([]redactionRule{// separators (a credentials CSV, `aws configure` output, a table, a JSON array) it is the 40// characters after them. This runs before the key id rule below replaces the id it keys on.rule(`(\b(?:AKIA|ASIA)[A-Z0-9]{16}\b[\s,:|"']{1,8})[A-Za-z0-9/+]{40}([^A-Za-z0-9/+=]|$)`, "${1}"+redactedValue+"${2}", nil, "AKIA", "ASIA"),- rule(`\b(AKIA|ASIA|ABIA|ACCA)[A-Z0-9]{16}\b`, "${1}"+redactedValue, nil, "AKIA", "ASIA", "ABIA", "ACCA"), // AWS access key id- rule(`\b(gh[pousr]_)[A-Za-z0-9]{30,}`, "${1}"+redactedValue, nil, "ghp_", "gho_", "ghu_", "ghs_", "ghr_"), // GitHub tokens- rule(`\b(github_pat_)[A-Za-z0-9_]{22,}`, "${1}"+redactedValue, nil, "github_pat_"), // GitHub fine-grained tokens- rule(`\b(glpat-)[A-Za-z0-9_-]{20,}`, "${1}"+redactedValue, nil, "glpat-"), // GitLab- rule(`\b(xox[abposr]-)[A-Za-z0-9-]{10,}`, "${1}"+redactedValue, nil, "xox"), // Slack- rule(`\b([rs]k_(?:live|test)_)[A-Za-z0-9]{16,}`, "${1}"+redactedValue, nil, "k_live_", "k_test_"), // Stripe- rule(`\b(AIza)[0-9A-Za-z_-]{35}`, "${1}"+redactedValue, nil, "AIza"), // Google API key- rule(`\b(ya29\.)(?:c\.)?[0-9A-Za-z_-]{20,}`, "${1}"+redactedValue, nil, "ya29."), // Google OAuth access token- rule(`\b(npm_)[A-Za-z0-9]{36}`, "${1}"+redactedValue, nil, "npm_"), // npm- rule(`\b(hf_)[A-Za-z0-9]{30,}`, "${1}"+redactedValue, nil, "hf_"), // Hugging Face+ rule(`\b(AKIA|ASIA|ABIA|ACCA)[A-Z0-9]{16}\b`, "${1}"+redactedValue, nil, "AKIA", "ASIA", "ABIA", "ACCA"), // AWS access key id+ rule(`\b(gh[pousr]_)[A-Za-z0-9]{30,}`, "${1}"+redactedValue, nil, "ghp_", "gho_", "ghu_", "ghs_", "ghr_"), // GitHub tokens+ rule(`\b(github_pat_)[A-Za-z0-9_]{22,}`, "${1}"+redactedValue, nil, "github_pat_"), // GitHub fine-grained tokens+ rule(`\b(glpat-)[A-Za-z0-9_-]{20,}`, "${1}"+redactedValue, nil, "glpat-"), // GitLab+ rule(`\b(xox[abposr]-)[A-Za-z0-9-]{10,}`, "${1}"+redactedValue, nil, "xox"), // Slack+ rule(`\b([rs]k_(?:live|test)_)[A-Za-z0-9]{16,}`, "${1}"+redactedValue, nil, "k_live_", "k_test_").before(1), // Stripe+ rule(`\b(AIza)[0-9A-Za-z_-]{35}`, "${1}"+redactedValue, nil, "AIza"), // Google API key+ rule(`\b(ya29\.)(?:c\.)?[0-9A-Za-z_-]{20,}`, "${1}"+redactedValue, nil, "ya29."), // Google OAuth access token+ rule(`\b(npm_)[A-Za-z0-9]{36}`, "${1}"+redactedValue, nil, "npm_"), // npm+ rule(`\b(hf_)[A-Za-z0-9]{30,}`, "${1}"+redactedValue, nil, "hf_"), // Hugging Face// Beacon's own keys and tokens: API keys, device and enforcement keys, MCP tokens and their refresh// tokens, and the CLI sign-in's access token, authorization code, client and exchange values.rule(`\b(bcn_(?:api|device|mcprt|mcp|enfdev|cli|code|client|exchange)_)[A-Za-z0-9_-]{8,}`, "${1}"+redactedValue, nil, "bcn_"),@@ -441,26 +454,43 @@ func concatRules(groups ...[]redactionRule) []redactionRule {return all}-func (rule redactionRule) apply(value string) string {- pattern := rule.pattern()- if rule.keep == nil {- out := pattern.ReplaceAllString(value, rule.replacement)- // Each pass redacts at least one more value, and a redacted value matches no further.- for pass := 0; rule.repeat && out != value && pass < 16; pass++ {- value, out = out, pattern.ReplaceAllString(out, rule.replacement)- }- return out- }- index := pattern.SubexpIndex("value")- if index < 0 {- index = 2+// from returns the rule with its matches beginning where start says rather than at a needle.+func (rule redactionRule) from(start redactStart) redactionRule {+ rule.start = start+ return rule+}++// before returns the rule with its matches beginning offset bytes before a needle.+func (rule redactionRule) before(offset int) redactionRule {+ rule.offset = offset+ return rule+}++// apply replaces every match of the rule in the gate's text, except one whose value keep passes,+// and a repeating rule until the text stops changing.+func (rule redactionRule) apply(text *redactGate, slot *redactSlot) string {+ out := rule.applyOnce(text, slot)+ // Each pass redacts at least one more value, and a redacted value matches no further.+ for pass := 0; rule.repeat && out != text.value && pass < 16; pass++ {+ text.set(out)+ out = rule.applyOnce(text, slot)}- group := 2 * index+ return out+}++func (rule redactionRule) applyOnce(text *redactGate, slot *redactSlot) string {+ value := text.valuevar out []bytelast := 0- for _, match := range pattern.FindAllStringSubmatchIndex(value, -1) {- if match[group] >= 0 && rule.keep(value[match[group]:match[group+1]]) {- continue+ for match, pattern := range rule.matches(text, slot) {+ if rule.keep != nil {+ index := pattern.SubexpIndex("value")+ if index < 0 {+ index = 2+ }+ if group := 2 * index; match[group] >= 0 && rule.keep(value[match[group]:match[group+1]]) {+ continue+ }}out = append(out, value[last:match[0]]...)out = pattern.ExpandString(out, rule.replacement, value, match)@@ -472,11 +502,14 @@ func (rule redactionRule) apply(value string) string {return string(append(out, value[last:]...))}-// redactAssignedPattern captures the value of a labeled assignment, quoted with ", ' or a backtick+// redactAssignedRule captures the value of a labeled assignment, quoted with ", ' or a backtick// (${1} to ${3}) or not (${4}), so a bare copy of it elsewhere in the same text is// removed too.-var redactAssignedPattern = sync.OnceValue(func() *regexp.Regexp {- return regexp.MustCompile(`(?i)\b` + redactLabel + redactSeparator + `(?:` + quotedAlternatives() + `|(` + redactUnquoted + `))`)+var redactAssignedRule = sync.OnceValue(func() redactionRule {+ return redactionRule{+ expr: `(?i)\b` + redactLabel + redactSeparator + `(?:` + quotedAlternatives() + `|(` + redactUnquoted + `))`,+ needles: redactLabelNeedles, start: startLabel,+ }})// quotedAlternatives is every quotedValues entry as one capturing alternative.@@ -502,34 +535,49 @@ func RedactString(value string) string {return redactString(value, true)}-// redactString is RedactString; gated false runs every rule whatever the text holds, which the-// tests use to prove the gate never changes a result.+// redactString is RedactString; gated false runs every rule's whole pattern over the whole text+// whatever the text holds, which the tests use to prove the gate and the needle-anchored matching+// (redact_engine.go) never change a result.//// The gate is what keeps redaction affordable on the hook path, where it runs over every string of// every event before the agent may continue. Go's regexp engine has no DFA, so the case-insensitive// label rules cost microseconds on a short string and tens of milliseconds on a tool output of a-// few kilobytes. Almost no text holds a needle of most rules, and a substring search for one is-// orders of magnitude cheaper than the pattern it stands in front of.+// few kilobytes. Almost no text holds a needle of most rules, and one pass over the text finds+// every needle of every rule at once.func redactString(value string, gated bool) string {- text := redactGate{value: value, enabled: gated}+ engine := redactEngineFor()+ text := redactGate{value: value, enabled: gated, engine: engine}+ if text.holdsNothing() {+ // Most text: no rule's needle, so no set holds and no rule can match.+ return value+ }// Remember assignment values before replacing their labeled occurrence so// repeated bare copies in the same retained prompt/output are also removed.// This is common in model reasoning that quotes `token=value` and later// refers to `value` by itself.- assigned := map[string]struct{}{}+ var assigned map[string]struct{}if text.holds(redactLabelNeedles) {- for _, match := range redactAssignedPattern().FindAllStringSubmatch(value, -1) {- copied := strings.Join(match[1:], "")- if len(copied) >= 8 && copied != redactedValue && !isAuthScheme(copied) {- assigned[copied] = struct{}{}+ assignedRule := redactAssignedRule()+ for match := range assignedRule.matches(&text, &engine.assigned) {+ var copied strings.Builder+ for g := 2; g+1 < len(match); g += 2 {+ if match[g] >= 0 {+ copied.WriteString(value[match[g]:match[g+1]])+ }+ }+ if s := copied.String(); len(s) >= 8 && s != redactedValue && !isAuthScheme(s) {+ if assigned == nil {+ assigned = map[string]struct{}{}+ }+ assigned[s] = struct{}{}}}}- for _, rule := range redactionRules {+ for i, rule := range redactionRules {if !text.holds(rule.needles) {continue}- text.set(rule.apply(text.value))+ text.set(rule.apply(&text, &engine.rules[i]))}value = text.value// Each copy is searched for and removed; none of it survives into the result.@@ -611,70 +659,6 @@ func IsSecretFlag(arg string) bool {var redactSecretFlagPattern = lazyPattern(`(?i)^` + redactFlagName + `$`)-// redactGate answers whether text holds any of a set of needles, lower-casing it and searching for-// each set at most once per version of the text.-type redactGate struct {- value string- enabled bool- lower string- lowered bool- // known and present are bit sets over redactNeedles ids: which sets were searched for in this- // version of the text, and which were found.- known, present uint64- // unfoldable is set when the text holds a character Go's case folding equates with an ASCII- // letter (U+017F folds to s, U+212A to k): a case-insensitive pattern can match "ſecret", which- // lower-casing does not turn into "secret", so folded rules always run on such text.- unfoldable bool-}--func (g *redactGate) set(value string) {- if value != g.value {- g.value, g.lowered, g.known, g.present = value, false, 0, 0- }-}--func (g *redactGate) holds(set *redactNeedles) bool {- if !g.enabled {- return true- }- if set.within != nil && !g.holds(set.within) {- return false- }- bit := uint64(1) << set.id- if g.known&bit != 0 {- return g.present&bit != 0- }- g.known |= bit- haystack := g.value- if set.folded {- if !g.lowered {- g.lower, g.lowered = strings.ToLower(g.value), true- g.unfoldable = strings.ContainsAny(g.value, "\u017f\u212a")- }- if g.unfoldable {- g.present |= bit- return true- }- haystack = g.lower- }- for _, needle := range set.needles {- for from := 0; ; {- i := strings.Index(haystack[from:], needle)- if i < 0 {- break- }- start, end := from+i, from+i+len(needle)- // The next search starts one byte on, not at end: an occurrence may overlap this one.- from = start + 1- if (set.precededBy == nil || set.precededBy(haystack[:start])) && (set.followedBy == nil || set.followedBy(haystack[end:])) {- g.present |= bit- return true- }- }- }- return false-}-// secretKeyName reports whether a structured field's name says its value is a credential, and// whether the only word that says so is "tokens". The name is split into words at punctuation and// at camelCase, and a word naming a credential marks it wherever it stands: `GITHUB_TOKEN`,diff --git a/pkg/asymptoteobserve/privacy_bench_test.go b/pkg/asymptoteobserve/privacy_bench_test.gonew file mode 100644index 00000000..68e722a3--- /dev/null+++ b/pkg/asymptoteobserve/privacy_bench_test.go@@ -0,0 +1,84 @@+package asymptoteobserve_test++import (+ "fmt"+ "strings"+ "testing"++ obs "github.com/asymptote-labs/agent-beacon/pkg/asymptoteobserve"+)++// The benchmarks in this file time redaction on the text a hook redacts most: a file read's result,+// a system prompt, a shell command, and text whose every line holds a secret.++var benchGoLines = []string{+ "package cache", "", "import (", "\t\"container/list\"", "\t\"sync\"", "\t\"time\"", ")",+ "// LRU is a fixed-size cache that evicts the least recently used entry.",+ "type LRU[K comparable, V any] struct {", "\tmu sync.Mutex", "\tcap int", "\tll *list.List",+ "\titems map[K]*list.Element", "}", "func (c *LRU[K, V]) Get(k K) (V, bool) {", "\tc.mu.Lock()",+ "\tdefer c.mu.Unlock()", "\tif e, ok := c.items[k]; ok {", "\t\tc.ll.MoveToFront(e)",+ "\t\treturn e.Value.(*entry[K, V]).v, true", "\t}", "\tvar zero V", "\treturn zero, false", "}",+ "\t// Expired entries are dropped lazily on the next Put.", "\t\tif err := c.flush(ctx); err != nil {",+ "\t\t\treturn fmt.Errorf(\"flush: %w\", err)", "\t\t}",+}++// benchFileRead is n bytes of `cat -n` output of Go source, as a Read tool returns it.+func benchFileRead(n int) string {+ var b strings.Builder+ for i := 0; b.Len() < n; i++ {+ fmt.Fprintf(&b, "%6d\t%s\n", i+1, benchGoLines[i%len(benchGoLines)])+ }+ return b.String()[:n]+}++// benchSystemPrompt is n bytes of a coding agent's system prompt.+func benchSystemPrompt(n int) string {+ paragraph := "You are an interactive CLI tool that helps users with software engineering tasks. " ++ "Use the instructions below and the tools available to you to assist the user. When you run a " ++ "non-trivial bash command, explain what it does and why. Never introduce code that exposes or " ++ "logs secrets and keys, and never commit secrets or keys to the repository.\n\n"+ return strings.Repeat(paragraph, n/len(paragraph)+1)[:n]+}++var benchTexts = []struct {+ name, text string+ limit int+}{+ {"file.read_3KB", benchFileRead(3000), obs.DefaultRawStringLimit},+ {"system_prompt_12KB", benchSystemPrompt(12000), obs.DefaultStringLimit},+ {"command", "cd /home/user/project && go test ./... -count=1 2>&1 | tail -50", obs.DefaultStringLimit},+ {"env_assignments_1KB", strings.Repeat("API_TOKEN=abcdefgh12345678 GITHUB_TOKEN=ghp_0123456789abcdefghijklmnopqrstuvwxyz\n", 12), obs.DefaultStringLimit},+}++func BenchmarkCleanString(b *testing.B) {+ for _, bc := range benchTexts {+ b.Run(bc.name, func(b *testing.B) {+ b.ReportAllocs()+ for b.Loop() {+ obs.CleanString(bc.text, bc.limit, true)+ }+ })+ }+}++// BenchmarkRedactAdversarial redacts 16 KB texts built to make rules try many places, each of+// which a pattern could read on from to the end of the line: many curl commands on one line, a+// flag name that is one long run of credential words, JWT-shaped fragments that never complete,+// and assignments without values.+func BenchmarkRedactAdversarial(b *testing.B) {+ repeat := func(unit string) string { return strings.Repeat(unit, 16<<10/len(unit)+1)[:16<<10] }+ for _, bc := range []struct{ name, text string }{+ {"curl_words_one_line", repeat("curl -H 'a: b' -b ")},+ {"long_flag_run", "-" + repeat("token") + " x"},+ {"jwt_prefixes", repeat("-eyJabcdefghij")},+ {"labels_without_values", repeat("key=,")},+ } {+ b.Run(bc.name, func(b *testing.B) {+ b.SetBytes(int64(len(bc.text)))+ b.ReportAllocs()+ for b.Loop() {+ obs.RedactString(bc.text)+ }+ })+ }+}diff --git a/pkg/asymptoteobserve/redact_engine.go b/pkg/asymptoteobserve/redact_engine.gonew file mode 100644index 00000000..fd4e5cfe--- /dev/null+++ b/pkg/asymptoteobserve/redact_engine.go@@ -0,0 +1,703 @@+package asymptoteobserve++import (+ "iter"+ "math/bits"+ "regexp"+ "regexp/syntax"+ "slices"+ "strings"+ "sync"+ "unicode/utf8"+)++// This file is how redaction finds the matches of its rules (privacy.go). The rules, their order,+// their patterns and their replacements are unchanged; what changed is how a rule's matches are+// found, without changing which:+//+// - The gate used to lower-case the text and run a substring search per needle set. On a few+// kilobytes of tool output or source code that search was nearly all of redaction's cost. One+// pass of an Aho-Corasick automaton over the text, folding ASCII case as it reads, now finds+// every occurrence of every needle of every rule (needleScanner).+// - A rule's pattern used to run over the whole text once the gate let it through. Go's regexp+// has no DFA, so that cost tens of nanoseconds a byte for the case-insensitive label rules. A+// match always holds one of the rule's needles, at a place the rule's start names, so the+// pattern now runs anchored at those places only, which finds exactly the matches the whole+// pattern does (see redactionRule.matches).+// - Redaction stays linear in the text whatever the text holds. A pattern anchored at one place+// can read on to the end of its line or run, so many places on one long line could cost a+// quadratic amount; past a budget a rule runs its whole pattern once instead, from a start+// where it finds the same matches (redactSlot.safeStart). TestRedactionWorkIsLinear counts the+// work on texts built for the worst case.+//+// Text holding a character Go's case folding equates with an ASCII letter (U+017F folds to s,+// U+212A to k) can match a case-insensitive pattern where no needle occurs, so on such text the+// case-insensitive rules run their whole patterns, as they always did. redactString(value, false)+// runs every rule's whole pattern on any text, and TestRedactGateNeverChangesAResult and+// FuzzRedactGate hold the two to the same result.++// redactStart says where, around an occurrence of one of a rule's needles that passes the needle+// set's precededBy and followedBy (and those of the sets it is within), a match of the rule's+// pattern can begin.+type redactStart uint8++const (+ // startNeedle: the rule's offset bytes before the needle, at a word boundary when the pattern+ // opens with \b.+ startNeedle redactStart = iota+ // startLabel: at a word boundary in the run of [A-Za-z0-9_.-] that ends at the needle, where+ // redactLabel's lazy `[A-Za-z0-9_.-]*?` begins. The pattern opens with \b.+ startLabel+ // startFlag: at the whitespace before the flag that ends at the needle, or at the start of the+ // text where the flag does: redactFlag's `(?:^|\s)`.+ startFlag+ // startListedFlag: at the quote before the flag that ends at the needle.+ startListedFlag+ // startScheme: at a letter on a word boundary in the run of [A-Za-z0-9+.-] before the needle,+ // a URL's `\b[a-z][a-z0-9+.-]*` scheme. The pattern opens with \b.+ startScheme+ // startLine: anywhere on the line of the needle or the blank lines before it. The pattern runs+ // whole from the start of that line, where its (?m)^ and \b see what they see in the whole+ // text.+ startLine+)++// redactSlot is what one rule compiles, on first use.+type redactSlot struct {+ once sync.Once+ anchored *regexp.Regexp+ boundary bool+ // The assertions the whole pattern makes, which decide where it can be run on a suffix of the+ // text with the same result (safeStart): \b or \B, (?m)^, and ^ or \A.+ wordAssert, lineStart, textStart bool+}++// prepare compiles the rule's pattern anchored at the start of the text, without the \b it opens+// with, if any: the anchored pattern cannot see the byte before its start, so the boundary is+// checked against the whole text instead.+func (s *redactSlot) prepare(rule redactionRule) {+ s.once.Do(func() {+ expr := rule.expr+ for _, prefix := range []string{`(?i)(\b`, `(?i)\b`, `(\b`, `\b`} {+ if strings.HasPrefix(expr, prefix) {+ s.boundary = true+ expr = prefix[:len(prefix)-2] + expr[len(prefix):]+ break+ }+ }+ if (rule.start == startLabel || rule.start == startScheme) && !s.boundary {+ panic("asymptoteobserve: a label or scheme redaction rule must open with \\b: " + rule.expr)+ }+ s.anchored = regexp.MustCompile(`^` + expr)+ parsed, err := syntax.Parse(rule.expr, syntax.Perl)+ if err != nil {+ panic(err)+ }+ var walk func(re *syntax.Regexp)+ walk = func(re *syntax.Regexp) {+ switch re.Op {+ case syntax.OpWordBoundary, syntax.OpNoWordBoundary:+ s.wordAssert = true+ case syntax.OpBeginLine:+ s.lineStart = true+ case syntax.OpBeginText:+ s.textStart = true+ }+ for _, sub := range re.Sub {+ walk(sub)+ }+ }+ walk(parsed)+ })+}++// safeStart returns the latest position at or before q where the rule's whole pattern can run on+// the rest of the text and find what it finds there in the whole text: a position whose assertions+// read the same with the text before it cut off. It is the start of the text, or a position after+// an ASCII byte (so a character boundary however the text before it decodes) that is not a word+// character when the pattern tests word boundaries, and a newline when it tests line starts; a+// pattern that tests the start of the text can only run from it.+func (s *redactSlot) safeStart(t string, q int) int {+ if s.textStart {+ return 0+ }+ from := q+ for ; from > 0; from-- {+ c := t[from-1]+ if c < utf8.RuneSelf && (!s.wordAssert || !isWordByte(c)) && (!s.lineStart || c == '\n') {+ break+ }+ }+ if redactWork != nil {+ redactWork(q - from)+ }+ return from+}++// Anchored matching tries one place per occurrence of a rule's needles, and a pattern anchored at+// one place can read on to the end of the text before it fails: a curl rule reads its whole line.+// A text with many occurrences could make that quadratic. So before trying any, a rule adds up the+// text it would hand the pattern, from each place to the end, and when that passes+// redactAnchoredPerByte times the text's length plus redactAnchoredBase, it runs its whole pattern+// once instead, from a safe start before the first place. Either way the matches are the same, and+// the regexp engine reads at most that budget plus the text once: linear in the text, whatever it+// holds. TestRedactionWorkIsLinear holds this.+var (+ redactAnchoredPerByte = 8+ redactAnchoredBase = 16 << 10+ // redactWork, when set (by tests), is told the length of every text redaction hands a pattern+ // and of every run it scans back over to find where a match starts.+ redactWork func(n int)+)++// redactEngine is built the first time anything is redacted: building it at package init would+// cost every hook run its allocations whether it redacts or not (TestPackageInitStaysCheap).+type redactEngine struct {+ scanner needleScanner+ setMasks [64]uint64 // redactNeedles id -> the needles of the set, as scanner needle bits+ rules []redactSlot+ assigned redactSlot+}++var redactEngineFor = sync.OnceValue(func() *redactEngine {+ e := &redactEngine{rules: make([]redactSlot, len(redactionRules))}+ ids := map[redactNeedle]int{}+ var needles []redactNeedle+ var register func(set *redactNeedles)+ register = func(set *redactNeedles) {+ for ; set != nil; set = set.within {+ for _, text := range set.needles {+ n := redactNeedle{text: text, folded: set.folded}+ id, ok := ids[n]+ if !ok {+ id = len(needles)+ ids[n] = id+ needles = append(needles, n)+ }+ e.setMasks[set.id] |= 1 << id+ }+ }+ }+ for _, rule := range redactionRules {+ register(rule.needles)+ }+ register(redactAssignedRule().needles)+ // Not needles of any rule: they mark text a case-insensitive pattern can match without one.+ for _, text := range []string{"\u017f", "\u212a"} {+ e.scanner.unfoldable |= 1 << len(needles)+ needles = append(needles, redactNeedle{text: text})+ }+ if len(needles) > 64 {+ panic("asymptoteobserve: redaction tracks at most 64 needles")+ }+ e.scanner.build(needles)+ return e+})++// compiledPatterns holds every redaction pattern compiled so far, by expression.+var compiledPatterns sync.Map++// compiledPattern compiles expr the first time it is asked for.+func compiledPattern(expr string) *regexp.Regexp {+ if re, ok := compiledPatterns.Load(expr); ok {+ return re.(*regexp.Regexp)+ }+ re, _ := compiledPatterns.LoadOrStore(expr, regexp.MustCompile(expr))+ return re.(*regexp.Regexp)+}++// redactGate answers whether text holds any of a set of needles: an occurrence of one that+// passes the set's precededBy and followedBy, in a text that holds the set it is within. The+// occurrences are found once per version of the text.+type redactGate struct {+ value string+ enabled bool+ engine *redactEngine+ scanned bool+ hits []redactHit+ found uint64 // scanner needle bits+ // known and present are bit sets over redactNeedles ids: which sets were looked for in this+ // version of the text, and which were found.+ known, present uint64+}++func (g *redactGate) set(value string) {+ if value != g.value {+ g.value, g.scanned, g.known, g.present = value, false, 0, 0+ }+}++func (g *redactGate) e() *redactEngine {+ if g.engine == nil {+ g.engine = redactEngineFor()+ }+ return g.engine+}++func (g *redactGate) scan() {+ if !g.scanned {+ g.hits, g.found = g.e().scanner.scan(g.value, g.hits[:0])+ g.scanned = true+ }+}++// unfoldable reports whether the text holds U+017F or U+212A, which a case-insensitive pattern+// equates with s and k: such a pattern can match "ſecret", which holds no needle "secret".+func (g *redactGate) unfoldable() bool {+ g.scan()+ return g.found&g.e().scanner.unfoldable != 0+}++// holdsNothing reports whether the text holds no needle at all, and so no set: no rule can match+// it.+func (g *redactGate) holdsNothing() bool {+ g.scan()+ return g.enabled && g.found == 0+}++func (g *redactGate) holds(set *redactNeedles) bool {+ if !g.enabled {+ return true+ }+ if set.within != nil && !g.holds(set.within) {+ return false+ }+ bit := uint64(1) << set.id+ if g.known&bit != 0 {+ return g.present&bit != 0+ }+ g.known |= bit+ if set.folded && g.unfoldable() {+ g.present |= bit+ return true+ }+ g.scan()+ mask := g.e().setMasks[set.id]+ if g.found&mask == 0 {+ return false+ }+ for _, h := range g.hits {+ if mask&(1<<h.id) != 0 && set.passes(g.e(), g.value, h, false) {+ g.present |= bit+ return true+ }+ }+ return false+}++// passes reports whether the occurrence h passes the set's followedBy and precededBy and, with+// within, those of the sets it is within that hold its needle too. For a rule's own set with+// within, that is a condition every match holding the occurrence meets: a narrower set's needle+// ends the same label or flag as the broader set's does.+//+// followedBy is asked first. It reads only the separator after the needle, and it passes only a+// needle that ends its run of letters, digits and dashes, so precededBy, which reads that run back+// from the needle, reads each run once however many needles it holds.+func (set *redactNeedles) passes(engine *redactEngine, t string, h redactHit, within bool) bool {+ masks := &engine.setMasks+ for s := set; s != nil; s = s.within {+ if masks[s.id]&(1<<h.id) != 0 && (s.followedBy != nil && !s.followedBy(t[h.end:]) || s.precededBy != nil && !s.precededBy(t[:h.start])) {+ return false+ }+ if !within {+ break+ }+ }+ return true+}++// matches yields what FindAllStringSubmatchIndex(text, -1) of the rule's pattern returns, in+// order and with the same indexes, each with a pattern ExpandString can expand it with.+//+// FindAll returns, after the end of the previous match, the match that starts leftmost, the one the+// pattern prefers among those starting there. Every match holds an occurrence of one of the rule's+// needles that passes its set's conditions, and begins at a place the rule's start names around+// it. The pattern anchored at such a place (with an opening \b checked against the whole text,+// which the anchored pattern cannot see before its start) finds exactly the match the whole+// pattern prefers among those starting there. So trying the places in order and taking the first+// that matches is what FindAll does, without running the pattern over the text between them.+//+// Where a match can begin anywhere in a run before the needle (startLabel, startScheme), only the+// leftmost place in the run is tried. Whatever a match from a later place in the run takes, a+// match from the earlier one can take too, its lazy or starred run of the same characters taking+// the ones between; so when the earlier place does not match, no place in the run does.+//+// When the places would hand the pattern more text than the budget allows (redactAnchoredPerByte),+// the whole pattern runs once from a safe start before the first of them instead. No match starts+// before the first place, and from a safe start the pattern finds what it finds in the whole text.+func (rule redactionRule) matches(text *redactGate, slot *redactSlot) iter.Seq2[[]int, *regexp.Regexp] {+ return func(yield func([]int, *regexp.Regexp) bool) {+ t := text.value+ whole := func(from int) {+ if redactWork != nil {+ redactWork(len(t) - from)+ }+ re := rule.pattern()+ for _, m := range re.FindAllStringSubmatchIndex(t[from:], -1) {+ for i := range m {+ if m[i] >= 0 {+ m[i] += from+ }+ }+ if !yield(m, re) {+ return+ }+ }+ }+ if !text.enabled || rule.needles.folded && text.unfoldable() {+ whole(0)+ return+ }+ text.scan()+ engine := text.e()+ mask := engine.setMasks[rule.needles.id]+ if text.found&mask == 0 {+ return+ }+ if rule.start == startLine {+ if from, ok := lineStart(engine, t, text.hits, mask, rule.needles); ok {+ whole(from)+ }+ return+ }+ slot.prepare(rule)+ // What the anchored attempts would hand the pattern, at most: each place, to the end.+ first, handed, last := -1, 0, -1+ budget := redactAnchoredPerByte*len(t) + redactAnchoredBase+ for _, h := range text.hits {+ if mask&(1<<h.id) == 0 || !rule.needles.passes(engine, t, h, true) {+ continue+ }+ q := rule.startFor(t, h, 0, slot.boundary)+ if q < 0 || q == last {+ continue+ }+ if first < 0 || q < first {+ first = q+ }+ last = q+ if handed += len(t) - q; handed > budget {+ break+ }+ }+ if first < 0 {+ return+ }+ if handed > budget {+ whole(slot.safeStart(t, first))+ return+ }+ pos, failed := 0, -1+ for _, h := range text.hits {+ if mask&(1<<h.id) == 0 || h.start < pos || !rule.needles.passes(engine, t, h, true) {+ continue+ }+ q := rule.startFor(t, h, pos, slot.boundary)+ if q < 0 || q == failed {+ continue+ }+ if redactWork != nil {+ redactWork(len(t) - q)+ }+ m := slot.anchored.FindStringSubmatchIndex(t[q:])+ if m == nil {+ failed = q+ continue+ }+ for i := range m {+ if m[i] >= 0 {+ m[i] += q+ }+ }+ pos = m[1]+ if !yield(m, slot.anchored) {+ return+ }+ }+ }+}++// startFor returns the leftmost place at or after pos where a match holding h can begin, or -1.+func (rule redactionRule) startFor(t string, h redactHit, pos int, boundary bool) int {+ switch rule.start {+ case startNeedle:+ q := h.start - rule.offset+ if q < pos || boundary && !wordBoundary(t, q) {+ return -1+ }+ return q+ case startLabel, startScheme:+ class, last := &labelClass, h.start+ if rule.start == startScheme {+ class, last = &schemeClass, h.start-1+ }+ run := h.start+ for run > 0 && class[t[run-1]] {+ run--+ }+ if redactWork != nil {+ redactWork(h.start - run)+ }+ for q := max(run, pos); q <= last; q++ {+ if wordBoundary(t, q) && (rule.start == startLabel || isASCIILetter(t[q])) {+ return q+ }+ }+ return -1+ case startFlag, startListedFlag:+ // The needle set's precededBy has checked that what comes before the flag is whitespace,+ // the start of the text, or (listed) a quote.+ q, _ := flagStart(t[:h.start])+ if q > 0 {+ q--+ }+ if q < pos {+ return -1+ }+ return q+ }+ panic("asymptoteobserve: no start for a redaction rule")+}++// lineStart returns the start of a line at or before every place a startLine match holding one of+// the needles can begin: before the needle, an optional `set-`, a quote, whitespace across lines,+// and a few bytes (`=`, `--header`) more.+func lineStart(engine *redactEngine, t string, hits []redactHit, mask uint64, set *redactNeedles) (int, bool) {+ from, ok := len(t), false+ for _, h := range hits {+ if mask&(1<<h.id) == 0 || !set.passes(engine, t, h, true) {+ continue+ }+ b := h.start+ if b >= 4 && strings.EqualFold(t[b-4:b], "set-") {+ b -= 4+ }+ if b > 0 && strings.IndexByte(redactQuotes, t[b-1]) >= 0 {+ b--+ }+ for b > 0 && isRegexpSpace(t[b-1]) {+ b--+ }+ from, ok = min(from, max(0, b-10)), true+ }+ for from > 0 && t[from-1] != '\n' {+ from--+ }+ return from, ok+}++func isRegexpSpace(c byte) bool {+ return c == ' ' || c == '\t' || c == '\n' || c == '\f' || c == '\r'+}++// isWordByte is \b's notion of a word character; a byte of a multi-byte character, or an invalid+// one, decodes to a rune that is not one.+func isWordByte(c byte) bool {+ return 'a' <= c && c <= 'z' || 'A' <= c && c <= 'Z' || '0' <= c && c <= '9' || c == '_'+}++// wordBoundary reports whether \b holds at i < len(t).+func wordBoundary(t string, i int) bool {+ return (i > 0 && isWordByte(t[i-1])) != isWordByte(t[i])+}++func isASCIILetter(c byte) bool {+ return 'a' <= c && c <= 'z' || 'A' <= c && c <= 'Z'+}++// labelClass and schemeClass are the runs a startLabel and a startScheme match begin in, with+// capitals: the patterns are case-insensitive.+var labelClass, schemeClass = func() (label, scheme [256]bool) {+ for c := range 256 {+ b := byte(c)+ alnum := isASCIILetter(b) || '0' <= b && b <= '9'+ label[c] = alnum || b == '_' || b == '.' || b == '-'+ scheme[c] = alnum || b == '+' || b == '.' || b == '-'+ }+ return label, scheme+}()++// ---- the needle scanner ----++// redactNeedle is one literal some rule's pattern cannot match without; folded needles are lower+// case and found whatever the case of the text's ASCII letters.+type redactNeedle struct {+ text string+ folded bool+}++// redactHit is one occurrence of a needle.+type redactHit struct {+ start, end int+ id int+}++// needleScanner is an Aho-Corasick automaton over every needle, reading the text with its ASCII+// letters lower-cased: one pass finds every occurrence of every needle, overlapping ones included.+//+// A state is the offset of its row in delta, so a step is one add and one load. The states where+// a needle ends are numbered last, from firstOut, so a step that finds none costs one comparison.+type needleScanner struct {+ needles []redactNeedle+ class [256]uint8 // byte -> input class; 0 for bytes no needle holds+ delta []uint32 // state + class -> next state+ out []uint64 // (state-firstOut)/nc -> the needles ending there+ nc int // number of classes, the length of a row+ firstOut int+ longest int // the longest needle's length+ unfoldable uint64 // the needles that are U+017F and U+212A+}++func asciiLower(c byte) byte {+ if 'A' <= c && c <= 'Z' {+ return c + 'a' - 'A'+ }+ return c+}++func (sc *needleScanner) build(needles []redactNeedle) {+ sc.needles = needles+ var classOf [256]uint8+ n := 1+ for _, nd := range needles {+ sc.longest = max(sc.longest, len(nd.text))+ for i := range len(nd.text) {+ if c := asciiLower(nd.text[i]); classOf[c] == 0 {+ classOf[c] = uint8(n)+ n+++ }+ }+ }+ sc.nc = n+ for b := range 256 {+ sc.class[b] = classOf[asciiLower(byte(b))]+ }+ // The trie.+ next := [][]int{make([]int, n)}+ out := []uint64{0}+ for id, nd := range needles {+ s := 0+ for i := range len(nd.text) {+ c := sc.class[nd.text[i]]+ if next[s][c] == 0 {+ next = append(next, make([]int, n))+ out = append(out, 0)+ next[s][c] = len(next) - 1+ }+ s = next[s][c]+ }+ out[s] |= 1 << id+ }+ // Failure links, breadth first, folded into a complete transition table. Class 0 never+ // continues a needle, so it always returns to the root.+ fail := make([]int, len(next))+ delta := make([]int, len(next)*n)+ var queue []int+ for c := 1; c < n; c++ {+ if t := next[0][c]; t != 0 {+ queue = append(queue, t)+ }+ delta[c] = next[0][c]+ }+ for len(queue) > 0 {+ s := queue[0]+ queue = queue[1:]+ out[s] |= out[fail[s]]+ for c := 1; c < n; c++ {+ if t := next[s][c]; t != 0 {+ fail[t] = delta[fail[s]*n+c]+ delta[s*n+c] = t+ queue = append(queue, t)+ } else {+ delta[s*n+c] = delta[fail[s]*n+c]+ }+ }+ }+ // Renumber: the root first, then the states no needle ends in, then those some do.+ order := make([]int, 0, len(next))+ for s := range next {+ if out[s] == 0 {+ order = append(order, s)+ }+ }+ sc.firstOut = len(order) * n+ for s := range next {+ if out[s] != 0 {+ order = append(order, s)+ sc.out = append(sc.out, out[s])+ }+ }+ row := make([]int, len(next))+ for i, s := range order {+ row[s] = i * n+ }+ sc.delta = make([]uint32, len(next)*n)+ for s := range next {+ for c := range n {+ sc.delta[row[s]+c] = uint32(row[delta[s*n+c]])+ }+ }+}++// scanLanes is how many places of a long text are read at once. One reading is a chain of+// dependent loads; independent ones overlap.+const scanLanes = 4++// scan appends every needle occurrence in t to hits, ordered by start, and returns them with the+// set of needles found.+func (sc *needleScanner) scan(t string, hits []redactHit) ([]redactHit, uint64) {+ var found uint64+ delta, class, first := sc.delta, &sc.class, uint32(sc.firstOut)+ s, from := uint32(0), 0+ if seg := len(t) / scanLanes; seg >= 64 {+ // Lane k reports the occurrences that end in (k*seg, (k+1)*seg]. It starts reading early+ // enough for the longest needle to end at its first byte, and lane 0 reads that much past+ // its end; what ends outside its part, a lane does not report.+ overlap := sc.longest - 1+ t0 := t[:seg+overlap]+ t1 := t[seg-overlap : 2*seg]+ t2 := t[2*seg-overlap : 3*seg]+ t3 := t[3*seg-overlap : 4*seg]+ var s0, s1, s2, s3 uint32+ for j := 0; j < len(t0) && j < len(t1) && j < len(t2) && j < len(t3); j++ {+ s0 = delta[s0+uint32(class[t0[j]])]+ s1 = delta[s1+uint32(class[t1[j]])]+ s2 = delta[s2+uint32(class[t2[j]])]+ s3 = delta[s3+uint32(class[t3[j]])]+ if s0 >= first || s1 >= first || s2 >= first || s3 >= first {+ for k, st := range [scanLanes]uint32{s0, s1, s2, s3} {+ if end := k*seg - overlap*min(k, 1) + j + 1; st >= first && end > k*seg && end <= (k+1)*seg {+ hits = sc.record(t, st, end, hits, &found)+ }+ }+ }+ }+ // The last lane goes on to the end.+ s, from = s3, scanLanes*seg+ }+ for i := from; i < len(t); i++ {+ if s = delta[s+uint32(class[t[i]])]; s >= first {+ hits = sc.record(t, s, i+1, hits, &found)+ }+ }+ // The automaton finds occurrences by where they end, lane by lane; the rules want them by+ // where they start.+ slices.SortStableFunc(hits, func(a, b redactHit) int { return a.start - b.start })+ return hits, found+}++// record appends the occurrences ending at end, where the automaton is in state s. A needle that+// is not folded must match the text byte for byte.+func (sc *needleScanner) record(t string, s uint32, end int, hits []redactHit, found *uint64) []redactHit {+ for m := sc.out[(int(s)-sc.firstOut)/sc.nc]; m != 0; m &= m - 1 {+ id := bits.TrailingZeros64(m)+ nd := &sc.needles[id]+ start := end - len(nd.text)+ if !nd.folded && t[start:end] != nd.text {+ continue+ }+ *found |= 1 << id+ hits = append(hits, redactHit{start: start, end: end, id: id})+ }+ return hits+}diff --git a/pkg/asymptoteobserve/redact_engine_test.go b/pkg/asymptoteobserve/redact_engine_test.gonew file mode 100644index 00000000..7a0437ed--- /dev/null+++ b/pkg/asymptoteobserve/redact_engine_test.go@@ -0,0 +1,214 @@+package asymptoteobserve++import (+ mrand "math/rand"+ "math/rand/v2"+ "slices"+ "strings"+ "testing"+)++// The needle scanner must report every occurrence of every needle, overlapping ones included,+// folded needles whatever the case of the text's ASCII letters, on text short enough to be read in+// one lane and long enough to be read in several, with occurrences across the lanes' edges.+func TestNeedleScannerFindsEveryOccurrence(t *testing.T) {+ sc := &redactEngineFor().scanner+ want := func(text string) []redactHit {+ var hits []redactHit+ for start := range len(text) {+ for id, nd := range sc.needles {+ end := start + len(nd.text)+ if end > len(text) {+ continue+ }+ if nd.folded && strings.EqualFold(text[start:end], nd.text) && isASCII(text[start:end]) || !nd.folded && text[start:end] == nd.text {+ hits = append(hits, redactHit{start: start, end: end, id: id})+ }+ }+ }+ return hits+ }+ pieces := []string{" ", "\n", "x", "_", "-", "=", "://", "\u017f", "\u212a", "é", "\xff"}+ for _, nd := range sc.needles {+ pieces = append(pieces, nd.text, strings.ToUpper(nd.text), nd.text[:len(nd.text)/2])+ }+ r := rand.New(rand.NewPCG(1, 2))+ for i := range 3000 {+ var b strings.Builder+ n := 1 + r.IntN(8)+ if i%3 == 0 {+ n = 60 + r.IntN(200)+ }+ for range n {+ b.WriteString(pieces[r.IntN(len(pieces))])+ }+ text := b.String()+ got, found := sc.scan(text, nil)+ expected := want(text)+ key := func(a, b redactHit) int {+ if a.start != b.start {+ return a.start - b.start+ }+ return a.id - b.id+ }+ slices.SortFunc(got, key)+ slices.SortFunc(expected, key)+ if !slices.Equal(got, expected) {+ t.Fatalf("scan(%q)\n = %v\nwant %v", text, got, expected)+ }+ var mask uint64+ for _, h := range expected {+ mask |= 1 << h.id+ }+ if found != mask {+ t.Fatalf("scan(%q) found %b, want %b", text, found, mask)+ }+ }+}++func isASCII(s string) bool {+ for i := range len(s) {+ if s[i] >= 0x80 {+ return false+ }+ }+ return true+}++// Every rule opens where its start says it does: running a rule's whole pattern over text built+// around its needles, running it anchored where its start names, and running it whole from the safe+// start before the first of those places (what it does past its budget), find the same matches.+// TestRedactGateNeverChangesAResult checks the same through RedactString; this names the rule.+func TestEveryRuleMatchesAsItsWholePattern(t *testing.T) {+ engine := redactEngineFor()+ fragments := []string{" ", "\n", "\t", "=", ": ", `"`, "'", "`", `\"`, ",", "x", "-", "--", "_", ".", "@", "/",+ "abcdefghijklmnopqrstuvwxyz0123456789ABCDEF", "[REDACTED]", "eyJ", "set-", "-H ", "curl ", "-u ", "-b ",+ "token=abcdefgh1234", `"api_key": "x y"`, "'secret': 42", "--password hunter2", `["--api-key", "k"]`,+ "curl -u admin:pw https://x", "curl -b 'sid=1' x", "Cookie: sid=1", "postgres://u:p@h", "redis://:p@h",+ "Bearer abc.def", "authorization: basic x", "ghp_abcdefghijklmnopqrstuvwxyz0123", "AKIAIOSFODNN7EXAMPLE",+ "sk-abcdefghijklmnopqrstuvwxyz", "sk_live_abcdefghijklmnop", "ya29.abcdefghijklmnopqrstuvwxyz",+ "eyJabcdefghij.eyJabcdefghij.abcdefghij", "-----BEGIN RSA PRIVATE KEY-----\nMIIEowIBAAKCAQEAabcdefghijklmnop\n"}+ for _, set := range []*redactNeedles{redactLabelNeedles, redactFlagNeedles} {+ fragments = append(fragments, set.needles...)+ }+ for _, rule := range redactionRules {+ fragments = append(fragments, rule.needles.needles...)+ }+ r := rand.New(rand.NewPCG(3, 4))+ matched := 0+ for i, rule := range redactionRules {+ // The fragments the rule's pattern matches on their own, and its needles, are drawn often.+ near := slices.Clone(rule.needles.needles)+ for _, f := range fragments {+ if rule.pattern().MatchString(f) {+ near = append(near, f)+ }+ }+ for range 400 {+ var b strings.Builder+ for range 1 + r.IntN(10) {+ f := fragments[r.IntN(len(fragments))]+ if r.IntN(2) == 0 {+ f = near[r.IntN(len(near))]+ }+ b.WriteString(f)+ }+ text := b.String()+ collect := func(text *redactGate) (out [][]int) {+ for m := range rule.matches(text, &engine.rules[i]) {+ out = append(out, m)+ }+ return out+ }+ want := collect(&redactGate{value: text, enabled: false, engine: engine})+ got := collect(&redactGate{value: text, enabled: true, engine: engine})+ if !slices.EqualFunc(got, want, slices.Equal[[]int]) {+ t.Fatalf("rule %d (%s) on %q: anchored matches %v, whole pattern %v", i, rule.expr, text, got, want)+ }+ withBudget(t, 0, -1, func() {+ got = collect(&redactGate{value: text, enabled: true, engine: engine})+ })+ if !slices.EqualFunc(got, want, slices.Equal[[]int]) {+ t.Fatalf("rule %d (%s) on %q: from a safe start %v, whole pattern %v", i, rule.expr, text, got, want)+ }+ matched += len(want)+ }+ }+ // The texts have to reach the patterns for the comparison to mean anything.+ if matched < 3000 {+ t.Fatalf("only %d matches across every rule", matched)+ }+}++// withBudget runs f with the anchored-matching budget set to perByte times the text plus base: 0+// and -1 make every rule run its whole pattern from a safe start.+func withBudget(t *testing.T, perByte, base int, f func()) {+ t.Helper()+ savedPerByte, savedBase := redactAnchoredPerByte, redactAnchoredBase+ redactAnchoredPerByte, redactAnchoredBase = perByte, base+ defer func() { redactAnchoredPerByte, redactAnchoredBase = savedPerByte, savedBase }()+ f()+}++// Redaction from safe starts gives what every rule's whole pattern gives, on the gate test's own+// random inputs: the safe start is exact for every rule, whatever comes before its first place.+// (randomRedactInput takes the legacy math/rand generator the gate test uses.)+func TestRedactFromSafeStartsNeverChangesAResult(t *testing.T) {+ r := mrand.New(mrand.NewSource(7))+ withBudget(t, 0, -1, func() {+ for range 5000 {+ input := randomRedactInput(r)+ if gated, all := redactString(input, true), redactString(input, false); gated != all {+ t.Fatalf("redaction from safe starts of %q = %q, every rule gives %q", input, gated, all)+ }+ }+ })+}++// Redaction's work is linear in the text whatever the text holds. Each text below makes some rule+// try many places, each of which a pattern could read on from to the end of a long line or run:+// many curl commands on one line, JWT-shaped and private-key-shaped fragments that never complete,+// assignments and flags without values, long runs of label words, URL schemes. The work counted+// is every byte redaction hands a pattern, and every byte it scans back over to find where a match+// starts: an upper bound on what the regexp engine and the needle checks read. Doubling the text+// must not more than double it, give or take, at every size.+func TestRedactionWorkIsLinear(t *testing.T) {+ families := map[string]string{+ "curl words on one line": "curl x ",+ "curl flags on one line": "curl -H 'a: b' -b ",+ "JWT prefixes": "-eyJabcdefghij",+ "private key headers": "-----BEGIN PRIVATE KEY-----Proc-Type: x ",+ "labels without values": "key=,",+ "quoted labels without values": `"token": ,`,+ "flags without values": " --token ''",+ "listed flags": `"--token", `,+ "one long flag": "token",+ "one long label": "key",+ "URL schemes": "a://",+ "authorization headers": "authorization: bearer ,",+ "cookie headers": "\ncookie: ",+ "assignments": "token=abcdefgh12345678 ",+ }+ for name, unit := range families {+ work := func(n int) int {+ text := strings.Repeat(unit, n/len(unit)+1)[:n]+ if name == "one long flag" {+ text = "-" + text + " x"+ }+ total := 0+ redactWork = func(bytes int) { total += bytes }+ defer func() { redactWork = nil }()+ redactString(text, true)+ return total+ }+ previous := work(16 << 10)+ for n := 32 << 10; n <= 256<<10; n *= 2 {+ w := work(n)+ if w > 5*previous/2+4096 {+ t.Errorf("%s: work %d on %d bytes, %d on half as many: not linear", name, w, n, previous)+ break+ }+ previous = w+ }+ }+}