Skip to content

The cross-harness, self-improving memory layer for AI agents. Join our community: https://discord.com/invite/zdNChS2fBu

Merge origin/main into claude/perf-sprint

Claude2f1025e1db5f
diff --cc pkg/asymptoteobserve/privacy.go
index 32eb8703,33cd00f8..13db415d
--- a/pkg/asymptoteobserve/privacy.go
+++ b/pkg/asymptoteobserve/privacy.go
@@@ -2,9 -2,11 +2,12 @@@ package asymptoteobserv
import (
"encoding/json"
+ "maps"
"regexp"
+ "slices"
"strings"
+ "sync"
+ "unicode/utf8"
)
const (
@@@ -324,18 -168,18 +327,18 @@@ var redactionRules = concatRules([]reda
// and stays (`ghp_[REDACTED]`, `AKIA[REDACTED]`), the way secret scanners report a finding: the
// line still says a GitHub token or an AWS key was there, which is what detection rules match on
// in the log, and none of the credential is left.
- {regexp.MustCompile(`\b(sk-)[A-Za-z0-9_-]{20,}`), "${1}" + redactedValue, nil}, // Anthropic sk-ant-, OpenAI sk-proj- and sk-
- {regexp.MustCompile(`\b(AKIA|ASIA|ABIA|ACCA)[A-Z0-9]{16}\b`), "${1}" + redactedValue, nil}, // AWS access key id
- {regexp.MustCompile(`\b(gh[pousr]_)[A-Za-z0-9]{30,}`), "${1}" + redactedValue, nil}, // GitHub tokens
- {regexp.MustCompile(`\b(github_pat_)[A-Za-z0-9_]{22,}`), "${1}" + redactedValue, nil}, // GitHub fine-grained tokens
- {regexp.MustCompile(`\b(glpat-)[A-Za-z0-9_-]{20,}`), "${1}" + redactedValue, nil}, // GitLab
- {regexp.MustCompile(`\b(xox[abposr]-)[A-Za-z0-9-]{10,}`), "${1}" + redactedValue, nil}, // Slack
- {regexp.MustCompile(`\b([rs]k_(?:live|test)_)[A-Za-z0-9]{16,}`), "${1}" + redactedValue, nil}, // Stripe
- {regexp.MustCompile(`\b(AIza)[0-9A-Za-z_-]{35}`), "${1}" + redactedValue, nil}, // Google API key
- {regexp.MustCompile(`\b(npm_)[A-Za-z0-9]{36}`), "${1}" + redactedValue, nil}, // npm
- {regexp.MustCompile(`\b(hf_)[A-Za-z0-9]{30,}`), "${1}" + redactedValue, nil}, // Hugging Face
- {regexp.MustCompile(`\b(bcn_(?:api|device|mcp|enfdev)_)[A-Za-z0-9_-]{8,}`), "${1}" + redactedValue, nil}, // Beacon's own keys
- {regexp.MustCompile(`\b(eyJ)[A-Za-z0-9_-]{8,}\.eyJ[A-Za-z0-9_-]{8,}\.[A-Za-z0-9_-]{8,}`), "${1}" + redactedValue, nil}, // JWT
+ rule(`\b(sk-)[A-Za-z0-9_-]{20,}`, "${1}"+redactedValue, nil, "sk-"), // Anthropic sk-ant-, OpenAI sk-proj- and sk-
+ rule(`\b(AKIA|ASIA|ABIA|ACCA)[A-Z0-9]{16}\b`, "${1}"+redactedValue, nil, "AKIA", "ASIA", "ABIA", "ACCA"), // AWS access key id
+ rule(`\b(gh[pousr]_)[A-Za-z0-9]{30,}`, "${1}"+redactedValue, nil, "ghp_", "gho_", "ghu_", "ghs_", "ghr_"), // GitHub tokens
+ rule(`\b(github_pat_)[A-Za-z0-9_]{22,}`, "${1}"+redactedValue, nil, "github_pat_"), // GitHub fine-grained tokens
+ rule(`\b(glpat-)[A-Za-z0-9_-]{20,}`, "${1}"+redactedValue, nil, "glpat-"), // GitLab
+ rule(`\b(xox[abposr]-)[A-Za-z0-9-]{10,}`, "${1}"+redactedValue, nil, "xox"), // Slack
+ rule(`\b([rs]k_(?:live|test)_)[A-Za-z0-9]{16,}`, "${1}"+redactedValue, nil, "k_live_", "k_test_"), // Stripe
+ rule(`\b(AIza)[0-9A-Za-z_-]{35}`, "${1}"+redactedValue, nil, "AIza"), // Google API key
+ rule(`\b(npm_)[A-Za-z0-9]{36}`, "${1}"+redactedValue, nil, "npm_"), // npm
+ rule(`\b(hf_)[A-Za-z0-9]{30,}`, "${1}"+redactedValue, nil, "hf_"), // Hugging Face
- rule(`\b(bcn_(?:api|device|mcp)_)[A-Za-z0-9_-]{8,}`, "${1}"+redactedValue, nil, "bcn_"), // Beacon's own keys
++ rule(`\b(bcn_(?:api|device|mcp|enfdev)_)[A-Za-z0-9_-]{8,}`, "${1}"+redactedValue, nil, "bcn_"), // Beacon's own keys
+ rule(`\b(eyJ)[A-Za-z0-9_-]{8,}\.eyJ[A-Za-z0-9_-]{8,}\.[A-Za-z0-9_-]{8,}`, "${1}"+redactedValue, nil, "eyJ"), // JWT
})
func concatRules(groups ...[]redactionRule) []redactionRule {
@@@ -416,93 -244,48 +419,118 @@@ func redactString(value string, gated b
// repeated bare copies in the same retained prompt/output are also removed.
// This is common in model reasoning that quotes `token=value` and later
// refers to `value` by itself.
- var assigned []string
+ assigned := map[string]struct{}{}
- for _, match := range redactAssignedPattern.FindAllStringSubmatch(value, -1) {
- copied := strings.Join(match[1:], "")
- if len(copied) >= 8 && copied != redactedValue && !isAuthScheme(copied) {
- assigned[copied] = struct{}{}
+ if text.holds(redactLabelNeedles) {
+ for _, match := range redactAssignedPattern().FindAllStringSubmatch(value, -1) {
+ copied := strings.Join(match[1:], "")
+ if len(copied) >= 8 && copied != redactedValue && !isAuthScheme(copied) {
- assigned = append(assigned, copied)
++ assigned[copied] = struct{}{}
+ }
}
}
for _, rule := range redactionRules {
- value = rule.apply(value)
+ if !text.holds(rule.needles) {
+ continue
+ }
+ text.set(rule.apply(text.value))
}
+ value = text.value
// Each copy is searched for and removed; none of it survives into the result.
- for _, copied := range assigned {
- value = strings.ReplaceAll(value, copied, redactedValue)
+ return replaceAssigned(value, assigned)
+ }
+
+ // replaceAssigned replaces every copy of each assigned value with redactedValue in one pass.
+ // Replacing them one strings.ReplaceAll at a time scanned the text once per assignment, and both
+ // grow with the text, so the cost was quadratic: Beacon Cloud redacts uploaded shared traces with
+ // this function, where that let a small gzip body hold a core for minutes. The values are a set,
+ // longest first, so where two start at the same byte the longer is removed whole.
+ func replaceAssigned(value string, assigned map[string]struct{}) string {
+ if len(assigned) == 0 {
+ return value
}
- return value
+ values := slices.SortedFunc(maps.Keys(assigned), func(a, b string) int {
+ if len(a) != len(b) {
+ return len(b) - len(a)
+ }
+ return strings.Compare(a, b)
+ })
+ pairs := make([]string, 0, 2*len(values))
+ for _, copied := range values {
+ pairs = append(pairs, copied, redactedValue)
+ }
+ return strings.NewReplacer(pairs...).Replace(value)
+ }
+
+ // IsSecretKeyName reports whether a structured field's name says its value is a credential, the
+ // rule SanitizeMap applies, for callers that walk structured values of their own.
+ func IsSecretKeyName(key string) bool {
+ return isSecretKeyName(key)
}
+// redactGate answers whether text holds any of a set of needles, lower-casing it and searching for
+// each set at most once per version of the text.
+type redactGate struct {
+ value string
+ enabled bool
+ lower string
+ lowered bool
+ // known and present are bit sets over redactNeedles ids: which sets were searched for in this
+ // version of the text, and which were found.
+ known, present uint64
+ // unfoldable is set when the text holds a character Go's case folding equates with an ASCII
+ // letter (U+017F folds to s, U+212A to k): a case-insensitive pattern can match "ſecret", which
+ // lower-casing does not turn into "secret", so folded rules always run on such text.
+ unfoldable bool
+}
+
+func (g *redactGate) set(value string) {
+ if value != g.value {
+ g.value, g.lowered, g.known, g.present = value, false, 0, 0
+ }
+}
+
+func (g *redactGate) holds(set *redactNeedles) bool {
+ if !g.enabled {
+ return true
+ }
+ if set.within != nil && !g.holds(set.within) {
+ return false
+ }
+ bit := uint64(1) << set.id
+ if g.known&bit != 0 {
+ return g.present&bit != 0
+ }
+ g.known |= bit
+ haystack := g.value
+ if set.folded {
+ if !g.lowered {
+ g.lower, g.lowered = strings.ToLower(g.value), true
+ g.unfoldable = strings.ContainsAny(g.value, "\u017f\u212a")
+ }
+ if g.unfoldable {
+ g.present |= bit
+ return true
+ }
+ haystack = g.lower
+ }
+ for _, needle := range set.needles {
+ for from := 0; ; {
+ i := strings.Index(haystack[from:], needle)
+ if i < 0 {
+ break
+ }
+ start, end := from+i, from+i+len(needle)
+ // The next search starts one byte on, not at end: an occurrence may overlap this one.
+ from = start + 1
+ if (set.precededBy == nil || set.precededBy(haystack[:start])) && (set.followedBy == nil || set.followedBy(haystack[end:])) {
+ g.present |= bit
+ return true
+ }
+ }
+ }
+ return false
+}
+
// isSecretKeyName reports whether a structured field's name says its value is a credential:
// `password`, `api_key`, `apiKey`, `X-Api-Key`, `GITHUB_TOKEN`, `AWS_SECRET_ACCESS_KEY`,
// `client_secret`, `Cookie`. The name must end in the marker, so `input_tokens`, `token_type`,