The cross-harness, self-improving memory layer for AI agents. Join our community: https://discord.com/invite/zdNChS2fBu
Go agent-ai-cliagent-harnessesagent-securitybrowser-extensionciclaude-codecloudcodexcursordetection-engineeringjevknowledge-basememorymobile-device-managementobservabilitysecuritysecurity-information-and-event-managementsecurity-toolstelemetrytraces
Merge origin/main into claude/perf-sprint
Claude2f1025e1db5f
diff --cc pkg/asymptoteobserve/privacy.goindex 32eb8703,33cd00f8..13db415d--- a/pkg/asymptoteobserve/privacy.go+++ b/pkg/asymptoteobserve/privacy.go@@@ -2,9 -2,11 +2,12 @@@ package asymptoteobservimport ("encoding/json"+ "maps""regexp"+ "slices""strings"+ "sync"+ "unicode/utf8")const (@@@ -324,18 -168,18 +327,18 @@@ var redactionRules = concatRules([]reda// and stays (`ghp_[REDACTED]`, `AKIA[REDACTED]`), the way secret scanners report a finding: the// line still says a GitHub token or an AWS key was there, which is what detection rules match on// in the log, and none of the credential is left.- {regexp.MustCompile(`\b(sk-)[A-Za-z0-9_-]{20,}`), "${1}" + redactedValue, nil}, // Anthropic sk-ant-, OpenAI sk-proj- and sk-- {regexp.MustCompile(`\b(AKIA|ASIA|ABIA|ACCA)[A-Z0-9]{16}\b`), "${1}" + redactedValue, nil}, // AWS access key id- {regexp.MustCompile(`\b(gh[pousr]_)[A-Za-z0-9]{30,}`), "${1}" + redactedValue, nil}, // GitHub tokens- {regexp.MustCompile(`\b(github_pat_)[A-Za-z0-9_]{22,}`), "${1}" + redactedValue, nil}, // GitHub fine-grained tokens- {regexp.MustCompile(`\b(glpat-)[A-Za-z0-9_-]{20,}`), "${1}" + redactedValue, nil}, // GitLab- {regexp.MustCompile(`\b(xox[abposr]-)[A-Za-z0-9-]{10,}`), "${1}" + redactedValue, nil}, // Slack- {regexp.MustCompile(`\b([rs]k_(?:live|test)_)[A-Za-z0-9]{16,}`), "${1}" + redactedValue, nil}, // Stripe- {regexp.MustCompile(`\b(AIza)[0-9A-Za-z_-]{35}`), "${1}" + redactedValue, nil}, // Google API key- {regexp.MustCompile(`\b(npm_)[A-Za-z0-9]{36}`), "${1}" + redactedValue, nil}, // npm- {regexp.MustCompile(`\b(hf_)[A-Za-z0-9]{30,}`), "${1}" + redactedValue, nil}, // Hugging Face- {regexp.MustCompile(`\b(bcn_(?:api|device|mcp|enfdev)_)[A-Za-z0-9_-]{8,}`), "${1}" + redactedValue, nil}, // Beacon's own keys- {regexp.MustCompile(`\b(eyJ)[A-Za-z0-9_-]{8,}\.eyJ[A-Za-z0-9_-]{8,}\.[A-Za-z0-9_-]{8,}`), "${1}" + redactedValue, nil}, // JWT+ rule(`\b(sk-)[A-Za-z0-9_-]{20,}`, "${1}"+redactedValue, nil, "sk-"), // Anthropic sk-ant-, OpenAI sk-proj- and sk-+ rule(`\b(AKIA|ASIA|ABIA|ACCA)[A-Z0-9]{16}\b`, "${1}"+redactedValue, nil, "AKIA", "ASIA", "ABIA", "ACCA"), // AWS access key id+ rule(`\b(gh[pousr]_)[A-Za-z0-9]{30,}`, "${1}"+redactedValue, nil, "ghp_", "gho_", "ghu_", "ghs_", "ghr_"), // GitHub tokens+ rule(`\b(github_pat_)[A-Za-z0-9_]{22,}`, "${1}"+redactedValue, nil, "github_pat_"), // GitHub fine-grained tokens+ rule(`\b(glpat-)[A-Za-z0-9_-]{20,}`, "${1}"+redactedValue, nil, "glpat-"), // GitLab+ rule(`\b(xox[abposr]-)[A-Za-z0-9-]{10,}`, "${1}"+redactedValue, nil, "xox"), // Slack+ rule(`\b([rs]k_(?:live|test)_)[A-Za-z0-9]{16,}`, "${1}"+redactedValue, nil, "k_live_", "k_test_"), // Stripe+ rule(`\b(AIza)[0-9A-Za-z_-]{35}`, "${1}"+redactedValue, nil, "AIza"), // Google API key+ rule(`\b(npm_)[A-Za-z0-9]{36}`, "${1}"+redactedValue, nil, "npm_"), // npm+ rule(`\b(hf_)[A-Za-z0-9]{30,}`, "${1}"+redactedValue, nil, "hf_"), // Hugging Face- rule(`\b(bcn_(?:api|device|mcp)_)[A-Za-z0-9_-]{8,}`, "${1}"+redactedValue, nil, "bcn_"), // Beacon's own keys++ rule(`\b(bcn_(?:api|device|mcp|enfdev)_)[A-Za-z0-9_-]{8,}`, "${1}"+redactedValue, nil, "bcn_"), // Beacon's own keys+ rule(`\b(eyJ)[A-Za-z0-9_-]{8,}\.eyJ[A-Za-z0-9_-]{8,}\.[A-Za-z0-9_-]{8,}`, "${1}"+redactedValue, nil, "eyJ"), // JWT})func concatRules(groups ...[]redactionRule) []redactionRule {@@@ -416,93 -244,48 +419,118 @@@ func redactString(value string, gated b// repeated bare copies in the same retained prompt/output are also removed.// This is common in model reasoning that quotes `token=value` and later// refers to `value` by itself.- var assigned []string+ assigned := map[string]struct{}{}- for _, match := range redactAssignedPattern.FindAllStringSubmatch(value, -1) {- copied := strings.Join(match[1:], "")- if len(copied) >= 8 && copied != redactedValue && !isAuthScheme(copied) {- assigned[copied] = struct{}{}+ if text.holds(redactLabelNeedles) {+ for _, match := range redactAssignedPattern().FindAllStringSubmatch(value, -1) {+ copied := strings.Join(match[1:], "")+ if len(copied) >= 8 && copied != redactedValue && !isAuthScheme(copied) {- assigned = append(assigned, copied)++ assigned[copied] = struct{}{}+ }}}for _, rule := range redactionRules {- value = rule.apply(value)+ if !text.holds(rule.needles) {+ continue+ }+ text.set(rule.apply(text.value))}+ value = text.value// Each copy is searched for and removed; none of it survives into the result.- for _, copied := range assigned {- value = strings.ReplaceAll(value, copied, redactedValue)+ return replaceAssigned(value, assigned)+ }++ // replaceAssigned replaces every copy of each assigned value with redactedValue in one pass.+ // Replacing them one strings.ReplaceAll at a time scanned the text once per assignment, and both+ // grow with the text, so the cost was quadratic: Beacon Cloud redacts uploaded shared traces with+ // this function, where that let a small gzip body hold a core for minutes. The values are a set,+ // longest first, so where two start at the same byte the longer is removed whole.+ func replaceAssigned(value string, assigned map[string]struct{}) string {+ if len(assigned) == 0 {+ return value}- return value+ values := slices.SortedFunc(maps.Keys(assigned), func(a, b string) int {+ if len(a) != len(b) {+ return len(b) - len(a)+ }+ return strings.Compare(a, b)+ })+ pairs := make([]string, 0, 2*len(values))+ for _, copied := range values {+ pairs = append(pairs, copied, redactedValue)+ }+ return strings.NewReplacer(pairs...).Replace(value)+ }++ // IsSecretKeyName reports whether a structured field's name says its value is a credential, the+ // rule SanitizeMap applies, for callers that walk structured values of their own.+ func IsSecretKeyName(key string) bool {+ return isSecretKeyName(key)}+// redactGate answers whether text holds any of a set of needles, lower-casing it and searching for+// each set at most once per version of the text.+type redactGate struct {+ value string+ enabled bool+ lower string+ lowered bool+ // known and present are bit sets over redactNeedles ids: which sets were searched for in this+ // version of the text, and which were found.+ known, present uint64+ // unfoldable is set when the text holds a character Go's case folding equates with an ASCII+ // letter (U+017F folds to s, U+212A to k): a case-insensitive pattern can match "ſecret", which+ // lower-casing does not turn into "secret", so folded rules always run on such text.+ unfoldable bool+}++func (g *redactGate) set(value string) {+ if value != g.value {+ g.value, g.lowered, g.known, g.present = value, false, 0, 0+ }+}++func (g *redactGate) holds(set *redactNeedles) bool {+ if !g.enabled {+ return true+ }+ if set.within != nil && !g.holds(set.within) {+ return false+ }+ bit := uint64(1) << set.id+ if g.known&bit != 0 {+ return g.present&bit != 0+ }+ g.known |= bit+ haystack := g.value+ if set.folded {+ if !g.lowered {+ g.lower, g.lowered = strings.ToLower(g.value), true+ g.unfoldable = strings.ContainsAny(g.value, "\u017f\u212a")+ }+ if g.unfoldable {+ g.present |= bit+ return true+ }+ haystack = g.lower+ }+ for _, needle := range set.needles {+ for from := 0; ; {+ i := strings.Index(haystack[from:], needle)+ if i < 0 {+ break+ }+ start, end := from+i, from+i+len(needle)+ // The next search starts one byte on, not at end: an occurrence may overlap this one.+ from = start + 1+ if (set.precededBy == nil || set.precededBy(haystack[:start])) && (set.followedBy == nil || set.followedBy(haystack[end:])) {+ g.present |= bit+ return true+ }+ }+ }+ return false+}+// isSecretKeyName reports whether a structured field's name says its value is a credential:// `password`, `api_key`, `apiKey`, `X-Api-Key`, `GITHUB_TOKEN`, `AWS_SECRET_ACCESS_KEY`,// `client_secret`, `Cookie`. The name must end in the marker, so `input_tokens`, `token_type`,