The cross-harness, self-improving memory layer for AI agents. Join our community: https://discord.com/invite/zdNChS2fBu
Go agent-ai-cliagent-harnessesagent-securitybrowser-extensionciclaude-codecloudcodexcursordetection-engineeringjevknowledge-basememorymobile-device-managementobservabilitysecuritysecurity-information-and-event-managementsecurity-toolstelemetrytraces
Make the hook path faster: gated redaction, lazy patterns, dedupe prefilter
Claude13366b702549
diff --git a/cli/beacon-hooks/cmd/endpoint_events.go b/cli/beacon-hooks/cmd/endpoint_events.goindex be7ca143..cbffc568 100644--- a/cli/beacon-hooks/cmd/endpoint_events.go+++ b/cli/beacon-hooks/cmd/endpoint_events.go@@ -10,6 +10,7 @@ import ("regexp""strconv""strings"+ "sync""time""github.com/asymptote-labs/agent-beacon/cli/beacon-hooks/internal/cloudshuttle"@@ -442,7 +443,7 @@ func applyIngestedToolResult(fields map[string]interface{}, action, toolName str// claudeBashExitLine is the first line Claude Code writes into a failed Bash call's `error` when// the command ran and exited. Its absence means the shell itself could not be started.-var claudeBashExitLine = regexp.MustCompile(`^Exit code (-?\d+)$`)+var claudeBashExitLine = sync.OnceValue(func() *regexp.Regexp { return regexp.MustCompile(`^Exit code (-?\d+)$`) })// applyClaudeBashResult maps the result of Claude Code's Bash tool onto the command field, and// reports whether the payload is a failure of a command that ran and exited.@@ -466,7 +467,7 @@ func applyClaudeBashResult(fields, input map[string]interface{}, toolName stringexited := falseif errText := getFirstStr(input, "error"); errText != "" {firstLine, rest, _ := strings.Cut(errText, "\n")- match := claudeBashExitLine.FindStringSubmatch(strings.TrimSpace(firstLine))+ match := claudeBashExitLine().FindStringSubmatch(strings.TrimSpace(firstLine))if match == nil {return false}diff --git a/cli/beacon-hooks/cmd/hook_bench_test.go b/cli/beacon-hooks/cmd/hook_bench_test.gonew file mode 100644index 00000000..f8ad448a--- /dev/null+++ b/cli/beacon-hooks/cmd/hook_bench_test.go@@ -0,0 +1,72 @@+package cmd++import (+ "encoding/json"+ "fmt"+ "path/filepath"+ "strings"+ "testing"++ "github.com/spf13/cobra"+)++// The hook benchmarks time one hook run in process, the work a runtime waits on before every tool+// call continues. Process start is not in them; `GODEBUG=inittrace=1 beacon-hooks version` shows+// package initialization, which every run also pays.++func benchmarkHook(b *testing.B, run func(*cobra.Command, []string), input map[string]interface{}) {+ setupHookConfigDirs(b)+ platformFlag = "claude"+ b.Setenv("BEACON_ENDPOINT_LOG", filepath.Join(b.TempDir(), "runtime.jsonl"))+ b.ReportAllocs()+ for i := 0; i < b.N; i++ {+ // A distinct call id per run, so no run is taken for a repeat of the one before.+ input["tool_use_id"] = fmt.Sprintf("toolu_%d", i)+ stdin, err := json.Marshal(input)+ if err != nil {+ b.Fatal(err)+ }+ runHookWithRawInput(b, run, stdin)+ }+}++func benchClaudePayload(event, tool string, toolInput map[string]interface{}) map[string]interface{} {+ return map[string]interface{}{+ "session_id": "bench-session",+ "cwd": "/repo",+ "hook_event_name": event,+ "tool_name": tool,+ "tool_input": toolInput,+ "tool_use_id": "toolu_bench",+ }+}++func BenchmarkHookPreToolBash(b *testing.B) {+ benchmarkHook(b, runPreTool, benchClaudePayload("PreToolUse", "Bash", map[string]interface{}{"command": "go test ./... -count=1", "description": "Run tests"}))+}++func BenchmarkHookPostToolBashOutput(b *testing.B) {+ var out strings.Builder+ for i := 0; i < 400; i++ {+ fmt.Fprintf(&out, "ok \tgithub.com/example/project/internal/pkg%d\t0.%03ds // refreshes the api token cache\n", i, i)+ }+ input := benchClaudePayload("PostToolUse", "Bash", map[string]interface{}{"command": "go test ./..."})+ input["tool_response"] = map[string]interface{}{"stdout": out.String(), "stderr": "", "interrupted": false}+ benchmarkHook(b, runPostTool, input)+}++func BenchmarkHookPostToolReadCode(b *testing.B) {+ var code strings.Builder+ for i := 0; i < 300; i++ {+ fmt.Fprintf(&code, "func load%d(cfg Config) (string, error) { token := cfg.Token; if token == \"\" { return \"\", errMissing }; return token, nil }\n", i)+ }+ input := benchClaudePayload("PostToolUse", "Read", map[string]interface{}{"file_path": "/repo/config.go"})+ input["tool_response"] = map[string]interface{}{"type": "text", "file": map[string]interface{}{"filePath": "/repo/config.go", "content": code.String()}}+ benchmarkHook(b, runPostTool, input)+}++func BenchmarkHookPromptSubmit(b *testing.B) {+ input := benchClaudePayload("UserPromptSubmit", "", nil)+ input["prompt"] = strings.Repeat("Please refactor the token refresh logic in auth.go so the api key is read from the env, then run the tests. ", 8)+ benchmarkHook(b, runPromptSubmit, input)+}diff --git a/cli/beacon-hooks/cmd/kimi.go b/cli/beacon-hooks/cmd/kimi.goindex 86bea151..efebb330 100644--- a/cli/beacon-hooks/cmd/kimi.go+++ b/cli/beacon-hooks/cmd/kimi.go@@ -4,6 +4,7 @@ import ("regexp""strconv""strings"+ "sync""github.com/asymptote-labs/agent-beacon/cli/beacon-hooks/internal/config"hookdiff "github.com/asymptote-labs/agent-beacon/cli/beacon-hooks/internal/diff"@@ -207,7 +208,7 @@ const kimiAppendMode = "append"//// A successful command produces no such sentence, and its absence is deliberately not read as// "exited 0" -- see kimiExitCode.-var kimiExitCodeMarker = regexp.MustCompile(`Command failed with exit code: (\d+)`)+var kimiExitCodeMarker = sync.OnceValue(func() *regexp.Regexp { return regexp.MustCompile(`Command failed with exit code: (\d+)`) })// kimiToolKindFor classifies a Kimi Code tool by name, and reports whether the name is known at// all.@@ -379,7 +380,7 @@ func kimiResultText(input map[string]interface{}) string {// that has only just been launched and whose result is a task id. Those are different facts, and// writing 0 would make the second one indistinguishable from the first.func kimiExitCode(output string) (int, bool) {- matches := kimiExitCodeMarker.FindAllStringSubmatch(output, -1)+ matches := kimiExitCodeMarker().FindAllStringSubmatch(output, -1)if len(matches) == 0 {return 0, false}diff --git a/cli/beacon-hooks/cmd/perf_ratchet_test.go b/cli/beacon-hooks/cmd/perf_ratchet_test.gonew file mode 100644index 00000000..6f6c9bc8--- /dev/null+++ b/cli/beacon-hooks/cmd/perf_ratchet_test.go@@ -0,0 +1,34 @@+package cmd++import (+ "os"+ "os/exec"+ "regexp"+ "strconv"+ "testing"+)++// initAllocCeiling bounds the allocations this package's initialization makes, test files+// included (267 when it was set; 3,518 before the package-level patterns compiled on first use).+// Every hook run pays it before reading its input. Lower it when a change lowers the count.+const initAllocCeiling = 350++func TestHookInitStaysCheap(t *testing.T) {+ if os.Getenv("BEACON_INIT_TRACE_CHILD") != "" {+ return+ }+ cmd := exec.Command(os.Args[0], "-test.run=^TestHookInitStaysCheap$")+ cmd.Env = append(os.Environ(), "GODEBUG=inittrace=1", "BEACON_INIT_TRACE_CHILD=1")+ out, err := cmd.CombinedOutput()+ if err != nil {+ t.Fatalf("re-run test binary: %v\n%s", err, out)+ }+ match := regexp.MustCompile(`(?m)^init github\.com/asymptote-labs/agent-beacon/cli/beacon-hooks/cmd @.* (\d+) allocs$`).FindSubmatch(out)+ if match == nil {+ t.Skipf("no init trace for this package in:\n%s", out)+ }+ allocs, _ := strconv.Atoi(string(match[1]))+ if allocs > initAllocCeiling {+ t.Fatalf("package init made %d allocations, ceiling %d; compile patterns on first use (sync.OnceValue)", allocs, initAllocCeiling)+ }+}diff --git a/cli/beacon-hooks/cmd/pi_family.go b/cli/beacon-hooks/cmd/pi_family.goindex 6847e2e9..79116d86 100644--- a/cli/beacon-hooks/cmd/pi_family.go+++ b/cli/beacon-hooks/cmd/pi_family.go@@ -8,6 +8,7 @@ import ("regexp""runtime""strings"+ "sync""github.com/asymptote-labs/agent-beacon/cli/beacon-hooks/internal/diff""github.com/asymptote-labs/agent-beacon/pkg/asymptoteobserve"@@ -349,7 +350,7 @@ const ompDeviceScheme = "xd://"// ompDeviceHelpContent matches a device write that asks for documentation rather than making a// call: an empty payload, `?`, or `help`. It mirrors HELP_CONTENT_RE in Oh My Pi's tools/xdev.ts,// which is how the runtime itself tells the two apart before anything has run.-var ompDeviceHelpContent = regexp.MustCompile(`(?i)^\s*(\?|help)?\s*$`)+var ompDeviceHelpContent = sync.OnceValue(func() *regexp.Regexp { return regexp.MustCompile(`(?i)^\s*(\?|help)?\s*$`) })// ompDeviceCall resolves a write to `xd://<tool>` into the call of <tool> it carries.//@@ -383,7 +384,7 @@ func ompDeviceCall(call piToolCall) (piToolCall, bool) {if getFirstStr(xdev, "mode") != "execute" {return piToolCall{}, false}- } else if ompDeviceHelpContent.MatchString(getFirstStr(call.args, "content")) {+ } else if ompDeviceHelpContent().MatchString(getFirstStr(call.args, "content")) {return piToolCall{}, false}@@ -565,17 +566,19 @@ func piResolvedPath(details map[string]interface{}) (string, bool) {// packages/tui/src/tools/read.ts: comma-joined line ranges (`10`, `10-40`, `10..40`, `10+30`,// `10-`), a tail (`-60`), or `raw`, `conflicts` or `img`. A range may not end in `+`, which is// what the runtime's trailing lookbehind enforces.-var ompLineSelector = regexp.MustCompile(`(?i)^(?:` + ompLineRange + `(?:,` + ompLineRange + `)*|-\d+|raw|conflicts|img)$`)+var ompLineSelector = sync.OnceValue(func() *regexp.Regexp {+ return regexp.MustCompile(`(?i)^(?:` + ompLineRange + `(?:,` + ompLineRange + `)*|-\d+|raw|conflicts|img)$`)+})const ompLineRange = `L?\d+(?:(?:\.\.|-)(?:L?\d+)?|\+L?\d+)?`// ompStrayColon matches the stray `:` some models put in front of a path (`:/abs`, `:../rel`,// `:C:\repo`), which Oh My Pi drops before resolving: no real path starts with `:`.-var ompStrayColon = regexp.MustCompile(`^:(?:[/\\~]|\.\.?[/\\]|[A-Za-z]:)`)+var ompStrayColon = sync.OnceValue(func() *regexp.Regexp { return regexp.MustCompile(`^:(?:[/\\~]|\.\.?[/\\]|[A-Za-z]:)`) })// ompWindowsAbsolute matches a Windows absolute path: a drive (`C:\`, `C:/`), a UNC share, or a// root-relative `\path`, the forms Node's path.win32.isAbsolute accepts.-var ompWindowsAbsolute = regexp.MustCompile(`^(?:[A-Za-z]:[\\/]|[\\/]{2}|\\)`)+var ompWindowsAbsolute = sync.OnceValue(func() *regexp.Regexp { return regexp.MustCompile(`^(?:[A-Za-z]:[\\/]|[\\/]{2}|\\)`) })// ompStripPathMarker drops the marker characters Oh My Pi strips from the front of a path before// it reads one, mirroring the first steps of its expandPath (tools/path-utils.ts).@@ -586,14 +589,14 @@ var ompWindowsAbsolute = regexp.MustCompile(`^(?:[A-Za-z]:[\\/]|[\\/]{2}|\\)`)// than a file or web URL is taken for one here, because it is not a file either way. Stripping the// `@` before the URI check is what keeps `@skill://name` from being read as a file path.func ompStripPathMarker(target string) string {- if ompStrayColon.MatchString(target) {+ if ompStrayColon().MatchString(target) {target = target[1:]}rest, ok := strings.CutPrefix(target, "@")if !ok {return target}- if strings.HasPrefix(rest, "/") || rest == "~" || strings.HasPrefix(rest, "~/") || ompWindowsAbsolute.MatchString(rest) {+ if strings.HasPrefix(rest, "/") || rest == "~" || strings.HasPrefix(rest, "~/") || ompWindowsAbsolute().MatchString(rest) {return rest}if scheme, _, ok := strings.Cut(rest, "://"); ok && isURIScheme(scheme) {@@ -608,7 +611,7 @@ func ompStripPathMarker(target string) string {// ompUnicodeSpaces are the space characters Oh My Pi reads as a plain space in a path: no-break,// the U+2000 block, narrow no-break, medium mathematical and ideographic.-var ompUnicodeSpaces = regexp.MustCompile("[\u00A0\u2000-\u200A\u202F\u205F\u3000]")+var ompUnicodeSpaces = sync.OnceValue(func() *regexp.Regexp { return regexp.MustCompile("[\u00A0\u2000-\u200A\u202F\u205F\u3000]") })// ompToolPath turns a path as an Oh My Pi tool was given it into the file it names.//@@ -637,7 +640,7 @@ var ompUnicodeSpaces = regexp.MustCompile("[\u00A0\u2000-\u200A\u202F\u205F\u300// edit rebound by its snapshot tag, or a macOS spelling variant of the name. Those reach only the// result, which reports the file the runtime actually used.func ompToolPath(target, cwd, goos string, wsl bool) string {- p := ompUnicodeSpaces.ReplaceAllString(target, " ")+ p := ompUnicodeSpaces().ReplaceAllString(target, " ")if strings.HasPrefix(p, "~") {if home, err := os.UserHomeDir(); err == nil {p = filepath.Join(home, p[1:])@@ -655,7 +658,7 @@ func ompToolPath(target, cwd, goos string, wsl bool) string {stripped := pfor range 2 {i := strings.LastIndexByte(stripped, ':')- if i <= 0 || !ompLineSelector.MatchString(stripped[i+1:]) {+ if i <= 0 || !ompLineSelector().MatchString(stripped[i+1:]) {break}stripped = stripped[:i]@@ -697,7 +700,7 @@ func ompDriveAliasPath(p, goos string, wsl bool) string {}return drive + `:\` + strings.Join(nonEmpty(tail), `\`)case wsl:- match := ompWindowsDrivePath.FindStringSubmatch(strings.ReplaceAll(strings.TrimSpace(p), "/", `\`))+ match := ompWindowsDrivePath().FindStringSubmatch(strings.ReplaceAll(strings.TrimSpace(p), "/", `\`))if match == nil {return p}@@ -723,7 +726,7 @@ func ompDriveAliasPath(p, goos string, wsl bool) string {}}-var ompWindowsDrivePath = regexp.MustCompile(`^([A-Za-z]):\\(.*)$`)+var ompWindowsDrivePath = sync.OnceValue(func() *regexp.Regexp { return regexp.MustCompile(`^([A-Za-z]):\\(.*)$`) })func isDriveLetter(s string) bool {return len(s) == 1 && (s[0] >= 'a' && s[0] <= 'z' || s[0] >= 'A' && s[0] <= 'Z')diff --git a/cli/beacon-hooks/cmd/pre_tool_response_test.go b/cli/beacon-hooks/cmd/pre_tool_response_test.goindex 32ba9734..cc3d8ff5 100644--- a/cli/beacon-hooks/cmd/pre_tool_response_test.go+++ b/cli/beacon-hooks/cmd/pre_tool_response_test.go@@ -161,7 +161,7 @@ func mustJSON(t *testing.T, value interface{}) []byte {// runHookWithRawInput feeds stdin bytes as given and returns stdout as written, so a test can tell// no output from an empty object and can send a payload that is not JSON. The payloads here are// small, so writing before the hook runs fits in any pipe buffer.-func runHookWithRawInput(t *testing.T, run func(cmd *cobra.Command, args []string), stdin []byte) []byte {+func runHookWithRawInput(t testing.TB, run func(cmd *cobra.Command, args []string), stdin []byte) []byte {t.Helper()stdinR, stdinW, err := os.Pipe()if err != nil {diff --git a/cli/beacon-hooks/cmd/pre_tool_test.go b/cli/beacon-hooks/cmd/pre_tool_test.goindex 94dbd290..d3f09b4b 100644--- a/cli/beacon-hooks/cmd/pre_tool_test.go+++ b/cli/beacon-hooks/cmd/pre_tool_test.go@@ -950,7 +950,7 @@ func TestRunSessionEndRemovesSessionLog(t *testing.T) {}}-func setupHookConfigDirs(t *testing.T) {+func setupHookConfigDirs(t testing.TB) {t.Helper()tmp := t.TempDir()diff --git a/cli/beacon-hooks/cmd/session_start.go b/cli/beacon-hooks/cmd/session_start.goindex d0fbbbb4..c755f835 100644--- a/cli/beacon-hooks/cmd/session_start.go+++ b/cli/beacon-hooks/cmd/session_start.go@@ -5,6 +5,7 @@ import ("os""regexp""strings"+ "sync""github.com/spf13/cobra"@@ -81,7 +82,7 @@ func runSessionStart(cmd *cobra.Command, args []string) {// claudeEnvSessionID is the shape of session id written to CLAUDE_ENV_FILE. The file is run as shell// code before every Bash command, so anything outside this set is refused rather than quoted.-var claudeEnvSessionID = regexp.MustCompile(`^[A-Za-z0-9][A-Za-z0-9._:-]{0,199}$`)+var claudeEnvSessionID = sync.OnceValue(func() *regexp.Regexp { return regexp.MustCompile(`^[A-Za-z0-9][A-Za-z0-9._:-]{0,199}$`) })// exportClaudeSessionEnv appends BEACON_SESSION_ID and BEACON_HARNESS to the file Claude Code names// in CLAUDE_ENV_FILE, so the shell commands an agent runs later in the session -- `beacon share@@ -96,7 +97,7 @@ func exportClaudeSessionEnv(sessionID string) error {return nil}path := strings.TrimSpace(os.Getenv("CLAUDE_ENV_FILE"))- if path == "" || !claudeEnvSessionID.MatchString(sessionID) {+ if path == "" || !claudeEnvSessionID().MatchString(sessionID) {return nil}f, err := os.OpenFile(path, os.O_APPEND|os.O_CREATE|os.O_WRONLY, 0o600)diff --git a/cli/beacon-hooks/cmd/stop.go b/cli/beacon-hooks/cmd/stop.goindex 55b4c1fc..5a0b0d90 100644--- a/cli/beacon-hooks/cmd/stop.go+++ b/cli/beacon-hooks/cmd/stop.go@@ -6,6 +6,7 @@ import ("os""regexp""strings"+ "sync""time""github.com/spf13/cobra"@@ -292,11 +293,13 @@ func extractMessagesFromCursorTranscript(transcriptPath string) []map[string]int// These tags are added by Cursor to wrap user input and context — they are not part// of the actual conversation content.// Known tags: <user_query>, <attached_files>, <git_diff_from_branch_to_main>-var cursorXMLTagPattern = regexp.MustCompile(`</?(?:user_query|attached_files|git_diff_from_branch_to_main)>`)+var cursorXMLTagPattern = sync.OnceValue(func() *regexp.Regexp {+ return regexp.MustCompile(`</?(?:user_query|attached_files|git_diff_from_branch_to_main)>`)+})// stripCursorXMLTags removes Cursor's wrapper tags from transcript text.func stripCursorXMLTags(text string) string {- text = cursorXMLTagPattern.ReplaceAllString(text, "")+ text = cursorXMLTagPattern().ReplaceAllString(text, "")return strings.TrimSpace(text)}@@ -431,9 +434,11 @@ func isLocalCommandBlock(text string) bool {// systemInstructionPattern matches <system_instruction>...</system_instruction> and// <system-instruction>...</system-instruction> blocks including their content.-var systemInstructionPattern = regexp.MustCompile(`(?s)<system[_-]instruction>.*?</system[_-]instruction>\s*`)+var systemInstructionPattern = sync.OnceValue(func() *regexp.Regexp {+ return regexp.MustCompile(`(?s)<system[_-]instruction>.*?</system[_-]instruction>\s*`)+})// stripSystemInstructions removes system instruction blocks from user message text.func stripSystemInstructions(text string) string {- return strings.TrimSpace(systemInstructionPattern.ReplaceAllString(text, ""))+ return strings.TrimSpace(systemInstructionPattern().ReplaceAllString(text, ""))}diff --git a/pkg/asymptoteobserve/dedupe.go b/pkg/asymptoteobserve/dedupe.goindex e42585a7..d6079353 100644--- a/pkg/asymptoteobserve/dedupe.go+++ b/pkg/asymptoteobserve/dedupe.go@@ -17,11 +17,12 @@ const ()type endpointDedupeEvent struct {- action string- harness string- key string- callID string- ts time.Time+ action string+ harness string+ sessionID string+ key string+ callID string+ ts time.Time}// IsDuplicateEndpointEvent reports whether candidateLine duplicates a recently@@ -37,7 +38,11 @@ func IsDuplicateEndpointEvent(path string, candidateLine []byte, window time.Durreturn false}effectiveWindow := endpointDedupeWindow(candidate.action, window)+ sessionBytes := dedupeSessionNeedle(candidate.sessionID)for _, line := range bytes.Split(data, []byte{'\n'}) {+ if sessionBytes != nil && !mayDuplicate(line, sessionBytes, candidate.action) {+ continue+ }existing, ok := endpointDedupeCandidate(line)if !ok {continue@@ -170,7 +175,63 @@ func endpointDedupeCandidate(line []byte) (endpointDedupeEvent, bool) {strings.ToLower(workspace),target,}, "\x00")- return endpointDedupeEvent{action: strings.ToLower(action), harness: harness, key: key, callID: callID, ts: ts.UTC()}, true+ return endpointDedupeEvent{action: strings.ToLower(action), harness: harness, sessionID: sessionID, key: key, callID: callID, ts: ts.UTC()}, true+}++// dedupeSessionNeedle is the bytes every log line describing sessionID holds, or nil when there is+// no such guarantee.+//+// Decoding a line is what the tail scan costs: a hook pays it for every line of the last 256 KiB+// of the log, before every event it writes. A line can only duplicate the candidate when its+// session.id and its event.action are the same (the action compared without case), and a value of+// printable ASCII without a quote or backslash has one spelling in JSON that does not escape+// ordinary characters, so a line that does not hold both verbatim can be skipped undecoded. An id+// that a JSON number, literal, object or array would also produce (stringValue accepts those) gets+// no needle, and mayDuplicate still decodes every line with a \u or \/ escape in it. Every action+// dedupeAction accepts is such a value.+func dedupeSessionNeedle(sessionID string) []byte {+ if sessionID == "" || sessionID == "true" || sessionID == "false" || sessionID == "null" {+ return nil+ }+ if sessionID[0] == '{' || sessionID[0] == '[' || sessionID[0] == '-' || (sessionID[0] >= '0' && sessionID[0] <= '9') {+ return nil+ }+ for i := 0; i < len(sessionID); i++ {+ if c := sessionID[i]; c < 0x20 || c > 0x7e || c == '"' || c == '\\' {+ return nil+ }+ }+ return []byte(sessionID)+}++// mayDuplicate reports whether line could describe action in the session, by its bytes alone.+func mayDuplicate(line, session []byte, action string) bool {+ if bytes.Contains(line, []byte(`\u`)) || bytes.Contains(line, []byte(`\/`)) {+ return true+ }+ // The action is compared after strings.ToLower, which takes U+0130 to i and U+212A (the+ // Kelvin sign) to k: an action spelled with either matches in bytes no fold of ours finds.+ if bytes.Contains(line, []byte("\u0130")) || bytes.Contains(line, []byte("\u212a")) {+ return true+ }+ return bytes.Contains(line, session) && containsFold(line, action)+}++// containsFold reports whether s holds needle, which is lower case ASCII, in any case.+func containsFold(s []byte, needle string) bool {+ if needle == "" {+ return true+ }+ first, upper := needle[0], needle[0]+ if first >= 'a' && first <= 'z' {+ upper = first - 'a' + 'A'+ }+ for i := 0; i+len(needle) <= len(s); i++ {+ if (s[i] == first || s[i] == upper) && bytes.EqualFold(s[i:i+len(needle)], []byte(needle)) {+ return true+ }+ }+ return false}func endpointDedupeWindow(action string, fallback time.Duration) time.Duration {diff --git a/pkg/asymptoteobserve/dedupe_prefilter_test.go b/pkg/asymptoteobserve/dedupe_prefilter_test.gonew file mode 100644index 00000000..5e2dfecc--- /dev/null+++ b/pkg/asymptoteobserve/dedupe_prefilter_test.go@@ -0,0 +1,83 @@+package asymptoteobserve++import (+ "encoding/json"+ "fmt"+ "math/rand"+ "os"+ "path/filepath"+ "strings"+ "testing"+ "time"+)++// The tail scan skips lines whose bytes rule them out without decoding them. This holds that+// shortcut to the answers decoding every line gives.++// isDuplicateEndpointEventReference is the tail scan without its shortcut: every line decoded.+func isDuplicateEndpointEventReference(path string, candidateLine []byte, window time.Duration) bool {+ decode := endpointDedupeCandidate+ candidate, ok := decode(candidateLine)+ if !ok {+ return false+ }+ data, err := readEndpointDedupeTail(path)+ if err != nil {+ return false+ }+ for _, line := range strings.Split(string(data), "\n") {+ if existing, ok := decode([]byte(strings.TrimSpace(line))); ok && duplicateEndpointEvents(existing, candidate, endpointDedupeWindow(candidate.action, window)) {+ return true+ }+ }+ return false+}++func TestDedupeShortcutsNeverChangeTheAnswer(t *testing.T) {+ r := rand.New(rand.NewSource(4))+ base := time.Date(2026, 10, 12, 3, 47, 36, 0, time.UTC)+ sessions := []interface{}{"abc-123", "other", "ABC-123", "abc-123 ", 42.0, "a\"b", "séance", "<x>"}+ actions := []string{"tool.invoked", "TOOL.INVOKED", "command.executed", "tool.completed", "file.read", "tool.İnvoked", "tool.invoKed"}+ harnesses := []string{"claude_code", "codex", "Claude_Code"}+ line := func() []byte {+ event := map[string]interface{}{+ "event": map[string]interface{}{"action": actions[r.Intn(len(actions))]},+ "session": map[string]interface{}{"id": sessions[r.Intn(len(sessions))], "working_directory": "/repo"},+ "harness": map[string]interface{}{"name": harnesses[r.Intn(len(harnesses))]},+ "timestamp": base.Add(time.Duration(r.Intn(20)) * time.Second).Format(time.RFC3339Nano),+ "tool": map[string]interface{}{"name": []string{"Bash", "Read"}[r.Intn(2)]},+ "command": map[string]interface{}{"command": "ls"},+ "file": map[string]interface{}{"path": "/repo/a.go"},+ "message": []string{"m", "<html>&", "a/b"}[r.Intn(3)],+ }+ if r.Intn(2) == 0 {+ event["gen_ai"] = map[string]interface{}{"tool": map[string]interface{}{"call": map[string]interface{}{"id": fmt.Sprintf("toolu_%d", r.Intn(3))}}}+ }+ data, _ := json.Marshal(event)+ return data+ }+ dir := t.TempDir()+ answers := map[bool]int{}+ for round := 0; round < 80; round++ {+ var log strings.Builder+ for i := r.Intn(40); i >= 0; i-- {+ log.Write(line())+ log.WriteByte('\n')+ }+ path := filepath.Join(dir, fmt.Sprintf("runtime-%d.jsonl", round))+ if err := os.WriteFile(path, []byte(log.String()), 0o600); err != nil {+ t.Fatal(err)+ }+ for i := 0; i < 20; i++ {+ candidate := line()+ got := IsDuplicateEndpointEvent(path, candidate, EndpointDuplicateWindow)+ if want := isDuplicateEndpointEventReference(path, candidate, EndpointDuplicateWindow); got != want {+ t.Fatalf("candidate %s against\n%s\nis a duplicate: %v, decoded whole: %v", candidate, log.String(), got, want)+ }+ answers[got]+++ }+ }+ if answers[true] < 50 || answers[false] < 50 {+ t.Fatalf("the corpus has to produce both answers to test anything: %v", answers)+ }+}diff --git a/pkg/asymptoteobserve/handoff.go b/pkg/asymptoteobserve/handoff.goindex 852b65a9..e5c289fd 100644--- a/pkg/asymptoteobserve/handoff.go+++ b/pkg/asymptoteobserve/handoff.go@@ -3,6 +3,8 @@ package asymptoteobserveimport ("fmt""regexp"+ "strings"+ "sync")// HandoffInfo links a session to the session it was handed off from: the one whose brief@@ -16,10 +18,15 @@ type HandoffInfo struct {// that whatever records the prompt can link the new session to the old one. It is a claim made in// prompt text, not a verified identity: anyone can type it, and a session.handoff event records that// the marker was observed, nothing more. A harness name may carry a hyphen, as devin-cli does.+//+// The patterns compile on first use: counted repetition compiles to one instruction per count, and+// a hook process that never meets a marker should not pay for building them.var (- handoffMarkerPattern = regexp.MustCompile(`\[beacon-handoff from=([a-z0-9_-]{1,64}) session=([A-Za-z0-9._:\-]{1,200})\]`)- handoffHarnessChars = regexp.MustCompile(`^[a-z0-9_-]{1,64}$`)- handoffSessionChars = regexp.MustCompile(`^[A-Za-z0-9._:\-]{1,200}$`)+ handoffMarkerPattern = sync.OnceValue(func() *regexp.Regexp {+ return regexp.MustCompile(`\[beacon-handoff from=([a-z0-9_-]{1,64}) session=([A-Za-z0-9._:\-]{1,200})\]`)+ })+ handoffHarnessChars = sync.OnceValue(func() *regexp.Regexp { return regexp.MustCompile(`^[a-z0-9_-]{1,64}$`) })+ handoffSessionChars = sync.OnceValue(func() *regexp.Regexp { return regexp.MustCompile(`^[A-Za-z0-9._:\-]{1,200}$`) }))// HandoffMarker returns the marker naming harness's session sessionID, or "" when either value@@ -27,7 +34,7 @@ var (// that parses back differently.func HandoffMarker(harness, sessionID string) string {harness = NormalizeHarnessName(harness)- if !handoffHarnessChars.MatchString(harness) || !handoffSessionChars.MatchString(sessionID) {+ if !handoffHarnessChars().MatchString(harness) || !handoffSessionChars().MatchString(sessionID) {return ""}return fmt.Sprintf("[beacon-handoff from=%s session=%s]", harness, sessionID)@@ -35,9 +42,23 @@ func HandoffMarker(harness, sessionID string) string {// ParseHandoffMarker finds the first handoff marker in text.func ParseHandoffMarker(text string) (HandoffInfo, bool) {- match := handoffMarkerPattern.FindStringSubmatch(text)+ if !strings.Contains(text, handoffMarkerPrefix) {+ return HandoffInfo{}, false+ }+ match := handoffMarkerPattern().FindStringSubmatch(text)if match == nil {return HandoffInfo{}, false}return HandoffInfo{SourceHarness: NormalizeHarnessName(match[1]), SourceSessionID: match[2]}, true}++// handoffMarkerPrefix opens every handoff marker; text without it holds none.+const handoffMarkerPrefix = "[beacon-handoff "++// handoffMarker is the first handoff marker in text, or "".+func handoffMarker(text string) string {+ if !strings.Contains(text, handoffMarkerPrefix) {+ return ""+ }+ return handoffMarkerPattern().FindString(text)+}diff --git a/pkg/asymptoteobserve/lens.go b/pkg/asymptoteobserve/lens.goindex ee61f998..49aef201 100644--- a/pkg/asymptoteobserve/lens.go+++ b/pkg/asymptoteobserve/lens.go@@ -7,6 +7,7 @@ import ("fmt""regexp""strings"+ "sync""time""unicode/utf8")@@ -114,8 +115,8 @@ type LensTokenModelV1 struct {}var (- lensIDPattern = regexp.MustCompile(`^[a-z0-9][a-z0-9-]{0,62}$`)- lensIconPattern = regexp.MustCompile(`^[a-z][a-z0-9-]{0,31}$`)+ lensIDPattern = sync.OnceValue(func() *regexp.Regexp { return regexp.MustCompile(`^[a-z0-9][a-z0-9-]{0,62}$`) })+ lensIconPattern = sync.OnceValue(func() *regexp.Regexp { return regexp.MustCompile(`^[a-z][a-z0-9-]{0,31}$`) })// The manifest element is matched by its exact opening tag. The spec fixes the spelling so that// a reader needs no HTML parser and two readers cannot disagree about which element it is.lensManifestOpen = []byte(`<script type="` + LensManifestScriptType + `">`)@@ -134,13 +135,13 @@ var ErrLensManifestMissing = errors.New("lens has no <script type=\"" + LensMani// Validate reports every way the manifest breaks the spec, joined, or nil.func (m LensManifestV1) Validate() error {var problems []error- if !lensIDPattern.MatchString(m.ID) {+ if !lensIDPattern().MatchString(m.ID) {problems = append(problems, fmt.Errorf("id %q must be 1-63 lowercase letters, digits or hyphens, starting with a letter or digit", m.ID))}problems = append(problems, lensText("title", m.Title, lensTitleMaxRunes, true)...)problems = append(problems, lensText("description", m.Description, lensDescriptionMaxRunes, false)...)problems = append(problems, lensText("author", m.Author, lensAuthorMaxRunes, false)...)- if m.Icon != "" && !lensIconPattern.MatchString(m.Icon) {+ if m.Icon != "" && !lensIconPattern().MatchString(m.Icon) {problems = append(problems, fmt.Errorf("icon %q must be one lowercase word (letters, digits, hyphens)", m.Icon))}if m.Version < 1 {@@ -176,7 +177,7 @@ func lensText(field, value string, maxRunes int, required bool) []error {// ValidLensID reports whether id is a well-formed lens ID. Callers that turn an ID into a file name// check it first, so an ID can never name a path outside the lens store.-func ValidLensID(id string) bool { return lensIDPattern.MatchString(id) }+func ValidLensID(id string) bool { return lensIDPattern().MatchString(id) }// CheckLensFile applies the spec's file-level rules -- the size cap, exactly one manifest, and a// valid manifest -- and returns the manifest. Everything that installs, serves or lints a lensdiff --git a/pkg/asymptoteobserve/lens_test.go b/pkg/asymptoteobserve/lens_test.goindex c0a85ce0..191c5525 100644--- a/pkg/asymptoteobserve/lens_test.go+++ b/pkg/asymptoteobserve/lens_test.go@@ -271,11 +271,11 @@ func TestLensManifestSchemaInSync(t *testing.T) {}props := schema.Properties- if props["id"].Pattern != lensIDPattern.String() {- t.Errorf("id pattern: schema %q, Go %q", props["id"].Pattern, lensIDPattern.String())+ if props["id"].Pattern != lensIDPattern().String() {+ t.Errorf("id pattern: schema %q, Go %q", props["id"].Pattern, lensIDPattern().String())}- if props["icon"].Pattern != lensIconPattern.String() {- t.Errorf("icon pattern: schema %q, Go %q", props["icon"].Pattern, lensIconPattern.String())+ if props["icon"].Pattern != lensIconPattern().String() {+ t.Errorf("icon pattern: schema %q, Go %q", props["icon"].Pattern, lensIconPattern().String())}for field, max := range map[string]int{"title": lensTitleMaxRunes, "description": lensDescriptionMaxRunes, "author": lensAuthorMaxRunes} {if props[field].MaxLength != max {diff --git a/pkg/asymptoteobserve/perf_bench_test.go b/pkg/asymptoteobserve/perf_bench_test.gonew file mode 100644index 00000000..4bbc66ff--- /dev/null+++ b/pkg/asymptoteobserve/perf_bench_test.go@@ -0,0 +1,75 @@+package asymptoteobserve++import (+ "fmt"+ "os"+ "path/filepath"+ "strings"+ "testing"+)++// Benchmarks for the work a hook does on every event. perf_ratchet_test.go holds the deterministic+// ceilings; these give the time.++var benchSink string++func BenchmarkRedactShortFields(b *testing.B) {+ fields := []string{"info", "claude_code", "abc-123", "2026-10-12T03:47:36.515970306Z", "tool.invoked", "/home/user/project", "Bash", "toolu_01"}+ for i := 0; i < b.N; i++ {+ for _, field := range fields {+ benchSink = RedactString(field)+ }+ }+}++func BenchmarkRedactCommand(b *testing.B) {+ command := "cd /home/user/project/cli && go test ./internal/endpoint/... -run TestX -count=1 2>&1 | tail -50"+ for i := 0; i < b.N; i++ {+ benchSink = RedactString(command)+ }+}++func BenchmarkRedactOutput8K(b *testing.B) {+ output := strings.Repeat("ok \tgithub.com/example/project/internal/endpoint\t0.123s // refreshes the api token cache\n", 100)[:8192]+ b.SetBytes(int64(len(output)))+ for i := 0; i < b.N; i++ {+ benchSink = RedactString(output)+ }+}++// BenchmarkRedactCodeWithAssignments is the slow path: source code whose labels are followed by a+// separator, so the label patterns run.+func BenchmarkRedactCodeWithAssignments(b *testing.B) {+ code := strings.Repeat("func load(cfg Config) string { token := cfg.Token; return token }\n", 130)[:8192]+ b.SetBytes(int64(len(code)))+ for i := 0; i < b.N; i++ {+ benchSink = RedactString(code)+ }+}++// benchmarkDedupe checks a candidate against a log whose tail is full, as a long-lived install's+// always is. sessions is how many sessions the tail's lines are spread across.+func benchmarkDedupe(b *testing.B, sessions int) {+ path := filepath.Join(b.TempDir(), "runtime.jsonl")+ line := `{"event":{"action":"tool.invoked","category":"tool","id":"bea06685-774f-5ac2-9c14-2bcb68e4a551"},"gen_ai":{"tool":{"call":{"id":"toolu_%d"}}},"harness":{"collection_method":"hook","name":"claude_code"},"message":"Tool invocation observed","repository":"/home/user/project","sequence":%d,"session":{"id":"session-%d","working_directory":"/home/user/project"},"severity":"info","timestamp":"2026-10-12T03:47:36.515970306Z","tool":{"command":"ls -la","name":"Bash"},"vendor":"beacon"%s}` + "\n"+ output := `,"content":{"text":"` + strings.Repeat(`ok \tgithub.com/example/project/internal/pkg\t0.123s\n`, 30) + `"}`+ var log strings.Builder+ for i := 0; log.Len() < 2*endpointDedupeTailBytes; i++ {+ extra := ""+ if i%2 == 0 {+ extra = output+ }+ fmt.Fprintf(&log, line, i, i, i%sessions, extra)+ }+ if err := os.WriteFile(path, []byte(log.String()), 0o600); err != nil {+ b.Fatal(err)+ }+ candidate := []byte(strings.TrimSpace(fmt.Sprintf(line, -1, 1, 0, "")))+ b.ResetTimer()+ for i := 0; i < b.N; i++ {+ IsDuplicateEndpointEvent(path, candidate, EndpointDuplicateWindow)+ }+}++func BenchmarkDedupeOneSession(b *testing.B) { benchmarkDedupe(b, 1) }+func BenchmarkDedupeEightSessions(b *testing.B) { benchmarkDedupe(b, 8) }diff --git a/pkg/asymptoteobserve/perf_ratchet_test.go b/pkg/asymptoteobserve/perf_ratchet_test.gonew file mode 100644index 00000000..daa09eb5--- /dev/null+++ b/pkg/asymptoteobserve/perf_ratchet_test.go@@ -0,0 +1,68 @@+package asymptoteobserve++import (+ "os"+ "os/exec"+ "regexp"+ "strconv"+ "strings"+ "testing"+)++// The tests in this file are ceilings on work that is deterministic, so they hold the hook path's+// speed without timing anything. A hook is a process per event that the agent waits on, and both+// of the costs below were paid by every one of them before they were measured.+//+// When a change lowers a measured value well below its ceiling, lower the ceiling with it, so the+// gain cannot be given back unnoticed.++// initAllocCeiling bounds the allocations this package's initialization makes, test files+// included (409 when it was set). Compiling the redaction patterns at init made it 11,508: a+// package-level regexp.MustCompile belongs behind sync.OnceValue.+const initAllocCeiling = 500++func TestPackageInitStaysCheap(t *testing.T) {+ if os.Getenv("BEACON_INIT_TRACE_CHILD") != "" {+ return+ }+ cmd := exec.Command(os.Args[0], "-test.run=^TestPackageInitStaysCheap$")+ cmd.Env = append(os.Environ(), "GODEBUG=inittrace=1", "BEACON_INIT_TRACE_CHILD=1")+ out, err := cmd.CombinedOutput()+ if err != nil {+ t.Fatalf("re-run test binary: %v\n%s", err, out)+ }+ match := regexp.MustCompile(`(?m)^init github\.com/asymptote-labs/agent-beacon/pkg/asymptoteobserve @.* (\d+) allocs$`).FindSubmatch(out)+ if match == nil {+ t.Skipf("no init trace for this package in:\n%s", out)+ }+ allocs, _ := strconv.Atoi(string(match[1]))+ if allocs > initAllocCeiling {+ t.Fatalf("package init made %d allocations, ceiling %d; compile patterns and build tables on first use", allocs, initAllocCeiling)+ }+}++// TestOrdinaryTextReachesNoRedactionPattern holds the redaction gate: text with no credential+// shape in it, which is nearly all of what a hook redacts, must not run a single pattern.+func TestOrdinaryTextReachesNoRedactionPattern(t *testing.T) {+ ordinary := []string{+ "info", "tool.invoked", "claude_code", "abc-123", "2026-10-12T03:47:36.515970306Z",+ "/home/user/project/internal/auth/config.go",+ "cd /home/user/project && go test ./... -count=1 2>&1 | tail -50",+ strings.Repeat("ok \tgithub.com/example/project/internal/pkg\t0.123s // refreshes the api token cache\n", 100),+ "input_tokens: 812, output_tokens: 90, cache_read_input_tokens: 4000",+ "Please refactor the token refresh logic so the api key is read from the environment.",+ "func load(cfg Config) string { return cfg.Endpoint }",+ "keyboard shortcuts, passport photos, secretary notes",+ }+ for _, text := range ordinary {+ gate := redactGate{value: text, enabled: true}+ if gate.holds(redactLabelNeedles) {+ t.Errorf("%.60q reaches the assignment pattern", text)+ }+ for i, rule := range redactionRules {+ if gate.holds(rule.needles) {+ t.Errorf("%.60q reaches redaction rule %d (%s)", text, i, rule.pattern())+ }+ }+ }+}diff --git a/pkg/asymptoteobserve/privacy.go b/pkg/asymptoteobserve/privacy.goindex a55618dc..32eb8703 100644--- a/pkg/asymptoteobserve/privacy.go+++ b/pkg/asymptoteobserve/privacy.go@@ -4,6 +4,7 @@ import ("encoding/json""regexp""strings"+ "sync")const (@@ -68,13 +69,22 @@ const (// argument is a constant naming a credential (the pattern spells out "password") as returning that// credential, and these rules hold patterns, not secrets.func quotedRules(kind quotedKind, keep func(string) bool) []redactionRule {- prefix := `\b` + redactLabel + redactSeparator+ prefix, words, separator, within := `\b`+redactLabel+redactSeparator, redactLabelWords, skipSeparator, redactLabelNeedlesif kind == quotedFlag {- prefix = redactFlag+ prefix, words, separator, within = redactFlag, redactFlagWords, skipFlagSeparator, redactFlagNeedles}rules := make([]redactionRule, 0, len(quotedValues))for _, quoted := range quotedValues {- rules = append(rules, redactionRule{regexp.MustCompile(`(?i)(` + prefix + quoted[0] + `)(?P<value>` + quoted[1] + `)`), "${1}" + redactedValue, keep})+ // The pattern opens the value with this one quote right after the separator, so text+ // where no label is followed by it cannot match.+ open := strings.ReplaceAll(quoted[0], `\\`, `\`)+ needles := newRedactNeedles(true, words...)+ needles.within = within+ needles.followedBy = func(rest string) bool {+ rest, ok := separator(rest)+ return ok && strings.HasPrefix(rest, open)+ }+ rules = append(rules, lazyRule(`(?i)(`+prefix+quoted[0]+`)(?P<value>`+quoted[1]+`)`, "${1}"+redactedValue, keep, needles))}return rules}@@ -95,12 +105,161 @@ const redactFlag = `(?:^|\s)--?[a-z0-9-]*(?:password|passwd|passphrase|token|sec// redactionRule is one redaction. ${1} and ${3}, when the pattern has them, are kept around the// value.type redactionRule struct {- pattern *regexp.Regexp+ pattern func() *regexp.Regexpreplacement string// keep, when set, leaves a match alone if its value (the group named value) passes.keep func(value string) bool+ // needles are literals the pattern cannot match without: when the text holds none of them,+ // the rule is skipped without running the pattern.+ needles *redactNeedles+}++// redactNeedles is a set of literals a pattern cannot match without. folded needles are lower+// case and compared against the lower-cased text, for a case-insensitive pattern. followedBy, when+// set, must also hold for the text right after the needle. within, when set, is a broader set the+// text must also hold, searched first: it is what keeps a family of narrow sets to one search on+// text that holds none of them.+type redactNeedles struct {+ id uint+ folded bool+ needles []string+ followedBy func(rest string) bool+ precededBy func(before string) bool+ within *redactNeedles+}++var redactNeedleSets uint++func newRedactNeedles(folded bool, needles ...string) *redactNeedles {+ if len(needles) == 0 {+ panic("asymptoteobserve: a redaction rule needs at least one needle")+ }+ if redactNeedleSets >= 64 {+ panic("asymptoteobserve: redactGate tracks at most 64 needle sets")+ }+ set := &redactNeedles{id: redactNeedleSets, folded: folded, needles: needles}+ redactNeedleSets+++ return set+}++// rule is a case-sensitive rule that cannot match text holding none of needles.+func rule(expr, replacement string, keep func(string) bool, needles ...string) redactionRule {+ return lazyRule(expr, replacement, keep, newRedactNeedles(false, needles...))+}++// foldedRule is a case-insensitive rule that cannot match text whose lower-cased form holds none+// of needles, which must be lower case.+func foldedRule(expr, replacement string, keep func(string) bool, needles ...string) redactionRule {+ return lazyRule(expr, replacement, keep, newRedactNeedles(true, needles...))+}++// lazyRule compiles its pattern the first time text gets past the needles. A hook is a process+// per event, and most events never need most rules: compiling all of them up front cost every+// hook run a few milliseconds before it had read its input.+func lazyRule(expr, replacement string, keep func(string) bool, needles *redactNeedles) redactionRule {+ return redactionRule{pattern: sync.OnceValue(func() *regexp.Regexp { return regexp.MustCompile(expr) }), replacement: replacement, keep: keep, needles: needles}+}++// redactLabelWords are the words every redactLabel match ends in, lower-cased: each marker in its+// alternation ends in one of them (api_key in "key", client_secret in "secret").+var redactLabelWords = []string{"key", "secret", "token", "password", "passwd", "passphrase", "authorization", "credential", "credentials"}++// redactLabelNeedles gate the rules that take a label and redactSeparator. A word that is not+// followed by a separator (`input_tokens`, `keyboard`) cannot start a match. The rules share the+// one set, so the gate searches for it once per text rather than once per rule.+var redactLabelNeedles = func() *redactNeedles {+ set := newRedactNeedles(true, redactLabelWords...)+ set.followedBy = func(rest string) bool {+ _, ok := skipSeparator(rest)+ return ok+ }+ return set+}()++// redactQuotedLabelNeedles gate the rule whose label is closed by a quote before its separator.+var redactQuotedLabelNeedles = func() *redactNeedles {+ set := newRedactNeedles(true, redactLabelWords...)+ set.within = redactLabelNeedles+ set.followedBy = func(rest string) bool {+ rest = strings.TrimPrefix(rest, `\`)+ if rest == "" || !strings.ContainsRune(redactQuotes, rune(rest[0])) {+ return false+ }+ _, ok := skipSeparator(rest)+ return ok+ }+ return set+}()++// redactFlagWords are the words every redactFlag match ends in, lower-cased.+var redactFlagWords = []string{"password", "passwd", "passphrase", "token", "secret", "key"}++// redactFlagNeedles gate the flag rules: a flag's name is followed by `=` or whitespace, and is the+// end of a word that starts with a dash at the start of the text or after whitespace.+var redactFlagNeedles = func() *redactNeedles {+ set := newRedactNeedles(true, redactFlagWords...)+ set.followedBy = func(rest string) bool {+ _, ok := skipFlagSeparator(rest)+ return ok+ }+ set.precededBy = startsFlag+ return set+}()++// startsFlag reports whether the text before a flag word ends in what redactFlag puts ahead of it:+// `(?:^|\s)--?[a-z0-9-]*`. That run of letters, digits and dashes can only begin where the+// maximal one does, because the character before it must be whitespace, which the run excludes.+func startsFlag(before string) bool {+ start := len(before)+ for start > 0 {+ c := before[start-1]+ if !(c >= 'a' && c <= 'z' || c >= '0' && c <= '9' || c == '-') {+ break+ }+ start--+ }+ if start == len(before) || before[start] != '-' {+ return false+ }+ return start == 0 || strings.ContainsRune(regexpSpace, rune(before[start-1]))+}++// redactQuotes are the quote characters redactSeparator and quotedValues know.+const redactQuotes = "\"'`"++// skipSeparator consumes what redactSeparator matches, `(?:\\?["'`])?\s*[:=]\s*`, from the start+// of rest. Taking all the whitespace is what the pattern does too: nothing that may follow it can+// start with whitespace.+func skipSeparator(rest string) (string, bool) {+ if strings.HasPrefix(rest, `\`) {+ if len(rest) < 2 || !strings.ContainsRune(redactQuotes, rune(rest[1])) {+ return "", false+ }+ rest = rest[2:]+ } else if rest != "" && strings.ContainsRune(redactQuotes, rune(rest[0])) {+ rest = rest[1:]+ }+ rest = strings.TrimLeft(rest, regexpSpace)+ if rest == "" || (rest[0] != ':' && rest[0] != '=') {+ return "", false+ }+ return strings.TrimLeft(rest[1:], regexpSpace), true+}++// skipFlagSeparator consumes the separator ending redactFlag, `(?:=|\s+)`, from the start of rest.+func skipFlagSeparator(rest string) (string, bool) {+ if strings.HasPrefix(rest, "=") {+ return rest[1:], true+ }+ if trimmed := strings.TrimLeft(rest, regexpSpace); len(trimmed) < len(rest) {+ return trimmed, true+ }+ return "", false}+// regexpSpace is \s in Go's regexp syntax: ASCII whitespace, never Unicode.+const regexpSpace = " \t\n\f\r"+// isAuthScheme reports whether a labeled value is an Authorization scheme, alone or with its// credential already redacted, rather than a credential. `Authorization: Bearer x` is redacted to// `Authorization: Bearer [REDACTED]` by the header rule, and the assignment rules must not then@@ -136,14 +295,14 @@ var redactionRules = concatRules([]redactionRule{// A private key block. The header must be followed by key material, so source code, a diff or// grep output that only names the header keeps the text after it. Without an END line (the// block was cut) the rest of the text goes.- {regexp.MustCompile(`-----BEGIN [A-Z0-9 ]*PRIVATE KEY( BLOCK)?-----(?:\s|\\[rn]|(?:Proc-Type|DEK-Info|Comment|Version):[^\n\\]*)*` +- `[A-Za-z0-9+/=]{16,}[\s\S]*?(?:-----END [A-Z0-9 ]*PRIVATE KEY( BLOCK)?-----|$)`), redactedValue, nil},+ rule(`-----BEGIN [A-Z0-9 ]*PRIVATE KEY( BLOCK)?-----(?:\s|\\[rn]|(?:Proc-Type|DEK-Info|Comment|Version):[^\n\\]*)*`++ `[A-Za-z0-9+/=]{16,}[\s\S]*?(?:-----END [A-Z0-9 ]*PRIVATE KEY( BLOCK)?-----|$)`, redactedValue, nil, "-----BEGIN "),// An Authorization header's credential after its scheme: Bearer, Basic, Token, Digest. A JWT// keeps its public `eyJ` prefix, as the shape rules below keep theirs, so `Bearer eyJ[REDACTED]`// still says a token was there where `Bearer [REDACTED]` may only have been the word "token" or// a `$VARIABLE`.- {regexp.MustCompile(`(?i)(authorization["']?\s*[:=]\s*["']?(?:bearer|basic|token|digest)\s+(?-i:eyJ)?)([^"'\s,]+)()`), "${1}" + redactedValue, nil},- {regexp.MustCompile(`(?i)(\bbearer\s+(?-i:eyJ)?)(?P<value>[a-z0-9._~+/=-]+)`), "${1}" + redactedValue, isRedactedJWTPrefix},+ foldedRule(`(?i)(authorization["']?\s*[:=]\s*["']?(?:bearer|basic|token|digest)\s+(?-i:eyJ)?)([^"'\s,]+)()`, "${1}"+redactedValue, nil, "authorization"),+ foldedRule(`(?i)(\bbearer\s+(?-i:eyJ)?)(?P<value>[a-z0-9._~+/=-]+)`, "${1}"+redactedValue, isRedactedJWTPrefix, "bearer"),// A labeled assignment in prose, shell, env files, YAML, TOML or JSON: `token=x`,// `export GITHUB_TOKEN="x"`, `{"api_key": "x"}`. The closing quote of a quoted label is allowed// between the label and the separator. A quoted value runs to its closing quote; the unquoted@@ -153,30 +312,30 @@ var redactionRules = concatRules([]redactionRule{// literal or a structure. The replacement is quoted the way the label is, so the text still// parses where something decodes it again (string-encoded tool arguments do); null, true, false// and nested objects or arrays are left for their own fields' rules.- {regexp.MustCompile(`(?i)(\b` + redactLabel + `(?P<q>` + redactOpenQuote + `)\s*[:=]\s*)(?P<value>` + redactBare + `)`), "${1}${q}" + redactedValue + "${q}", isLiteralOrAuthScheme},- {regexp.MustCompile(`(?i)(\b` + redactLabel + `\s*[:=]\s*)(?P<value>` + redactUnquoted + `)`), "${1}" + redactedValue, isAuthScheme},+ lazyRule(`(?i)(\b`+redactLabel+`(?P<q>`+redactOpenQuote+`)\s*[:=]\s*)(?P<value>`+redactBare+`)`, "${1}${q}"+redactedValue+"${q}", isLiteralOrAuthScheme, redactQuotedLabelNeedles),+ lazyRule(`(?i)(\b`+redactLabel+`\s*[:=]\s*)(?P<value>`+redactUnquoted+`)`, "${1}"+redactedValue, isAuthScheme, redactLabelNeedles),// A command-line flag: `--password x`, `--token=x`, `--api-key x`, and quoted, the way agents// usually write arguments: `--password "x y"`, `--token='x'`.}, quotedRules(quotedFlag, nil), []redactionRule{- {regexp.MustCompile(`(?i)(` + redactFlag + `)([^\s"'-][^\s"']*)`), "${1}" + redactedValue, nil},+ lazyRule(`(?i)(`+redactFlag+`)([^\s"'-][^\s"']*)`, "${1}"+redactedValue, nil, redactFlagNeedles),// The password in a URL's userinfo: `postgres://user:pw@host`, `https://x-access-token:[email protected]`.- {regexp.MustCompile(`(?i)(\b[a-z][a-z0-9+.-]*://[^/\s:@"'<>]+:)([^@/\s"'<>]+)(@)`), "${1}" + redactedValue + "${3}", nil},+ foldedRule(`(?i)(\b[a-z][a-z0-9+.-]*://[^/\s:@"'<>]+:)([^@/\s"'<>]+)(@)`, "${1}"+redactedValue+"${3}", nil, "://"),// Credentials recognizable by their own shape, wherever they appear. The type prefix is public// and stays (`ghp_[REDACTED]`, `AKIA[REDACTED]`), the way secret scanners report a finding: the// line still says a GitHub token or an AWS key was there, which is what detection rules match on// in the log, and none of the credential is left.- {regexp.MustCompile(`\b(sk-)[A-Za-z0-9_-]{20,}`), "${1}" + redactedValue, nil}, // Anthropic sk-ant-, OpenAI sk-proj- and sk-- {regexp.MustCompile(`\b(AKIA|ASIA|ABIA|ACCA)[A-Z0-9]{16}\b`), "${1}" + redactedValue, nil}, // AWS access key id- {regexp.MustCompile(`\b(gh[pousr]_)[A-Za-z0-9]{30,}`), "${1}" + redactedValue, nil}, // GitHub tokens- {regexp.MustCompile(`\b(github_pat_)[A-Za-z0-9_]{22,}`), "${1}" + redactedValue, nil}, // GitHub fine-grained tokens- {regexp.MustCompile(`\b(glpat-)[A-Za-z0-9_-]{20,}`), "${1}" + redactedValue, nil}, // GitLab- {regexp.MustCompile(`\b(xox[abposr]-)[A-Za-z0-9-]{10,}`), "${1}" + redactedValue, nil}, // Slack- {regexp.MustCompile(`\b([rs]k_(?:live|test)_)[A-Za-z0-9]{16,}`), "${1}" + redactedValue, nil}, // Stripe- {regexp.MustCompile(`\b(AIza)[0-9A-Za-z_-]{35}`), "${1}" + redactedValue, nil}, // Google API key- {regexp.MustCompile(`\b(npm_)[A-Za-z0-9]{36}`), "${1}" + redactedValue, nil}, // npm- {regexp.MustCompile(`\b(hf_)[A-Za-z0-9]{30,}`), "${1}" + redactedValue, nil}, // Hugging Face- {regexp.MustCompile(`\b(bcn_(?:api|device|mcp)_)[A-Za-z0-9_-]{8,}`), "${1}" + redactedValue, nil}, // Beacon's own keys- {regexp.MustCompile(`\b(eyJ)[A-Za-z0-9_-]{8,}\.eyJ[A-Za-z0-9_-]{8,}\.[A-Za-z0-9_-]{8,}`), "${1}" + redactedValue, nil}, // JWT+ rule(`\b(sk-)[A-Za-z0-9_-]{20,}`, "${1}"+redactedValue, nil, "sk-"), // Anthropic sk-ant-, OpenAI sk-proj- and sk-+ rule(`\b(AKIA|ASIA|ABIA|ACCA)[A-Z0-9]{16}\b`, "${1}"+redactedValue, nil, "AKIA", "ASIA", "ABIA", "ACCA"), // AWS access key id+ rule(`\b(gh[pousr]_)[A-Za-z0-9]{30,}`, "${1}"+redactedValue, nil, "ghp_", "gho_", "ghu_", "ghs_", "ghr_"), // GitHub tokens+ rule(`\b(github_pat_)[A-Za-z0-9_]{22,}`, "${1}"+redactedValue, nil, "github_pat_"), // GitHub fine-grained tokens+ rule(`\b(glpat-)[A-Za-z0-9_-]{20,}`, "${1}"+redactedValue, nil, "glpat-"), // GitLab+ rule(`\b(xox[abposr]-)[A-Za-z0-9-]{10,}`, "${1}"+redactedValue, nil, "xox"), // Slack+ rule(`\b([rs]k_(?:live|test)_)[A-Za-z0-9]{16,}`, "${1}"+redactedValue, nil, "k_live_", "k_test_"), // Stripe+ rule(`\b(AIza)[0-9A-Za-z_-]{35}`, "${1}"+redactedValue, nil, "AIza"), // Google API key+ rule(`\b(npm_)[A-Za-z0-9]{36}`, "${1}"+redactedValue, nil, "npm_"), // npm+ rule(`\b(hf_)[A-Za-z0-9]{30,}`, "${1}"+redactedValue, nil, "hf_"), // Hugging Face+ rule(`\b(bcn_(?:api|device|mcp)_)[A-Za-z0-9_-]{8,}`, "${1}"+redactedValue, nil, "bcn_"), // Beacon's own keys+ rule(`\b(eyJ)[A-Za-z0-9_-]{8,}\.eyJ[A-Za-z0-9_-]{8,}\.[A-Za-z0-9_-]{8,}`, "${1}"+redactedValue, nil, "eyJ"), // JWT})func concatRules(groups ...[]redactionRule) []redactionRule {@@ -188,22 +347,23 @@ func concatRules(groups ...[]redactionRule) []redactionRule {}func (rule redactionRule) apply(value string) string {+ pattern := rule.pattern()if rule.keep == nil {- return rule.pattern.ReplaceAllString(value, rule.replacement)+ return pattern.ReplaceAllString(value, rule.replacement)}- index := rule.pattern.SubexpIndex("value")+ index := pattern.SubexpIndex("value")if index < 0 {index = 2}group := 2 * indexvar out []bytelast := 0- for _, match := range rule.pattern.FindAllStringSubmatchIndex(value, -1) {+ for _, match := range pattern.FindAllStringSubmatchIndex(value, -1) {if match[group] >= 0 && rule.keep(value[match[group]:match[group+1]]) {continue}out = append(out, value[last:match[0]]...)- out = rule.pattern.ExpandString(out, rule.replacement, value, match)+ out = pattern.ExpandString(out, rule.replacement, value, match)last = match[1]}if out == nil {@@ -215,7 +375,9 @@ func (rule redactionRule) apply(value string) string {// redactAssignedPattern captures the value of a labeled assignment, quoted with ", ' or a backtick// (${1} to ${3}) or not (${4}), so a bare copy of it elsewhere in the same text is// removed too.-var redactAssignedPattern = regexp.MustCompile(`(?i)\b` + redactLabel + redactSeparator + `(?:` + quotedAlternatives() + `|(` + redactUnquoted + `))`)+var redactAssignedPattern = sync.OnceValue(func() *regexp.Regexp {+ return regexp.MustCompile(`(?i)\b` + redactLabel + redactSeparator + `(?:` + quotedAlternatives() + `|(` + redactUnquoted + `))`)+})// quotedAlternatives is every quotedValues entry as one capturing alternative.func quotedAlternatives() string {@@ -237,20 +399,39 @@ type PrivacyOptions struct {}func RedactString(value string) string {+ return redactString(value, true)+}++// redactString is RedactString; gated false runs every rule whatever the text holds, which the+// tests use to prove the gate never changes a result.+//+// The gate is what keeps redaction affordable on the hook path, where it runs over every string of+// every event before the agent may continue. Go's regexp engine has no DFA, so the case-insensitive+// label rules cost microseconds on a short string and tens of milliseconds on a tool output of a+// few kilobytes. Almost no text holds a needle of most rules, and a substring search for one is+// orders of magnitude cheaper than the pattern it stands in front of.+func redactString(value string, gated bool) string {+ text := redactGate{value: value, enabled: gated}// Remember assignment values before replacing their labeled occurrence so// repeated bare copies in the same retained prompt/output are also removed.// This is common in model reasoning that quotes `token=value` and later// refers to `value` by itself.var assigned []string- for _, match := range redactAssignedPattern.FindAllStringSubmatch(value, -1) {- copied := strings.Join(match[1:], "")- if len(copied) >= 8 && copied != redactedValue && !isAuthScheme(copied) {- assigned = append(assigned, copied)+ if text.holds(redactLabelNeedles) {+ for _, match := range redactAssignedPattern().FindAllStringSubmatch(value, -1) {+ copied := strings.Join(match[1:], "")+ if len(copied) >= 8 && copied != redactedValue && !isAuthScheme(copied) {+ assigned = append(assigned, copied)+ }}}for _, rule := range redactionRules {- value = rule.apply(value)+ if !text.holds(rule.needles) {+ continue+ }+ text.set(rule.apply(text.value))}+ value = text.value// Each copy is searched for and removed; none of it survives into the result.for _, copied := range assigned {value = strings.ReplaceAll(value, copied, redactedValue)@@ -258,6 +439,70 @@ func RedactString(value string) string {return value}+// redactGate answers whether text holds any of a set of needles, lower-casing it and searching for+// each set at most once per version of the text.+type redactGate struct {+ value string+ enabled bool+ lower string+ lowered bool+ // known and present are bit sets over redactNeedles ids: which sets were searched for in this+ // version of the text, and which were found.+ known, present uint64+ // unfoldable is set when the text holds a character Go's case folding equates with an ASCII+ // letter (U+017F folds to s, U+212A to k): a case-insensitive pattern can match "ſecret", which+ // lower-casing does not turn into "secret", so folded rules always run on such text.+ unfoldable bool+}++func (g *redactGate) set(value string) {+ if value != g.value {+ g.value, g.lowered, g.known, g.present = value, false, 0, 0+ }+}++func (g *redactGate) holds(set *redactNeedles) bool {+ if !g.enabled {+ return true+ }+ if set.within != nil && !g.holds(set.within) {+ return false+ }+ bit := uint64(1) << set.id+ if g.known&bit != 0 {+ return g.present&bit != 0+ }+ g.known |= bit+ haystack := g.value+ if set.folded {+ if !g.lowered {+ g.lower, g.lowered = strings.ToLower(g.value), true+ g.unfoldable = strings.ContainsAny(g.value, "\u017f\u212a")+ }+ if g.unfoldable {+ g.present |= bit+ return true+ }+ haystack = g.lower+ }+ for _, needle := range set.needles {+ for from := 0; ; {+ i := strings.Index(haystack[from:], needle)+ if i < 0 {+ break+ }+ start, end := from+i, from+i+len(needle)+ // The next search starts one byte on, not at end: an occurrence may overlap this one.+ from = start + 1+ if (set.precededBy == nil || set.precededBy(haystack[:start])) && (set.followedBy == nil || set.followedBy(haystack[end:])) {+ g.present |= bit+ return true+ }+ }+ }+ return false+}+// isSecretKeyName reports whether a structured field's name says its value is a credential:// `password`, `api_key`, `apiKey`, `X-Api-Key`, `GITHUB_TOKEN`, `AWS_SECRET_ACCESS_KEY`,// `client_secret`, `Cookie`. The name must end in the marker, so `input_tokens`, `token_type`,diff --git a/pkg/asymptoteobserve/privacy_gate_test.go b/pkg/asymptoteobserve/privacy_gate_test.gonew file mode 100644index 00000000..3f924f57--- /dev/null+++ b/pkg/asymptoteobserve/privacy_gate_test.go@@ -0,0 +1,75 @@+package asymptoteobserve++import (+ "math/rand"+ "strings"+ "testing"+)++// redactGateFragments are the pieces the randomized gate test builds text from: every rule's+// needles and the text around them, near misses, and the case-folding edge cases.+var redactGateFragments = []string{+ "token", "Token", "TOKEN", "tokens", "input_tokens", "api_key", "API-KEY", "apikey", "access_key",+ "secret", "client_secret", "secret_key", "private-key", "password", "PassWord", "passwd",+ "passphrase", "authorization", "Authorization", "credential", "credentials", "keyboard", "key",+ "GITHUB_TOKEN", "export ", "--token", "--db-password", "-x-token", "a-token", "9", "_", "-password", "--api-key", "--access-key", "-x", "--", "-",+ "=", ":", " ", " ", "\t", "\n", "\r", ",", `"`, "'", "`", `\`, `\"`, `\\\"`, "{", "}", "[", "]",+ "Bearer ", "bearer", "Basic ", "Digest ", "eyJ", "eyJhbGciOiJIUzI1NiJ9", ".", "abcdefgh12345678",+ "hunter2hunter2", "null", "true", "42", "https://", "postgres://user:", "pw", "@host", "://",+ "sk-", "sk-ant-api03-", "AKIA", "ASIA", "ABCDEFGHIJKLMNOP", "ghp_", "gho_", "ghs_", "github_pat_",+ "glpat-", "xoxb-", "sk_live_", "rk_test_", "AIza", "npm_", "hf_", "bcn_api_", "bcn_device_",+ "0123456789abcdefghijklmnopqrstuvwxyzABCDEFGHIJ", "-----BEGIN RSA PRIVATE KEY-----",+ "-----BEGIN PRIVATE KEY-----", "-----END RSA PRIVATE KEY-----", "MIIEvQIBADANBgkqhkiG9w0BAQEFAASC",+ "Proc-Type: 4,ENCRYPTED", "\\n", "[REDACTED]", "ſecret", "toKen", "Key", "İ", "ß", "é", "日本",+}++func randomRedactInput(r *rand.Rand) string {+ var b strings.Builder+ for n := r.Intn(12); n >= 0; n-- {+ b.WriteString(redactGateFragments[r.Intn(len(redactGateFragments))])+ }+ return b.String()+}++// TestRedactGateNeverChangesAResult proves the gate is only a shortcut: every text redacts exactly+// as it does when every rule runs.+func TestRedactGateNeverChangesAResult(t *testing.T) {+ inputs := []string{+ "", "info", "token=abcdefgh12345678", `{"api_key": "x y z"}`, `"password": 42`,+ "export GITHUB_TOKEN=ghp_0123456789abcdefghijklmnopqrstuvwxyz",+ "curl -H 'Authorization: Bearer eyJhbGciOiJIUzI1NiJ9.eyJzdWIiOiIxMjM0NTY3ODkwIn0.abcdefghijk'",+ "mysql --password hunter2 -u root", "postgres://user:pw@host/db", "input_tokens: 812",+ "ſecret=abcdefgh12345678", "KEY=abcdefgh12345678", "toKen\\\"=\"abc\"",+ "-----BEGIN RSA PRIVATE KEY-----\nMIIEvQIBADANBgkqhkiG9w0BAQEFAASC\n-----END RSA PRIVATE KEY-----",+ }+ r := rand.New(rand.NewSource(1))+ for i := 0; i < 20000; i++ {+ inputs = append(inputs, randomRedactInput(r))+ }+ redacted := 0+ for _, input := range inputs {+ gated, all := redactString(input, true), redactString(input, false)+ if gated != all {+ t.Fatalf("gated redaction of %q = %q, every rule gives %q", input, gated, all)+ }+ if gated != input {+ redacted+++ }+ }+ // The corpus has to reach the rules for the comparison to mean anything.+ if redacted < len(inputs)/10 {+ t.Fatalf("only %d of %d inputs were redacted at all", redacted, len(inputs))+ }+}++func FuzzRedactGate(f *testing.F) {+ r := rand.New(rand.NewSource(2))+ for i := 0; i < 64; i++ {+ f.Add(randomRedactInput(r))+ }+ f.Fuzz(func(t *testing.T, input string) {+ if gated, all := redactString(input, true), redactString(input, false); gated != all {+ t.Fatalf("gated redaction of %q = %q, every rule gives %q", input, gated, all)+ }+ })+}diff --git a/pkg/asymptoteobserve/share.go b/pkg/asymptoteobserve/share.goindex 2d31ecff..1f48044b 100644--- a/pkg/asymptoteobserve/share.go+++ b/pkg/asymptoteobserve/share.go@@ -8,6 +8,7 @@ import ("regexp""sort""strings"+ "sync""time")@@ -153,11 +154,11 @@ type ShareBatchResultV1 struct {EventCount int `json:"event_count"`}-var shareExternalIDPattern = regexp.MustCompile(`^[A-Za-z0-9][A-Za-z0-9._:-]{0,199}$`)+var shareExternalIDPattern = sync.OnceValue(func() *regexp.Regexp { return regexp.MustCompile(`^[A-Za-z0-9][A-Za-z0-9._:-]{0,199}$`) })// ValidShareExternalID reports whether id is a well-formed external id.func ValidShareExternalID(id string) bool {- return shareExternalIDPattern.MatchString(id)+ return shareExternalIDPattern().MatchString(id)}// ShareExternalID is the client-chosen id of a session's shared trace: `<harness>:<session-id>`@@ -751,7 +752,7 @@ func scrubTyped[T any](s pathScrubber, input *T) *T {// ShareTitle is a one-line title of at most ShareTitleMaxRunes runes from prompt text, without the// handoff marker a resumed session's first prompt carries.func ShareTitle(text string) string {- if marker := handoffMarkerPattern.FindString(text); marker != "" {+ if marker := handoffMarker(text); marker != "" {text = strings.ReplaceAll(text, marker, " ")}text = strings.Join(strings.Fields(text), " ")