Every agent's model. One place. Codex on DeepSeek, Claude Code on Kimi, from the menu bar.
redact: a long string already scrubbed is not run through the rules again
yetoneb36819d349ca
diff --git a/internal/redact/known.go b/internal/redact/known.goindex 23f2ea6a8..edcab4b49 100644--- a/internal/redact/known.go+++ b/internal/redact/known.go@@ -3,6 +3,7 @@ package redactimport ("slices""strings"+ "sync/atomic")// A value masked once is masked again wherever it is in a request, as it@@ -33,6 +34,9 @@ var (// knownBits has a bit for the first four bytes of each known value, so// most places in a text are passed over without a map lookupknownBits [1 << 14]uint64+ // knownGen changes whenever known does, with mu held; what Scrub+ // remembers is good only at the knownGen it was made at+ knownGen atomic.Uint64)func gram(s string, i int) uint32 {@@ -55,6 +59,7 @@ func addKnown(p, v string) {// first, so it is the same after reindexKnownif p < k.p {known[head][i] = knownValue{v, p, kind}+ knownGen.Add(1)}return}@@ -62,6 +67,7 @@ func addKnown(p, v string) {known[head] = append(known[head], knownValue{v, p, kind})g := gram(v, 0)knownBits[g/64] |= 1 << (g % 64)+ knownGen.Add(1)}// reindexKnown makes known again from values, after some were let go;@@ -69,6 +75,7 @@ func addKnown(p, v string) {func reindexKnown() {known = map[string][]knownValue{}knownBits = [1 << 14]uint64{}+ knownGen.Add(1)for p, v := range values {addKnown(p, v)}diff --git a/internal/redact/scrub.go b/internal/redact/scrub.goindex 8f0082633..1737b4d69 100644--- a/internal/redact/scrub.go+++ b/internal/redact/scrub.go@@ -1,8 +1,11 @@package redactimport (+ "crypto/sha256""regexp""strings"+ "sync"+ "sync/atomic")// Scrubbing is for what is kept rather than sent — the request archive: a@@ -28,10 +31,109 @@ func SecretName(name string) bool {// Scrub takes the secrets Mask finds out of s.func Scrub(s string) string {+ if len(s) < scrubMemoMin {+ return scrub(s)+ }+ // What Scrub makes of s depends on s and on the values known (known.go)+ // alone: a result is kept under the knownGen it was made at, and none+ // made before a value became known is used again.+ gen := knownGen.Load()+ k := sha256.Sum256([]byte(s))+ if out, ok := scrubMemo.get(k, gen, s); ok {+ return out+ }+ out := scrub(s)+ scrubMemo.put(k, gen, s, out)+ return out+}++func scrub(s string) string {+ if len(s) >= 256 {+ scrubRuns.Add(1)+ }out, _ := mask(s, Options{Secrets: true}, scrubbed)return out}+// scrubRuns counts the strings of 256 bytes or more the rules have run+// over (a test reads it).+var scrubRuns atomic.Uint64++// Scrub remembers what it made of a long string. An agent sends its whole+// history on every turn (Codex: every item of a session, each turn), and+// what is kept of each request (the request archive, a gateway+// conversation) is scrubbed whole, so without it every turn ran each rule+// over the whole history again: a core's worth while Codex ran+// (yetone/magpie-releases#24). A short string costs less to scrub than to+// look up. Only a hash of a string is kept, and what it became when that+// is not the string itself.+const (+ scrubMemoMin = 256+ scrubMemoEntries = 1 << 15+ scrubMemoBytes = 32 << 20+)++type scrubEntry struct {+ gen uint64+ same bool // s came back as it was+ out string // else what it became+}++// scrubMemo holds two generations of results: when the newer is full, it+// becomes the older and the older goes, so what is in use stays and at+// most twice the bounds are held.+var scrubMemo memo++type memo struct {+ sync.Mutex+ cur, old map[[32]byte]scrubEntry+ bytes int+}++func (m *memo) get(k [32]byte, gen uint64, s string) (string, bool) {+ m.Lock()+ defer m.Unlock()+ e, ok := m.cur[k]+ if !ok {+ if e, ok = m.old[k]; ok && e.gen == gen {+ m.add(k, e)+ }+ }+ if !ok || e.gen != gen {+ return "", false+ }+ if e.same {+ return s, true+ }+ return e.out, true+}++func (m *memo) put(k [32]byte, gen uint64, s, out string) {+ e := scrubEntry{gen: gen, same: out == s}+ if !e.same {+ e.out = strings.Clone(out)+ }+ m.Lock()+ defer m.Unlock()+ m.add(k, e)+}++// reset forgets everything (a test starts from nothing).+func (m *memo) reset() {+ m.Lock()+ defer m.Unlock()+ m.cur, m.old, m.bytes = nil, nil, 0+}++// add is called with m held.+func (m *memo) add(k [32]byte, e scrubEntry) {+ if m.cur == nil || len(m.cur) >= scrubMemoEntries || m.bytes+len(e.out) > scrubMemoBytes {+ m.old, m.cur, m.bytes = m.cur, make(map[[32]byte]scrubEntry), 0+ }+ m.bytes += len(e.out) - len(m.cur[k].out)+ m.cur[k] = e+}+// ScrubHeader is a header's value with its secrets taken out: all of it// for a header named for one or a credential (Bearer, Basic), else what// Scrub finds in it.diff --git a/internal/redact/scrub_memo_test.go b/internal/redact/scrub_memo_test.gonew file mode 100644index 000000000..f7f8e51e3--- /dev/null+++ b/internal/redact/scrub_memo_test.go@@ -0,0 +1,120 @@+package redact++import (+ "fmt"+ "strings"+ "testing"+ "time"+)++// sourceLines are what a shell call reads back while an agent works on a+// gateway: code that names tokens, keys and URLs, as the rules look for.+const sourceLines = `// step %d+func (c *Client) refresh(ctx context.Context) (string, error) {+ req, _ := http.NewRequestWithContext(ctx, "POST", "https://auth.example.com/oauth/token", nil)+ req.Header.Set("Authorization", "Bearer "+c.token)+ if c.apiKey == "" { return "", errors.New("no api key") }+ var out struct{ AccessToken string ` + "`json:\"access_token\"`" + `; ExpiresIn int }+ return out.AccessToken, json.NewDecoder(resp.Body).Decode(&out) // see github.com/example/auth+}+`++// codexHistory is a Responses request as Codex sends one on turn n: every+// item of the session so far, each turn's tool output a few KB.+func codexHistory(n int) []byte {+ var items []string+ for i := 0; i < n; i++ {+ items = append(items,+ fmt.Sprintf(`{"type":"message","role":"user","content":[{"type":"input_text","text":"step %d: look at the build log and fix what failed, then run the tests again"}]}`, i),+ fmt.Sprintf(`{"type":"function_call","name":"shell","arguments":"{\"command\":[\"bash\",\"-lc\",\"go test ./internal/... 2>&1 | tail -n 200\"]}","call_id":"call_%d"}`, i),+ fmt.Sprintf(`{"type":"function_call_output","call_id":"call_%d","output":%q}`, i, strings.Repeat(fmt.Sprintf(sourceLines, i), 8)),+ )+ }+ return []byte(`{"model":"gpt-6-astra","instructions":"You are a coding agent.","input":[` + strings.Join(items, ",") + `],"stream":true}`)+}++// TestScrubRunsTheRulesOnceAString: a history sent again on the next turn+// has the rules run only over what it did not have before+// (yetone/magpie-releases#24), and comes out as it did.+func TestScrubRunsTheRulesOnceAString(t *testing.T) {+ scrubMemo.reset()+ first := codexHistory(40)+ want := ScrubJSON(first)+ runs := scrubRuns.Load()+ next := codexHistory(41)+ got := ScrubJSON(next)+ // the new turn's one long string, its tool's output+ if n := scrubRuns.Load() - runs; n != 1 {+ t.Fatalf("the rules ran over %d strings for one new turn", n)+ }+ if !strings.HasPrefix(string(got), string(want[:len(want)-len(`],"stream":true}`)])) {+ t.Fatal("the history scrubbed again is not what it was")+ }+ scrubMemo.reset()+ if again := ScrubJSON(next); string(again) != string(got) {+ t.Fatal("remembered and fresh scrubs differ")+ }+}++// TestScrubMemoForgetsWhenAValueBecomesKnown: a string remembered as having+// nothing to scrub is scrubbed again once a value in it is masked+// somewhere else: what is known is part of what it has to lose.+func TestScrubMemoForgetsWhenAValueBecomesKnown(t *testing.T) {+ // a value no run before this one made known+ secret := fmt.Sprintf("Qz8marmot%dPass", time.Now().UnixNano())+ long := strings.Repeat("the build log says nothing of note here. ", 10) + "we named the box " + secret + " and that is all"+ if got := Scrub(long); got != long {+ t.Fatalf("scrubbed before it was known: %q", got)+ }+ Mask("DB_PASSWORD="+secret, Options{Secrets: true})+ if got := Scrub(long); strings.Contains(got, secret) || !strings.Contains(got, "[REDACTED:") {+ t.Fatalf("a value known since was left in: %q", got)+ }+ // and through a JSON body as the conversation records it+ body := fmt.Sprintf(`{"input":[{"type":"message","role":"user","content":[{"type":"input_text","text":%q}]}]}`, long)+ if got := ScrubJSON([]byte(body)); strings.Contains(string(got), secret) {+ t.Fatalf("%s", got)+ }+}++// TestScrubMemoKeepsTheSecretOut: a long string with a secret in it comes+// out scrubbed every time, the remembered result as the first.+func TestScrubMemoKeepsTheSecretOut(t *testing.T) {+ const key = "sk-ant-api03-abcdefghijklmnopqrstuvwxyz0123456789"+ long := strings.Repeat("a line of the log with nothing in it\n", 10) + "export ANTHROPIC_API_KEY=" + key + "\n"+ first := Scrub(long)+ if strings.Contains(first, key) {+ t.Fatalf("%q", first)+ }+ for i := 0; i < 3; i++ {+ if got := Scrub(long); got != first {+ t.Fatalf("turn %d: %q", i, got)+ }+ }+}++// TestScrubMemoBounded: the memo holds at most two generations of its+// bounds, however many strings go through it.+func TestScrubMemoBounded(t *testing.T) {+ scrubMemo.reset()+ t.Cleanup(scrubMemo.reset)+ pad := strings.Repeat("x", scrubMemoMin)+ for i := 0; i < 2*scrubMemoEntries+10; i++ {+ Scrub(fmt.Sprintf("%d %s", i, pad))+ }+ if n := len(scrubMemo.cur) + len(scrubMemo.old); n > 2*scrubMemoEntries {+ t.Fatalf("%d entries held", n)+ }+}++// BenchmarkScrubCodexTurn is one Codex turn recorded: the request, its+// whole history, scrubbed after the turn before it was.+func BenchmarkScrubCodexTurn(b *testing.B) {+ prev, next := codexHistory(300), codexHistory(301)+ ScrubJSON(prev)+ b.SetBytes(int64(len(next)))+ b.ResetTimer()+ for i := 0; i < b.N; i++ {+ ScrubJSON(next)+ }+}