Skip to content

Every agent's model. One place. Codex on DeepSeek, Claude Code on Kimi, from the menu bar.

redact: a long string already scrubbed is not run through the rules again

yetoneb36819d349ca
diff --git a/internal/redact/known.go b/internal/redact/known.go
index 23f2ea6a8..edcab4b49 100644
--- a/internal/redact/known.go
+++ b/internal/redact/known.go
@@ -3,6 +3,7 @@ package redact
import (
"slices"
"strings"
+ "sync/atomic"
)
// A value masked once is masked again wherever it is in a request, as it
@@ -33,6 +34,9 @@ var (
// knownBits has a bit for the first four bytes of each known value, so
// most places in a text are passed over without a map lookup
knownBits [1 << 14]uint64
+ // knownGen changes whenever known does, with mu held; what Scrub
+ // remembers is good only at the knownGen it was made at
+ knownGen atomic.Uint64
)
func gram(s string, i int) uint32 {
@@ -55,6 +59,7 @@ func addKnown(p, v string) {
// first, so it is the same after reindexKnown
if p < k.p {
known[head][i] = knownValue{v, p, kind}
+ knownGen.Add(1)
}
return
}
@@ -62,6 +67,7 @@ func addKnown(p, v string) {
known[head] = append(known[head], knownValue{v, p, kind})
g := gram(v, 0)
knownBits[g/64] |= 1 << (g % 64)
+ knownGen.Add(1)
}
// reindexKnown makes known again from values, after some were let go;
@@ -69,6 +75,7 @@ func addKnown(p, v string) {
func reindexKnown() {
known = map[string][]knownValue{}
knownBits = [1 << 14]uint64{}
+ knownGen.Add(1)
for p, v := range values {
addKnown(p, v)
}
diff --git a/internal/redact/scrub.go b/internal/redact/scrub.go
index 8f0082633..1737b4d69 100644
--- a/internal/redact/scrub.go
+++ b/internal/redact/scrub.go
@@ -1,8 +1,11 @@
package redact
import (
+ "crypto/sha256"
"regexp"
"strings"
+ "sync"
+ "sync/atomic"
)
// Scrubbing is for what is kept rather than sent — the request archive: a
@@ -28,10 +31,109 @@ func SecretName(name string) bool {
// Scrub takes the secrets Mask finds out of s.
func Scrub(s string) string {
+ if len(s) < scrubMemoMin {
+ return scrub(s)
+ }
+ // What Scrub makes of s depends on s and on the values known (known.go)
+ // alone: a result is kept under the knownGen it was made at, and none
+ // made before a value became known is used again.
+ gen := knownGen.Load()
+ k := sha256.Sum256([]byte(s))
+ if out, ok := scrubMemo.get(k, gen, s); ok {
+ return out
+ }
+ out := scrub(s)
+ scrubMemo.put(k, gen, s, out)
+ return out
+}
+
+func scrub(s string) string {
+ if len(s) >= 256 {
+ scrubRuns.Add(1)
+ }
out, _ := mask(s, Options{Secrets: true}, scrubbed)
return out
}
+// scrubRuns counts the strings of 256 bytes or more the rules have run
+// over (a test reads it).
+var scrubRuns atomic.Uint64
+
+// Scrub remembers what it made of a long string. An agent sends its whole
+// history on every turn (Codex: every item of a session, each turn), and
+// what is kept of each request (the request archive, a gateway
+// conversation) is scrubbed whole, so without it every turn ran each rule
+// over the whole history again: a core's worth while Codex ran
+// (yetone/magpie-releases#24). A short string costs less to scrub than to
+// look up. Only a hash of a string is kept, and what it became when that
+// is not the string itself.
+const (
+ scrubMemoMin = 256
+ scrubMemoEntries = 1 << 15
+ scrubMemoBytes = 32 << 20
+)
+
+type scrubEntry struct {
+ gen uint64
+ same bool // s came back as it was
+ out string // else what it became
+}
+
+// scrubMemo holds two generations of results: when the newer is full, it
+// becomes the older and the older goes, so what is in use stays and at
+// most twice the bounds are held.
+var scrubMemo memo
+
+type memo struct {
+ sync.Mutex
+ cur, old map[[32]byte]scrubEntry
+ bytes int
+}
+
+func (m *memo) get(k [32]byte, gen uint64, s string) (string, bool) {
+ m.Lock()
+ defer m.Unlock()
+ e, ok := m.cur[k]
+ if !ok {
+ if e, ok = m.old[k]; ok && e.gen == gen {
+ m.add(k, e)
+ }
+ }
+ if !ok || e.gen != gen {
+ return "", false
+ }
+ if e.same {
+ return s, true
+ }
+ return e.out, true
+}
+
+func (m *memo) put(k [32]byte, gen uint64, s, out string) {
+ e := scrubEntry{gen: gen, same: out == s}
+ if !e.same {
+ e.out = strings.Clone(out)
+ }
+ m.Lock()
+ defer m.Unlock()
+ m.add(k, e)
+}
+
+// reset forgets everything (a test starts from nothing).
+func (m *memo) reset() {
+ m.Lock()
+ defer m.Unlock()
+ m.cur, m.old, m.bytes = nil, nil, 0
+}
+
+// add is called with m held.
+func (m *memo) add(k [32]byte, e scrubEntry) {
+ if m.cur == nil || len(m.cur) >= scrubMemoEntries || m.bytes+len(e.out) > scrubMemoBytes {
+ m.old, m.cur, m.bytes = m.cur, make(map[[32]byte]scrubEntry), 0
+ }
+ m.bytes += len(e.out) - len(m.cur[k].out)
+ m.cur[k] = e
+}
+
// ScrubHeader is a header's value with its secrets taken out: all of it
// for a header named for one or a credential (Bearer, Basic), else what
// Scrub finds in it.
diff --git a/internal/redact/scrub_memo_test.go b/internal/redact/scrub_memo_test.go
new file mode 100644
index 000000000..f7f8e51e3
--- /dev/null
+++ b/internal/redact/scrub_memo_test.go
@@ -0,0 +1,120 @@
+package redact
+
+import (
+ "fmt"
+ "strings"
+ "testing"
+ "time"
+)
+
+// sourceLines are what a shell call reads back while an agent works on a
+// gateway: code that names tokens, keys and URLs, as the rules look for.
+const sourceLines = `// step %d
+func (c *Client) refresh(ctx context.Context) (string, error) {
+ req, _ := http.NewRequestWithContext(ctx, "POST", "https://auth.example.com/oauth/token", nil)
+ req.Header.Set("Authorization", "Bearer "+c.token)
+ if c.apiKey == "" { return "", errors.New("no api key") }
+ var out struct{ AccessToken string ` + "`json:\"access_token\"`" + `; ExpiresIn int }
+ return out.AccessToken, json.NewDecoder(resp.Body).Decode(&out) // see github.com/example/auth
+}
+`
+
+// codexHistory is a Responses request as Codex sends one on turn n: every
+// item of the session so far, each turn's tool output a few KB.
+func codexHistory(n int) []byte {
+ var items []string
+ for i := 0; i < n; i++ {
+ items = append(items,
+ fmt.Sprintf(`{"type":"message","role":"user","content":[{"type":"input_text","text":"step %d: look at the build log and fix what failed, then run the tests again"}]}`, i),
+ fmt.Sprintf(`{"type":"function_call","name":"shell","arguments":"{\"command\":[\"bash\",\"-lc\",\"go test ./internal/... 2>&1 | tail -n 200\"]}","call_id":"call_%d"}`, i),
+ fmt.Sprintf(`{"type":"function_call_output","call_id":"call_%d","output":%q}`, i, strings.Repeat(fmt.Sprintf(sourceLines, i), 8)),
+ )
+ }
+ return []byte(`{"model":"gpt-6-astra","instructions":"You are a coding agent.","input":[` + strings.Join(items, ",") + `],"stream":true}`)
+}
+
+// TestScrubRunsTheRulesOnceAString: a history sent again on the next turn
+// has the rules run only over what it did not have before
+// (yetone/magpie-releases#24), and comes out as it did.
+func TestScrubRunsTheRulesOnceAString(t *testing.T) {
+ scrubMemo.reset()
+ first := codexHistory(40)
+ want := ScrubJSON(first)
+ runs := scrubRuns.Load()
+ next := codexHistory(41)
+ got := ScrubJSON(next)
+ // the new turn's one long string, its tool's output
+ if n := scrubRuns.Load() - runs; n != 1 {
+ t.Fatalf("the rules ran over %d strings for one new turn", n)
+ }
+ if !strings.HasPrefix(string(got), string(want[:len(want)-len(`],"stream":true}`)])) {
+ t.Fatal("the history scrubbed again is not what it was")
+ }
+ scrubMemo.reset()
+ if again := ScrubJSON(next); string(again) != string(got) {
+ t.Fatal("remembered and fresh scrubs differ")
+ }
+}
+
+// TestScrubMemoForgetsWhenAValueBecomesKnown: a string remembered as having
+// nothing to scrub is scrubbed again once a value in it is masked
+// somewhere else: what is known is part of what it has to lose.
+func TestScrubMemoForgetsWhenAValueBecomesKnown(t *testing.T) {
+ // a value no run before this one made known
+ secret := fmt.Sprintf("Qz8marmot%dPass", time.Now().UnixNano())
+ long := strings.Repeat("the build log says nothing of note here. ", 10) + "we named the box " + secret + " and that is all"
+ if got := Scrub(long); got != long {
+ t.Fatalf("scrubbed before it was known: %q", got)
+ }
+ Mask("DB_PASSWORD="+secret, Options{Secrets: true})
+ if got := Scrub(long); strings.Contains(got, secret) || !strings.Contains(got, "[REDACTED:") {
+ t.Fatalf("a value known since was left in: %q", got)
+ }
+ // and through a JSON body as the conversation records it
+ body := fmt.Sprintf(`{"input":[{"type":"message","role":"user","content":[{"type":"input_text","text":%q}]}]}`, long)
+ if got := ScrubJSON([]byte(body)); strings.Contains(string(got), secret) {
+ t.Fatalf("%s", got)
+ }
+}
+
+// TestScrubMemoKeepsTheSecretOut: a long string with a secret in it comes
+// out scrubbed every time, the remembered result as the first.
+func TestScrubMemoKeepsTheSecretOut(t *testing.T) {
+ const key = "sk-ant-api03-abcdefghijklmnopqrstuvwxyz0123456789"
+ long := strings.Repeat("a line of the log with nothing in it\n", 10) + "export ANTHROPIC_API_KEY=" + key + "\n"
+ first := Scrub(long)
+ if strings.Contains(first, key) {
+ t.Fatalf("%q", first)
+ }
+ for i := 0; i < 3; i++ {
+ if got := Scrub(long); got != first {
+ t.Fatalf("turn %d: %q", i, got)
+ }
+ }
+}
+
+// TestScrubMemoBounded: the memo holds at most two generations of its
+// bounds, however many strings go through it.
+func TestScrubMemoBounded(t *testing.T) {
+ scrubMemo.reset()
+ t.Cleanup(scrubMemo.reset)
+ pad := strings.Repeat("x", scrubMemoMin)
+ for i := 0; i < 2*scrubMemoEntries+10; i++ {
+ Scrub(fmt.Sprintf("%d %s", i, pad))
+ }
+ if n := len(scrubMemo.cur) + len(scrubMemo.old); n > 2*scrubMemoEntries {
+ t.Fatalf("%d entries held", n)
+ }
+}
+
+// BenchmarkScrubCodexTurn is one Codex turn recorded: the request, its
+// whole history, scrubbed after the turn before it was.
+func BenchmarkScrubCodexTurn(b *testing.B) {
+ prev, next := codexHistory(300), codexHistory(301)
+ ScrubJSON(prev)
+ b.SetBytes(int64(len(next)))
+ b.ResetTimer()
+ for i := 0; i < b.N; i++ {
+ ScrubJSON(next)
+ }
+}