Every agent's model. One place. Codex on DeepSeek, Claude Code on Kimi, from the menu bar.
redact: a value at the end of a tool call's arguments keeps its closing quote (xiaojiang091499 on X)
yetone5a85a9836cb0
diff --git a/internal/redact/argstail_test.go b/internal/redact/argstail_test.gonew file mode 100644index 000000000..b0c649700--- /dev/null+++ b/internal/redact/argstail_test.go@@ -0,0 +1,196 @@+package redact++import (+ "encoding/json"+ "net/http/httptest"+ "regexp"+ "strings"+ "testing"+)++// What the local model really found in a Codex tool call's arguments+// (the privacy filter run on them, 10-29): a password ending in ! or % is+// one token with the quote after it, so its span ran over the closing+// quote, and an address's over its full stop and the quote.+var argsTailModel = &fakeModel{values: map[string]string{+ `Hunt3r!"`: "secret",+ `Abc!23456%"`: "secret",+ `221B Baker Street, London."`: "private_address",+ `pwd=Qq123456@#"`: "secret",+}}++// A value the masking finds at the very end of a tool call's arguments+// ends at its string's closing quote: the "} after it go as written, the+// arguments are still JSON, and the value comes back in them as the+// agent wrote it. Masked with the quote, {"cmd":"export+// DB_PASSWORD=Hunt3r!"} went as {"cmd":"export DB_PASSWORD={{SECRET_…}}},+// the vendor turned away every turn that replayed the call, and Codex+// reconnected over and over (xiaojiang091499 on X). The same for each+// protocol whose arguments are JSON text, and for a user's rule that+// reads to the next space.+func TestArgumentsValueAtEndKeepsQuote(t *testing.T) {+ o := Options{Secrets: true, Personal: true, Model: argsTailModel.find,+ Rules: []Rule{{Kind: "TOK", Regex: `tok-\S+`}}}+ cases := []struct{ args, gone string }{+ {`{"cmd":"export DB_PASSWORD=Hunt3r!"}`, "Hunt3r"},+ {`{"cmd":["bash","-lc","echo 密码是 Abc!23456%"],"workdir":"/tmp"}`, "Abc!23456"},+ {`{"command":"pwd=Qq123456@#"}`, "Qq123456"},+ {`{"text":"ship to 221B Baker Street, London."}`, "Baker Street"},+ {`{"cmd":"curl -H 'X: tok-abcdef123'"}`, "tok-abcdef"},+ {`{"cmd":"curl -H tok-abcdef123"}`, "tok-abcdef"},+ {`{"a":["tok-abcdef123","x"],"b":1}`, "tok-abcdef"},+ {`{"cmd": "echo \"tok-abcdef123\""}`, "tok-abcdef"},+ {`{"cmd":"echo 13800138000"}`, "13800138000"},+ }+ for _, c := range cases {+ a, _ := json.Marshal(c.args)+ for _, body := range []string{+ // Responses, as Codex replays the call+ `{"model":"gpt-5.5","input":[{"type":"function_call","call_id":"call_1","name":"exec_command","arguments":` + string(a) + `}]}`,+ // Chat Completions+ string(chatWithCall(c.args)),+ } {+ out, n := MaskJSON([]byte(body), o)+ got := argumentsIn(t, out)+ if n == 0 || !json.Valid([]byte(got)) || strings.Contains(got, c.gone) {+ t.Errorf("%s\n masked %d: %s", c.args, n, got)+ continue+ }+ if back := Restore(got, true); back != c.args {+ t.Errorf("%s\n came back %s", c.args, back)+ }+ }+ }+}++// A value that is a whole bare value of the arguments, a number a user's+// rule finds, is masked as a string: written bare, its placeholder left+// them JSON no longer.+func TestArgumentsBareValueQuoted(t *testing.T) {+ o := Options{Secrets: true, Rules: []Rule{{Kind: "ACCT", Regex: `\b\d{11}\b`}}}+ out, n := MaskJSON(chatWithCall(`{"user":"app","acct": 12345678901,"ids":[12345678902]}`), o)+ got := argsOf(t, out)[0]+ if n != 2 || !regexp.MustCompile(`^\{"user":"app","acct": "\{\{ACCT_[a-z2-7]{8}\}\}","ids":\["\{\{ACCT_[a-z2-7]{8}\}\}"\]\}$`).MatchString(got) {+ t.Fatalf("%d: %s", n, got)+ }+ if back := Restore(got, true); back != `{"user":"app","acct": "12345678901","ids":["12345678902"]}` {+ t.Fatalf("came back %s", back)+ }+}++// argumentsIn are the arguments of the one tool call in a Responses or+// Chat Completions request.+func argumentsIn(t *testing.T, body []byte) string {+ t.Helper()+ var v struct {+ Input []struct {+ Arguments string `json:"arguments"`+ } `json:"input"`+ }+ if err := json.Unmarshal(body, &v); err != nil {+ t.Fatalf("body no longer JSON: %s", body)+ }+ if len(v.Input) > 0 {+ return v.Input[0].Arguments+ }+ return argsOf(t, body)[0]+}++// The model's arguments with the placeholder last, streamed in pieces+// that split it and its closing "}, come to the agent as the arguments+// with the value in them, escaped: in Responses' function_call_arguments+// deltas, Chat Completions' tool_calls and Anthropic's input_json_delta.+func TestWriterStreamArgumentsValueAtEnd(t *testing.T) {+ const value = `Hunt3r!"\x`+ // a value with a quote and a backslash in it, which go back escaped+ masked, _ := Mask("pw "+value, Options{Secrets: true, Rules: []Rule{{Kind: "PW", Regex: `Hunt3r\S+`}}})+ p := regexp.MustCompile(`\{\{PW_[a-z2-7]{8}\}\}`).FindString(masked)+ if p == "" {+ t.Fatalf("not masked: %s", masked)+ }+ args := `{"cmd":"export DB_PASSWORD=` + p + `"}`+ want, _ := json.Marshal(map[string]string{"cmd": "export DB_PASSWORD=" + value})+ protocols := []struct {+ name string+ event func(piece string) string+ piece func(data []byte) string+ }{+ {"responses", func(s string) string {+ b, _ := json.Marshal(map[string]any{"type": "response.function_call_arguments.delta", "item_id": "fc_1", "output_index": 1, "delta": s})+ return "event: response.function_call_arguments.delta\ndata: " + string(b) + "\n\n"+ }, func(d []byte) string {+ var e struct{ Delta string }+ json.Unmarshal(d, &e)+ return e.Delta+ }},+ {"chat", func(s string) string {+ b, _ := json.Marshal(map[string]any{"choices": []any{map[string]any{"index": 0, "delta": map[string]any{"tool_calls": []any{map[string]any{"index": 0, "function": map[string]any{"arguments": s}}}}}}})+ return "data: " + string(b) + "\n\n"+ }, func(d []byte) string {+ var e struct {+ Choices []struct {+ Delta struct {+ ToolCalls []struct{ Function struct{ Arguments string } } `json:"tool_calls"`+ }+ }+ }+ json.Unmarshal(d, &e)+ return e.Choices[0].Delta.ToolCalls[0].Function.Arguments+ }},+ {"anthropic", func(s string) string {+ b, _ := json.Marshal(map[string]any{"type": "content_block_delta", "index": 1, "delta": map[string]any{"type": "input_json_delta", "partial_json": s}})+ return "event: content_block_delta\ndata: " + string(b) + "\n\n"+ }, func(d []byte) string {+ var e struct {+ Delta struct {+ PartialJSON string `json:"partial_json"`+ }+ }+ json.Unmarshal(d, &e)+ return e.Delta.PartialJSON+ }},+ }+ at := strings.Index(args, p)+ for _, pr := range protocols {+ // every place to split it: inside the placeholder, between its+ // braces, before the "} and between the " and the }+ for cut := at + 1; cut < len(args); cut++ {+ rec := httptest.NewRecorder()+ rec.Header().Set("Content-Type", "text/event-stream")+ w := NewWriter(rec)+ w.WriteHeader(200)+ w.Write([]byte(pr.event(args[:cut])))+ w.Write([]byte(pr.event(args[cut:])))+ w.Finish()+ var js strings.Builder+ for _, e := range strings.Split(rec.Body.String(), "\n\n") {+ if _, d, ok := strings.Cut(e, "data: "); ok {+ js.WriteString(pr.piece([]byte(d)))+ }+ }+ if js.String() != string(want) {+ t.Errorf("%s, cut at %d: %s, want %s", pr.name, cut, js.String(), want)+ }+ }+ }+}++// A whole body's arguments delta whose "delta" comes before its "type" is+// put back escaped, as the stream's is.+func TestRestoreJSONDeltaBeforeType(t *testing.T) {+ masked, _ := Mask("pw Hunt3r!\"\\x", Options{Secrets: true, Rules: []Rule{{Kind: "PW", Regex: `Hunt3r\S+`}}})+ ph := regexp.MustCompile(`\{\{PW_[a-z2-7]{8}\}\}`).FindString(masked)+ if ph == "" {+ t.Fatalf("not masked: %s", masked)+ }+ args := `{"cmd":"export DB_PASSWORD=` + ph + `"}`+ in, _ := json.Marshal(map[string]any{"type": "response.function_call_arguments.delta", "delta": args})+ var ev map[string]string+ if err := json.Unmarshal(RestoreJSON(in), &ev); err != nil {+ t.Fatal(err)+ }+ want, _ := json.Marshal(map[string]string{"cmd": "export DB_PASSWORD=Hunt3r!\"\\x"})+ if ev["delta"] != string(want) {+ t.Fatalf("came back %s, want %s", ev["delta"], want)+ }+}diff --git a/internal/redact/dateword.go b/internal/redact/dateword.goindex b8c013c4e..9650ff599 100644--- a/internal/redact/dateword.go+++ b/internal/redact/dateword.go@@ -176,7 +176,7 @@ func dateSpans(s, w string) ([]span, bool) {var out []spanfor _, m := range re.FindAllStringIndex(s, -1) {if dateWhole(s, m[0], m[1]) {- out = append(out, span{m[0], m[1], "TERM", ByWord, 0})+ out = append(out, span{m[0], m[1], "TERM", ByWord, 0, false})}}return out, truediff --git a/internal/redact/escape_test.go b/internal/redact/escape_test.goindex 7696cd50a..71d96c17c 100644--- a/internal/redact/escape_test.go+++ b/internal/redact/escape_test.go@@ -233,12 +233,12 @@ func TestMaskJSONArgumentsCutShort(t *testing.T) {// inside it toos := `x\u0040y\nz`found := outsideEscapes(s, []span{- {3, 5, "A", ByRule, 0}, // 00 of \u0040: nothing left- {1, 7, "B", ByRule, 0}, // the whole \u0040: as it is- {0, 4, "C", ByRule, 0}, // ends in it: x- {7, 9, "D", ByRule, 0}, // y and the backslash of \n: y- {9, 11, "E", ByRule, 0}, // the n of \n and z: z- {8, 11, "F", ByRule, 0}, // \nz: as it is+ {3, 5, "A", ByRule, 0, false}, // 00 of \u0040: nothing left+ {1, 7, "B", ByRule, 0, false}, // the whole \u0040: as it is+ {0, 4, "C", ByRule, 0, false}, // ends in it: x+ {7, 9, "D", ByRule, 0, false}, // y and the backslash of \n: y+ {9, 11, "E", ByRule, 0, false}, // the n of \n and z: z+ {8, 11, "F", ByRule, 0, false}, // \nz: as it is})var kept []stringfor _, f := range found {diff --git a/internal/redact/json.go b/internal/redact/json.goindex 1f501cf8e..f7726f069 100644--- a/internal/redact/json.go+++ b/internal/redact/json.go@@ -496,6 +496,23 @@ func asJSON(key, eventType string) bool {return key == "arguments" || key == "partial_json" || key == "delta" && strings.Contains(eventType, "arguments")}+// typeOf is the type a JSON object says it is, wherever its "type" is+// among its keys. Read as the keys came, a Responses+// function_call_arguments.delta whose "delta" came before its "type", as+// a Go map writes them, had its value put back unescaped: a password+// with a quote in it ended the arguments' string there, and the arguments+// the agent put together were JSON no longer.+func typeOf(data []byte) string {+ typ := ""+ walk(data, func(path, _, s string) string {+ if path == "type" {+ typ = s+ }+ return s+ })+ return typ+}+// RestoreJSON puts the values back in a response body.func RestoreJSON(body []byte) []byte { return restoreJSON(body, nil) }@@ -504,11 +521,8 @@ func restoreJSON(body []byte, put map[string]int) []byte {if !bytes.Contains(body, []byte("{{")) && !hasShapes(body) {return body}- typ := ""+ typ := typeOf(body)out, ok := walk(body, func(path, key, s string) string {- if path == "type" {- typ = s- }return restore(s, asJSON(key, typ), put)})if !ok {diff --git a/internal/redact/known.go b/internal/redact/known.goindex c5719f9e8..adda68645 100644--- a/internal/redact/known.go+++ b/internal/redact/known.go@@ -208,7 +208,7 @@ func knownSpans(s string, o Options) []span {best = &known[s[i:i+minKnown]][j]}if best != nil {- out = append(out, span{i, i + len(best.v), best.kind, ByKnown, 0})+ out = append(out, span{i, i + len(best.v), best.kind, ByKnown, 0, false})i += len(best.v) - 1}}diff --git a/internal/redact/model.go b/internal/redact/model.goindex 0fa39dae1..8818c4d2b 100644--- a/internal/redact/model.go+++ b/internal/redact/model.go@@ -84,7 +84,7 @@ func modelSpans(s string, o Options) []span {a, b = zhTrim(s, a, b)}if a < b && utf8.ValidString(s[a:b]) && modelTakes(kind, s[a:b]) && modelFits(s, a, b, kind) && !(kind == "PERSON" && zhWordNotName(s, a, b)) {- out = append(out, span{a, b, kind, ByModel, f.Score})+ out = append(out, span{a, b, kind, ByModel, f.Score, false})}}if o.covers(catPersonal) {diff --git a/internal/redact/redact.go b/internal/redact/redact.goindex fa2e3a2c0..8468368ce 100644--- a/internal/redact/redact.go+++ b/internal/redact/redact.go@@ -691,6 +691,9 @@ type span struct {kind stringby string // which finder found it: one of the By constantsscore float64 // the model's score for one it found, else 0+ // bare: a whole bare value of JSON text (inStrings), whose+ // placeholder goes in quoted+ bare bool}// Which finder found a value (Hit.By): magpie's rules, the user's own@@ -913,7 +916,7 @@ func mask(raw string, o Options, put func(kind, v string) string, escaped bool)i = a + max(n, 1)continue}- found = append(found, span{a, b, "TERM", ByWord, 0})+ found = append(found, span{a, b, "TERM", ByWord, 0, false})i = max(b, a+1)}}@@ -948,12 +951,12 @@ func mask(raw string, o Options, put func(kind, v string) string, escaped bool)took := falsefor g, kind := range r.groups {if a, b := m[2*g+2], m[2*g+3]; a >= 0 && r.takes(s, a, b, r.groupOK[g]) {- found = append(found, span{a, b, kind, r.by(), 0})+ found = append(found, span{a, b, kind, r.by(), 0, false})took = true}}if took {- split = append(split, span{m[2], m[1], "", "", 0})+ split = append(split, span{m[2], m[1], "", "", 0, false})}continue}@@ -976,7 +979,7 @@ func mask(raw string, o Options, put func(kind, v string) string, escaped bool)ok = nil}if r.takes(s, a, b, ok) {- found = append(found, span{a, b, r.kind, r.by(), 0})+ found = append(found, span{a, b, r.kind, r.by(), 0, false})}}}@@ -994,7 +997,7 @@ func mask(raw string, o Options, put func(kind, v string) string, escaped bool)found = append(found, knownSpans(s, o)...)}if escaped {- found = outsideEscapes(raw, found)+ found = inStrings(raw, outsideEscapes(raw, found))}found = noNumbers(s, noIDs(s, found))if o.sent {@@ -1099,6 +1102,83 @@ func outsideEscapes(s string, found []span) []span {return out}+// inStrings is found, for s JSON text (a tool call's arguments), with+// each value inside the one string it starts in: cut at that string's+// closing quote, or started at the first string inside it. A value that+// is a whole bare value (a number under "password":) is kept, and its+// placeholder goes in quoted (span.bare); one in nothing else, across a+// key's quotes, a colon or a brace, goes. Masked over the quote, the+// arguments were JSON no longer: the local model reads a password's ! and+// the "} after it as one token, and {"cmd":"export DB_PASSWORD=Hunt3r!"}+// went as {"cmd":"export DB_PASSWORD={{SECRET_…}}}; a user's rule+// tok-\S+ took the "} too. The vendor turned away every turn that+// replayed the call, and Codex reconnected over and over+// (xiaojiang091499 on X). Every finder's values go through here, as+// through outsideEscapes. Text that isn't JSON is left as it is.+func inStrings(s string, found []span) []span {+ if len(found) == 0 || !json.Valid([]byte(s)) {+ return found+ }+ // in[i] is 1 + which string literal byte i is the text of, 0 for one+ // outside every string (its quotes among them)+ in := make([]int32, len(s))+ n := int32(0)+ for i := 0; i < len(s); i++ {+ if s[i] != '"' {+ continue+ }+ n+++ j := i + 1+ for ; j < len(s) && s[j] != '"'; j++ {+ in[j] = n+ if s[j] == '\\' && j+1 < len(s) {+ j+++ in[j] = n+ }+ }+ i = j+ }+ out := found[:0]+ for _, f := range found {+ a := f.start+ for a < f.end && in[a] == 0 {+ a+++ }+ if a == f.end {+ if bareValue(s, f.start, f.end) {+ f.bare = true+ out = append(out, f)+ }+ continue+ }+ b := a+ for b < f.end && in[b] == in[a] {+ b+++ }+ f.start, f.end = a, b+ out = append(out, f)+ }+ return out+}++// bareValue says s[a:b], JSON text outside its strings, is all of one+// bare value: an object's (after :) or an array's (after [ or ,), with+// the , } or ] that ends it after it.+func bareValue(s string, a, b int) bool {+ if a >= b || strings.ContainsAny(s[a:b], " \t\r\n{}[]\",:") {+ return false+ }+ i := a - 1+ for i >= 0 && isSpace(s[i]) {+ i--+ }+ k := b+ for k < len(s) && isSpace(s[k]) {+ k+++ }+ return i >= 0 && strings.IndexByte(":[,", s[i]) >= 0 && (k == len(s) || strings.IndexByte(",}]", s[k]) >= 0)+}+// noNumbers is found without what is a piece of a decimal number of s:// digits right after the point of 5.13800138000, or right before the// point of 13800138000.5. A number is judged whole, and the fraction of@@ -1313,7 +1393,11 @@ func apply(s string, found []span, put func(kind, v string) string, o Options) (}b.WriteString(s[last:f.start])h.Placeholder = put(f.kind, v)- b.WriteString(h.Placeholder)+ if f.bare {+ b.WriteString(`"` + h.Placeholder + `"`)+ } else {+ b.WriteString(h.Placeholder)+ }if o.Seen != nil {o.Seen(h)}diff --git a/internal/redact/writer.go b/internal/redact/writer.goindex 20a542c0a..d86251d9d 100644--- a/internal/redact/writer.go+++ b/internal/redact/writer.go@@ -208,12 +208,12 @@ func (rw *Writer) event(ev []byte) error {}rw.pending = nil}- typ := ""+ typ := typeOf(data)+ if typ == "" {+ typ = name+ }tailPath, tail, cut := "", "", ""restored, _ := walk(data, func(path, key, s string) string {- if path == "type" {- typ = s- }if path == at {s = carry + s}diff --git a/internal/redact/zhname.go b/internal/redact/zhname.goindex 007221718..067d685a1 100644--- a/internal/redact/zhname.go+++ b/internal/redact/zhname.go@@ -216,7 +216,7 @@ func zhNameSpans(s string, person []span, nicks bool) []span {if ok {kind = "PERSON"}- cs = append(cs, span{a, b, kind, ByModel, 0})+ cs = append(cs, span{a, b, kind, ByModel, 0, false})}// a name joined by 和, 与, 跟, 及 or 、 to one that is a name is one// too: 李娜和刘洋, where only 刘洋 has a word after it that says so