Skip to content

Every agent's model. One place. Codex on DeepSeek, Claude Code on Kimi, from the menu bar.

redact: an evaluation of the masking, with the rules, the model and both

yetone180cfb565e85
diff --git a/internal/redact/eval_cases_test.go b/internal/redact/eval_cases_test.go
new file mode 100644
index 000000000..c938db85c
--- /dev/null
+++ b/internal/redact/eval_cases_test.go
@@ -0,0 +1,228 @@
+package redact_test
+
+// evalCase is one text of the evaluation (eval_test.go): where it comes
+// from (an issue or a commit, or "true" for one of the values masking is
+// for), the values in it to mask and those to keep as written.
+type evalCase struct {
+ ID, From, Text string
+ Mask, Keep []evalValue
+}
+
+// evalValue is a value in a case's text. Kind is what it is, or for one
+// to keep what it was or could be taken for. Known, where set, is the
+// issue of a value that comes out wrong with the rules and the model.
+type evalValue struct{ V, Kind, Known string }
+
+// The keys are made up, in the vendors' shapes; the people, places and
+// numbers are made up or the ones vendors and standards give as examples.
+var (
+ evalOpenAIKey = "sk-proj-" + "Qm7vT2xLk9RbW4nZpE6sJc8YhD3fGa5uKiN1oVr0"
+ evalGitHub = "ghp_" + "0123456789abcdefghijABCDEFGHIJ0123"
+ evalAWSSecret = "wJalrXUtnFEMI/K7MDENG/" + "bPxRfiCYEXAMPLEKEY"
+ evalGoogleKey = "AIzaSy" + "B3xQ9mLk2Vr7TpW4nZs8YhC1dFgJ6uKe0oA"
+ evalJWT = "eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9." + "eyJzdWIiOiI0MjEiLCJuYW1lIjoiTWFyaWEifQ." + "x7Hq2LmZ9pVb4WnT8sRc1YeKd6gJ3uFa0oNi5vQwE"
+)
+
+var evalCases = []evalCase{
+ // ---- reported: masked and should not have been ----------------------
+ {
+ ID: "snake-arg", From: "#3144",
+ Text: "parser.add_argument(\"--sequential_cutoff\", type=int, default=3,\n help=\"stop after this many misses\")",
+ Keep: []evalValue{{V: "sequential_cutoff", Kind: "PERSON"}},
+ },
+ {
+ ID: "snake-words", From: "#3144",
+ Text: "def build_index(max_retries, warmup_steps):\n return Index(stream_options=None, include_usage=True)",
+ Keep: []evalValue{{V: "max_retries", Kind: "PERSON"}, {V: "warmup_steps", Kind: "PERSON"}, {V: "stream_options", Kind: "PERSON"}, {V: "include_usage", Kind: "PERSON"}},
+ },
+ {
+ ID: "tool-servers", From: "#3143",
+ Text: "Use mcp__claude-in-chrome__navigate to open the page, then mcp__linear__save_issue to file it.",
+ Keep: []evalValue{{V: "mcp__claude-in-chrome__navigate", Kind: "PERSON"}, {V: "mcp__linear__save_issue", Kind: "PERSON"}},
+ },
+ {
+ ID: "retina-file", From: "#3029",
+ Text: "Here is the screenshot: ![shot](/Users/dev/Desktop/[email protected]) — the button is cut off.",
+ Keep: []evalValue{{V: "[email protected]", Kind: "EMAIL"}},
+ },
+ {
+ ID: "asset-files", From: "#3029",
+ Text: "import logo from './assets/[email protected]';\nimport intro from './media/[email protected]';",
+ Keep: []evalValue{{V: "[email protected]", Kind: "EMAIL"}, {V: "[email protected]", Kind: "EMAIL"}},
+ },
+ {
+ ID: "url-userinfo", From: "#3538",
+ Text: "DATABASE_URL=postgres://app:[email protected]:5432/orders",
+ Mask: []evalValue{{V: "hunter2pass", Kind: "SECRET"}},
+ Keep: []evalValue{{V: "db.acme-corp.cn", Kind: "EMAIL"}},
+ },
+ {
+ ID: "git-user-url", From: "#3538",
+ Text: "git clone https://[email protected]/team/repo.git && cd repo",
+ Keep: []evalValue{{V: "[email protected]", Kind: "EMAIL"}},
+ },
+ {
+ ID: "unix-time", From: "#3577",
+ Text: `{"type":"tool_call","id":"call_7","ts":1761379200,"name":"read_file","args":{"path":"main.go"}}`,
+ Keep: []evalValue{{V: "1761379200", Kind: "PHONE"}},
+ },
+ {
+ ID: "unix-time-ms", From: "#3577",
+ Text: "tool_call finished at 1761379200123 (took 812 ms)",
+ Keep: []evalValue{{V: "1761379200123", Kind: "PHONE"}},
+ },
+ {
+ ID: "session-uuid", From: "d3db461bb",
+ Text: "Read ~/.claude/projects/-Users-dev-app/3f2b6c1e-8a4d-4f7e-9b1a-2c3d4e5f6a7b.jsonl for the transcript.",
+ Keep: []evalValue{{V: "3f2b6c1e-8a4d-4f7e-9b1a-2c3d4e5f6a7b", Kind: "SECRET"}},
+ },
+ {
+ ID: "digests", From: "d3db461bb",
+ Text: "sha256: 80efc1803eda7c095a79741d2008c07e2e0a57b01bac8825fbeb448fd097998c\ncommit b855cfee9a1c3d4e5f60718293a4b5c6d7e8f901",
+ Keep: []evalValue{{V: "80efc1803eda7c095a79741d2008c07e2e0a57b01bac8825fbeb448fd097998c", Kind: "SECRET"}, {V: "b855cfee9a1c3d4e5f60718293a4b5c6d7e8f901", Kind: "ACCOUNT"}},
+ },
+ {
+ ID: "tool-ids", From: "611b9e173",
+ Text: `{"type":"tool_result","tool_use_id":"toolu_01XkP9qLm2VbT7wZ4nR8sYc3","content":"ok"} request req_011CUq7vHn3kLp9WmT2xZb4R`,
+ Keep: []evalValue{{V: "toolu_01XkP9qLm2VbT7wZ4nR8sYc3", Kind: "SECRET"}, {V: "req_011CUq7vHn3kLp9WmT2xZb4R", Kind: "SECRET"}},
+ },
+ {
+ ID: "token-count", From: "611b9e173",
+ Text: "<system-reminder><total_tokens>14902266 tokens left</total_tokens></system-reminder>",
+ Keep: []evalValue{{V: "14902266", Kind: "ACCOUNT"}},
+ },
+ {
+ ID: "assistant-names", From: "611b9e173",
+ Text: "You are Claude, powered by Opus 5.5. Codex and Haiku 4.5 may also answer in this session.",
+ Keep: []evalValue{{V: "Claude", Kind: "PERSON"}, {V: "Opus 5.5", Kind: "PERSON"}, {V: "Codex", Kind: "PERSON"}, {V: "Haiku 4.5", Kind: "PERSON"}},
+ },
+ {
+ ID: "home-path", From: "611b9e173",
+ Text: "cd /Users/jdoe/workspace/projects/magpie && go test ./internal/redact",
+ Keep: []evalValue{{V: "jdoe", Kind: "PERSON"}, {V: "magpie", Kind: "PERSON"}},
+ },
+ {
+ ID: "grep-pattern", From: "611b9e173",
+ Text: "rg -n 'password|passwd|secret_key|api_token' --glob '*.go'",
+ Keep: []evalValue{{V: "password|passwd|secret_key|api_token", Kind: "SECRET"}},
+ },
+ {
+ ID: "sizes-versions", From: "611b9e173",
+ Text: "Screenshot 1624x1487 at 2x; magpie v0.1.1513 built with go1.26.8 on darwin/arm64.",
+ Keep: []evalValue{{V: "1624x1487", Kind: "PHONE"}, {V: "v0.1.1513", Kind: "ACCOUNT"}, {V: "1.26.8", Kind: "PHONE"}},
+ },
+ {
+ ID: "noreply", From: "true",
+ Text: "Author: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>",
+ Keep: []evalValue{{V: "49699333+dependabot[bot]@users.noreply.github.com", Kind: "EMAIL"}},
+ },
+ {
+ ID: "example-mail", From: "true",
+ Text: "// e.g. send(\"[email protected]\", subject)",
+ Keep: []evalValue{{V: "[email protected]", Kind: "EMAIL"}},
+ },
+ {
+ ID: "code-locations", From: "true",
+ Text: "internal/redact/model.go:212:9: undefined: epochNumber (pid 48213, exit status 2)",
+ Keep: []evalValue{{V: "48213", Kind: "PHONE"}, {V: "model.go:212", Kind: "ACCOUNT"}},
+ },
+ {
+ ID: "colors-ports", From: "true",
+ Text: ".btn { color: #1a2b3c; } server listening on 127.0.0.1:3425, retry in 30000ms",
+ Keep: []evalValue{{V: "#1a2b3c", Kind: "SECRET"}, {V: "30000", Kind: "PHONE"}},
+ },
+ {
+ ID: "zh-words", From: "#1977",
+ Text: "请帮我把这个函数改成异步的,并且加上错误处理和单元测试。",
+ Keep: []evalValue{{V: "函数", Kind: "PERSON"}, {V: "错误处理", Kind: "PERSON"}},
+ },
+
+ // ---- what masking is for --------------------------------------------
+ {
+ ID: "prose-person", From: "true",
+ Text: "Please email Maria Gonzalez about the overdue invoice before Friday.",
+ Mask: []evalValue{{V: "Maria Gonzalez", Kind: "PERSON"}},
+ },
+ {
+ ID: "git-author", From: "true",
+ Text: "commit 4f1c2d9\nAuthor: Jane Smith <[email protected]>\nDate: Tue Oct 21 09:14:03 2026 +0800",
+ Mask: []evalValue{{V: "Jane Smith", Kind: "PERSON"}, {V: "[email protected]", Kind: "EMAIL"}},
+ },
+ {
+ ID: "json-contact", From: "true",
+ Text: `{"customer":{"name":"Wei Zhang","email":"[email protected]","phone":"+86 139 1234 5678"}}`,
+ Mask: []evalValue{{V: "Wei Zhang", Kind: "PERSON"}, {V: "[email protected]", Kind: "EMAIL"}, {V: "139 1234 5678", Kind: "PHONE", Known: "#3591"}},
+ },
+ {
+ ID: "us-address", From: "true",
+ Text: "Ship the replacement to 742 Evergreen Terrace, Springfield, OR 97477 and call (212) 555-0199 on arrival.",
+ Mask: []evalValue{{V: "742 Evergreen Terrace", Kind: "ADDRESS"}, {V: "555-0199", Kind: "PHONE"}},
+ },
+ {
+ ID: "zh-contact", From: "#1977",
+ Text: "联系人:张伟,电话 13800138000,地址:北京市朝阳区建国路88号院3号楼",
+ Mask: []evalValue{{V: "张伟", Kind: "PERSON"}, {V: "13800138000", Kind: "PHONE"}, {V: "建国路88号", Kind: "ADDRESS"}},
+ },
+ {
+ ID: "zh-prose", From: "#1977",
+ Text: "王芳下周一去上海出差,酒店那边请联系她的助理刘洋。",
+ Mask: []evalValue{{V: "王芳", Kind: "PERSON"}, {V: "刘洋", Kind: "PERSON", Known: "#3592"}},
+ },
+ {
+ ID: "log-user", From: "true",
+ Text: "2026-10-21T09:14:03Z INFO login ok [email protected] tenant=acme ip=10.2.3.4",
+ Mask: []evalValue{{V: "[email protected]", Kind: "EMAIL"}},
+ },
+ {
+ ID: "phone-intl", From: "true",
+ Text: "If the build breaks overnight, page Tom on +1 415-555-0132.",
+ Mask: []evalValue{{V: "415-555-0132", Kind: "PHONE"}, {V: "Tom", Kind: "PERSON"}},
+ },
+ {
+ ID: "card", From: "true",
+ Text: "The customer's card number is 4111 1111 1111 1111, expiry 12/29.",
+ Mask: []evalValue{{V: "4111 1111 1111 1111", Kind: "ACCOUNT"}},
+ },
+ {
+ ID: "iban", From: "true",
+ Text: "Wire the refund to IBAN DE89 3704 0044 0532 0130 00 (Commerzbank).",
+ Mask: []evalValue{{V: "DE89 3704 0044 0532 0130 00", Kind: "ACCOUNT"}},
+ },
+ {
+ ID: "zh-id", From: "true",
+ Text: "身份证号 11010519491231002X,开户行 招商银行",
+ Mask: []evalValue{{V: "11010519491231002X", Kind: "ACCOUNT"}},
+ },
+ {
+ ID: "env-file", From: "true",
+ Text: "OPENAI_API_KEY=" + evalOpenAIKey + "\nDB_PASSWORD=Tr0ub4dor&3xyz\nDEBUG=true",
+ Mask: []evalValue{{V: evalOpenAIKey, Kind: "SECRET"}, {V: "Tr0ub4dor&3xyz", Kind: "SECRET"}},
+ Keep: []evalValue{{V: "DEBUG=true", Kind: "SECRET"}},
+ },
+ {
+ ID: "prose-password", From: "true",
+ Text: "the staging db password is hunter2Xq9z, don't commit it",
+ Mask: []evalValue{{V: "hunter2Xq9z", Kind: "SECRET"}},
+ },
+ {
+ ID: "zh-password", From: "true",
+ Text: "服务器密码是 Qw8!zLp3mN,用完记得改掉",
+ Mask: []evalValue{{V: "Qw8!zLp3mN", Kind: "SECRET"}},
+ },
+ {
+ ID: "bearer", From: "true",
+ Text: "curl -H 'Authorization: Bearer " + evalJWT + "' https://api.acme-corp.cn/v1/me",
+ Mask: []evalValue{{V: evalJWT, Kind: "SECRET"}},
+ },
+ {
+ ID: "code-keys", From: "true",
+ Text: "const gh = \"" + evalGitHub + "\";\nconst maps = '" + evalGoogleKey + "';\naws_secret_access_key = " + evalAWSSecret,
+ Mask: []evalValue{{V: evalGitHub, Kind: "SECRET"}, {V: evalGoogleKey, Kind: "SECRET"}, {V: evalAWSSecret, Kind: "SECRET"}},
+ },
+ {
+ ID: "tool-result-mixed", From: "true",
+ Text: "$ cat contacts.csv\nname,email,phone\nRina Kobayashi,[email protected],+81 90-1234-5678\nOmar Haddad,[email protected],+44 7700 900123",
+ Mask: []evalValue{{V: "Rina Kobayashi", Kind: "PERSON"}, {V: "[email protected]", Kind: "EMAIL"}, {V: "90-1234-5678", Kind: "PHONE", Known: "#3591"}, {V: "Omar Haddad", Kind: "PERSON"}, {V: "[email protected]", Kind: "EMAIL"}, {V: "7700 900123", Kind: "PHONE", Known: "#3591"}},
+ Keep: []evalValue{{V: "contacts.csv", Kind: "EMAIL"}},
+ },
+}
diff --git a/internal/redact/eval_test.go b/internal/redact/eval_test.go
new file mode 100644
index 000000000..8fa8b1625
--- /dev/null
+++ b/internal/redact/eval_test.go
@@ -0,0 +1,288 @@
+package redact_test
+
+import (
+ "crypto/sha256"
+ "encoding/hex"
+ "encoding/json"
+ "fmt"
+ "os"
+ "path/filepath"
+ "regexp"
+ "sort"
+ "strings"
+ "testing"
+
+ "github.com/yetone/magpie/internal/pii"
+ "github.com/yetone/magpie/internal/redact"
+)
+
+// The masking evaluation: made-up texts of the kinds agents send (prose,
+// code, logs, tool results, JSON), each with the values that must be
+// masked and those that must go as written. The cases (eval_cases_test.go)
+// start from what users reported masked wrongly, beside the values the
+// masking is for. Nothing in them is anyone's data.
+//
+// Three ways are scored on them:
+// - rules: the rules alone, the model off;
+// - model: what the local model found, as it said it, before any check;
+// - both: the rules and the model with magpie's checks on what it found,
+// as a user with the model on gets.
+//
+// What the model said is recorded in testdata/eval-model.json, so the
+// test runs the whole of "both" without the model or the network, and
+// fails when a value that came out right comes out wrong. A value known
+// to come out wrong says so (evalValue.Known, an issue), and fails once
+// it comes out right, to have its Known taken off.
+//
+// To ask the real model again, after a case is added or a new model,
+// point MAGPIE_EVAL_CACHE at the cache folder it was downloaded into
+// (the folder that holds magpie/privacy-filter):
+//
+// MAGPIE_EVAL_CACHE=$HOME/.cache go test -run TestMaskEval -v ./internal/redact
+//
+// which writes the recording again and logs the table.
+func TestMaskEval(t *testing.T) {
+ rec := readRecording(t)
+ if dir := os.Getenv("MAGPIE_EVAL_CACHE"); dir != "" {
+ rec = askModel(t, dir)
+ writeRecording(t, rec)
+ }
+ var stale []string
+ for _, c := range evalCases {
+ if r, ok := rec[c.ID]; !ok || r.Text != textHash(c.Text) {
+ stale = append(stale, c.ID)
+ }
+ }
+ if len(stale) > 0 {
+ t.Fatalf("the model's recording is not of cases %v; record it again with MAGPIE_EVAL_CACHE (see TestMaskEval)", stale)
+ }
+ replay := func(texts []string) [][]redact.Found {
+ out := make([][]redact.Found, len(texts))
+ for i, s := range texts {
+ for _, c := range evalCases {
+ if c.Text == s {
+ out[i] = rec[c.ID].Found
+ }
+ }
+ }
+ return out
+ }
+
+ var tally evalTally
+ for _, c := range evalCases {
+ rules := redact.Preview(c.Text, redact.Options{Secrets: true, Personal: true})
+ both := redact.Preview(c.Text, redact.Options{Secrets: true, Personal: true, Model: replay})
+ for _, v := range c.Mask {
+ checkValue(t, c, v, true, &tally, gone(rules, v.V), modelFound(c.Text, v.V, rec[c.ID].Found), gone(both, v.V))
+ }
+ for _, v := range c.Keep {
+ checkValue(t, c, v, false, &tally, gone(rules, v.V), modelFound(c.Text, v.V, rec[c.ID].Found), gone(both, v.V))
+ }
+ }
+ t.Log("\n" + tally.table())
+}
+
+// checkValue counts one value in each way, and holds "both" to what the
+// case says of it.
+func checkValue(t *testing.T, c evalCase, v evalValue, mask bool, tally *evalTally, rules, model, both bool) {
+ t.Helper()
+ tally.add(v.Kind, mask, rules, model, both)
+ if rules != mask || model != mask || both != mask {
+ tally.wrong = append(tally.wrong, fmt.Sprintf("%-18s %-6s %-7s %-5s %-5s %-5s %q",
+ c.ID, map[bool]string{true: "mask", false: "keep"}[mask], v.Kind, mark(rules == mask), mark(model == mask), mark(both == mask), v.V))
+ }
+ if v.Known != "" {
+ if both == mask {
+ t.Errorf("%s: %q now comes out right; take its Known (%s) off", c.ID, v.V, v.Known)
+ }
+ return
+ }
+ switch {
+ case mask && !both:
+ t.Errorf("%s: %s %q went as written", c.ID, v.Kind, v.V)
+ case !mask && both:
+ t.Errorf("%s: %q was masked, and should go as written", c.ID, v.V)
+ }
+}
+
+// placeholders are what masking puts in a value's place; a made-up key
+// keeps no trace of the value, so the value is gone if it isn't there.
+var placeholders = regexp.MustCompile(`\{\{[A-Z_]+_[a-z2-7]{8}\}\}`)
+
+// gone says v is not in out as it is: masked, or a part of it masked.
+// The placeholders are taken out first, since their suffix may spell v.
+func gone(out, v string) bool {
+ return !strings.Contains(placeholders.ReplaceAllString(out, "\x00"), v)
+}
+
+// modelFound says the model put a span of a kind magpie masks over some
+// of v, where v is in text.
+func modelFound(text, v string, fs []redact.Found) bool {
+ for at := 0; ; {
+ i := strings.Index(text[at:], v)
+ if i < 0 {
+ return false
+ }
+ a, b := at+i, at+i+len(v)
+ for _, f := range fs {
+ if redact.ModelKinds[f.Label] != "" && f.Start < b && f.End > a {
+ return true
+ }
+ }
+ at = b
+ }
+}
+
+// evalTally counts, by kind, the values each way masked: of those to
+// mask (found), and of those to keep (wrongly).
+type evalTally struct {
+ kinds map[string]*evalCount
+ wrong []string // the values some way got wrong
+}
+
+func mark(right bool) string {
+ if right {
+ return "ok"
+ }
+ return "WRONG"
+}
+
+type evalCount struct {
+ mask, keep int
+ found, wrong [3]int // rules, model, both
+}
+
+func (t *evalTally) add(kind string, mask bool, got ...bool) {
+ if t.kinds == nil {
+ t.kinds = map[string]*evalCount{}
+ }
+ n := t.kinds[kind]
+ if n == nil {
+ n = &evalCount{}
+ t.kinds[kind] = n
+ }
+ if mask {
+ n.mask++
+ } else {
+ n.keep++
+ }
+ for i, g := range got {
+ if g && mask {
+ n.found[i]++
+ } else if g {
+ n.wrong[i]++
+ }
+ }
+}
+
+// table is the tally: for each kind, how many of the values to mask each
+// way masked, and how many of those to keep it masked all the same.
+func (t *evalTally) table() string {
+ var ks []string
+ all := &evalCount{}
+ for k, n := range t.kinds {
+ ks = append(ks, k)
+ all.mask += n.mask
+ all.keep += n.keep
+ for i := range 3 {
+ all.found[i] += n.found[i]
+ all.wrong[i] += n.wrong[i]
+ }
+ }
+ sort.Strings(ks)
+ var b strings.Builder
+ fmt.Fprintf(&b, "%-10s %18s %18s %18s\n", "", "rules", "model", "both")
+ fmt.Fprintf(&b, "%-10s %8s %9s %8s %9s %8s %9s\n", "kind", "masked", "wrongly", "masked", "wrongly", "masked", "wrongly")
+ line := func(k string, n *evalCount) {
+ fmt.Fprintf(&b, "%-10s", k)
+ for i := range 3 {
+ fmt.Fprintf(&b, " %8s %9s ", frac(n.found[i], n.mask), frac(n.wrong[i], n.keep))
+ }
+ b.WriteString("\n")
+ }
+ for _, k := range ks {
+ line(k, t.kinds[k])
+ }
+ line("all", all)
+ fmt.Fprintf(&b, "\n%-18s %-6s %-7s %-5s %-5s %-5s %s\n", "case", "should", "kind", "rules", "model", "both", "value")
+ for _, w := range t.wrong {
+ b.WriteString(w + "\n")
+ }
+ return b.String()
+}
+
+func frac(n, of int) string {
+ if of == 0 {
+ return "-"
+ }
+ return fmt.Sprintf("%d/%d", n, of)
+}
+
+// ---- the recording ----------------------------------------------------------
+
+const recordingPath = "testdata/eval-model.json"
+
+// recorded is what the model found in one case's text, and the text's
+// hash, to tell a recording made of another text.
+type recorded struct {
+ Text string `json:"text"`
+ Found []redact.Found `json:"found"`
+}
+
+func textHash(s string) string {
+ h := sha256.Sum256([]byte(s))
+ return hex.EncodeToString(h[:8])
+}
+
+func readRecording(t *testing.T) map[string]recorded {
+ rec := map[string]recorded{}
+ b, err := os.ReadFile(recordingPath)
+ if os.IsNotExist(err) {
+ return rec
+ }
+ if err != nil {
+ t.Fatal(err)
+ }
+ if err := json.Unmarshal(b, &rec); err != nil {
+ t.Fatalf("%s: %v", recordingPath, err)
+ }
+ return rec
+}
+
+func writeRecording(t *testing.T, rec map[string]recorded) {
+ b, err := json.MarshalIndent(rec, "", "\t")
+ if err != nil {
+ t.Fatal(err)
+ }
+ tmp := recordingPath + ".tmp"
+ if err := os.WriteFile(tmp, append(b, '\n'), 0o644); err != nil {
+ t.Fatal(err)
+ }
+ if err := os.Rename(tmp, recordingPath); err != nil {
+ t.Fatal(err)
+ }
+}
+
+// askModel runs every case's text through the real model, downloaded
+// into the cache folder dir, one text at a time, as a request's text
+// that has the model to itself is.
+func askModel(t *testing.T, dir string) map[string]recorded {
+ t.Setenv("XDG_CACHE_HOME", dir)
+ if !pii.Installed() {
+ t.Fatalf("the model isn't in %s", filepath.Join(dir, "magpie", "privacy-filter"))
+ }
+ defer pii.Stop()
+ rec := map[string]recorded{}
+ for _, c := range evalCases {
+ got, err := pii.Find([]string{c.Text})
+ if err != nil {
+ t.Fatalf("%s: %v", c.ID, err)
+ }
+ fs := got[0]
+ if fs == nil {
+ fs = []redact.Found{}
+ }
+ rec[c.ID] = recorded{Text: textHash(c.Text), Found: fs}
+ }
+ return rec
+}
diff --git a/internal/redact/export_test.go b/internal/redact/export_test.go
new file mode 100644
index 000000000..9bc032e8d
--- /dev/null
+++ b/internal/redact/export_test.go
@@ -0,0 +1,5 @@
+package redact
+
+// ModelKinds is modelKinds, for the evaluation in package redact_test,
+// which reads the real model through internal/pii (eval_test.go).
+var ModelKinds = modelKinds
diff --git a/internal/redact/testdata/eval-model.json b/internal/redact/testdata/eval-model.json
new file mode 100644
index 000000000..01e53b05e
--- /dev/null
+++ b/internal/redact/testdata/eval-model.json
@@ -0,0 +1,403 @@
+{
+ "asset-files": {
+ "text": "8b019ca5e2af716b",
+ "found": []
+ },
+ "assistant-names": {
+ "text": "6f312259162fb240",
+ "found": []
+ },
+ "bearer": {
+ "text": "708d72f298141a4c",
+ "found": [
+ {
+ "Start": 31,
+ "End": 148,
+ "Label": "secret"
+ },
+ {
+ "Start": 150,
+ "End": 180,
+ "Label": "private_url"
+ }
+ ]
+ },
+ "card": {
+ "text": "7387f43ca5ce5824",
+ "found": []
+ },
+ "code-keys": {
+ "text": "d5f835bbf0a332b8",
+ "found": [
+ {
+ "Start": 12,
+ "End": 50,
+ "Label": "secret"
+ },
+ {
+ "Start": 67,
+ "End": 108,
+ "Label": "secret"
+ },
+ {
+ "Start": 135,
+ "End": 175,
+ "Label": "secret"
+ }
+ ]
+ },
+ "code-locations": {
+ "text": "ebbde1d7b5199206",
+ "found": []
+ },
+ "colors-ports": {
+ "text": "252b9ce2e8df3e82",
+ "found": [
+ {
+ "Start": 14,
+ "End": 21,
+ "Label": "secret"
+ }
+ ]
+ },
+ "digests": {
+ "text": "3b3e688a54f12e96",
+ "found": [
+ {
+ "Start": 8,
+ "End": 72,
+ "Label": "secret"
+ }
+ ]
+ },
+ "env-file": {
+ "text": "c1f1a7ab84f78373",
+ "found": [
+ {
+ "Start": 15,
+ "End": 63,
+ "Label": "secret"
+ },
+ {
+ "Start": 76,
+ "End": 90,
+ "Label": "secret"
+ }
+ ]
+ },
+ "example-mail": {
+ "text": "a1867625846dc315",
+ "found": [
+ {
+ "Start": 14,
+ "End": 30,
+ "Label": "private_email"
+ }
+ ]
+ },
+ "git-author": {
+ "text": "94a966005e2adee1",
+ "found": [
+ {
+ "Start": 23,
+ "End": 33,
+ "Label": "private_person"
+ },
+ {
+ "Start": 35,
+ "End": 55,
+ "Label": "private_email"
+ }
+ ]
+ },
+ "git-user-url": {
+ "text": "b92e10fa6b923a00",
+ "found": [
+ {
+ "Start": 10,
+ "End": 55,
+ "Label": "private_url"
+ }
+ ]
+ },
+ "grep-pattern": {
+ "text": "0d38dbff0a086bdf",
+ "found": []
+ },
+ "home-path": {
+ "text": "4f79a2f3631db506",
+ "found": [
+ {
+ "Start": 4,
+ "End": 40,
+ "Label": "private_url"
+ }
+ ]
+ },
+ "iban": {
+ "text": "cf32ccaec3a1dfc1",
+ "found": [
+ {
+ "Start": 24,
+ "End": 51,
+ "Label": "account_number"
+ }
+ ]
+ },
+ "json-contact": {
+ "text": "8b8300472a287a74",
+ "found": [
+ {
+ "Start": 21,
+ "End": 30,
+ "Label": "private_person"
+ },
+ {
+ "Start": 41,
+ "End": 62,
+ "Label": "private_email"
+ },
+ {
+ "Start": 70,
+ "End": 90,
+ "Label": "private_phone"
+ }
+ ]
+ },
+ "log-user": {
+ "text": "cda01fa1bd419b1d",
+ "found": [
+ {
+ "Start": 0,
+ "End": 19,
+ "Label": "private_date"
+ },
+ {
+ "Start": 35,
+ "End": 52,
+ "Label": "private_email"
+ },
+ {
+ "Start": 68,
+ "End": 76,
+ "Label": "private_url"
+ }
+ ]
+ },
+ "noreply": {
+ "text": "271c7650b6474a19",
+ "found": []
+ },
+ "phone-intl": {
+ "text": "28bf596367da1b2b",
+ "found": [
+ {
+ "Start": 36,
+ "End": 39,
+ "Label": "private_person"
+ },
+ {
+ "Start": 43,
+ "End": 58,
+ "Label": "private_phone"
+ }
+ ]
+ },
+ "prose-password": {
+ "text": "b45e2539ee55a672",
+ "found": [
+ {
+ "Start": 27,
+ "End": 38,
+ "Label": "secret"
+ }
+ ]
+ },
+ "prose-person": {
+ "text": "7090b229a3288b42",
+ "found": [
+ {
+ "Start": 13,
+ "End": 27,
+ "Label": "private_person"
+ }
+ ]
+ },
+ "retina-file": {
+ "text": "6210ece83f6b71ab",
+ "found": []
+ },
+ "session-uuid": {
+ "text": "45ff8afdbe77e369",
+ "found": [
+ {
+ "Start": 39,
+ "End": 81,
+ "Label": "secret"
+ }
+ ]
+ },
+ "sizes-versions": {
+ "text": "e08a850f552a995b",
+ "found": []
+ },
+ "snake-arg": {
+ "text": "7ade5b7b89a61dd4",
+ "found": []
+ },
+ "snake-words": {
+ "text": "89fcc6da0d4cc9fb",
+ "found": []
+ },
+ "token-count": {
+ "text": "359c740b1ce9a3c5",
+ "found": [
+ {
+ "Start": 31,
+ "End": 39,
+ "Label": "account_number"
+ }
+ ]
+ },
+ "tool-ids": {
+ "text": "50c8c97e610d704c",
+ "found": [
+ {
+ "Start": 37,
+ "End": 67,
+ "Label": "account_number"
+ },
+ {
+ "Start": 93,
+ "End": 121,
+ "Label": "account_number"
+ }
+ ]
+ },
+ "tool-result-mixed": {
+ "text": "f5a49344e0e7f392",
+ "found": [
+ {
+ "Start": 36,
+ "End": 50,
+ "Label": "private_person"
+ },
+ {
+ "Start": 50,
+ "End": 70,
+ "Label": "private_email"
+ },
+ {
+ "Start": 70,
+ "End": 87,
+ "Label": "private_phone"
+ },
+ {
+ "Start": 88,
+ "End": 99,
+ "Label": "private_person"
+ },
+ {
+ "Start": 100,
+ "End": 121,
+ "Label": "private_email"
+ },
+ {
+ "Start": 121,
+ "End": 137,
+ "Label": "private_phone"
+ }
+ ]
+ },
+ "tool-servers": {
+ "text": "6b9515c4bf396f51",
+ "found": []
+ },
+ "unix-time": {
+ "text": "3e3d22cf38729962",
+ "found": []
+ },
+ "unix-time-ms": {
+ "text": "7e29836c6aa4f105",
+ "found": []
+ },
+ "url-userinfo": {
+ "text": "a74a01c8225d54da",
+ "found": [
+ {
+ "Start": 27,
+ "End": 55,
+ "Label": "secret"
+ }
+ ]
+ },
+ "us-address": {
+ "text": "d73e7386a905373e",
+ "found": [
+ {
+ "Start": 24,
+ "End": 68,
+ "Label": "private_address"
+ },
+ {
+ "Start": 78,
+ "End": 92,
+ "Label": "private_phone"
+ }
+ ]
+ },
+ "zh-contact": {
+ "text": "5d654b2bbdd23aa1",
+ "found": [
+ {
+ "Start": 12,
+ "End": 18,
+ "Label": "private_person"
+ },
+ {
+ "Start": 28,
+ "End": 39,
+ "Label": "private_phone"
+ },
+ {
+ "Start": 51,
+ "End": 93,
+ "Label": "private_address"
+ }
+ ]
+ },
+ "zh-id": {
+ "text": "f058f91bb1800cbe",
+ "found": [
+ {
+ "Start": 13,
+ "End": 31,
+ "Label": "account_number"
+ }
+ ]
+ },
+ "zh-password": {
+ "text": "ca3835176b202e40",
+ "found": [
+ {
+ "Start": 19,
+ "End": 29,
+ "Label": "secret"
+ }
+ ]
+ },
+ "zh-prose": {
+ "text": "c75e8925f3dddfc7",
+ "found": [
+ {
+ "Start": 0,
+ "End": 15,
+ "Label": "private_person"
+ }
+ ]
+ },
+ "zh-words": {
+ "text": "dfccf70c683fe55b",
+ "found": []
+ }
+}