Every agent's model. One place. Codex on DeepSeek, Claude Code on Kimi, from the menu bar.
// Package redact keeps secrets on the machine. What the gateway sends a
// vendor has its secrets (API keys, private keys, tokens, passwords in
// connection strings and assignments), and when asked personal data and the
// user's own words, swapped for placeholders like {{API_KEY_k3v9x2mq}}; what
// the vendor says back has them swapped back, streams included, so the agent
// and the user see the real thing while the vendor never does.
//
// A placeholder is the same for the same value every time (a keyed hash of
// it), so a conversation's history reads the same turn after turn — what a
// vendor cached of it stays good, and a thinking block's signature still
// matches the text it was made for — and one placeholder is never two
// values. The values are held in memory only, by placeholder, as requests
// bring them; a restart loses nothing, since the next request brings them
// again. A value once masked is masked wherever a request has it, whatever
// words are around it (known.go): the vendor writes about a placeholder in
// words no rule knows, and the agent sends that text back with the value.
package redact
import (
"crypto/hmac"
"crypto/rand"
"crypto/sha256"
"encoding/base32"
"encoding/json"
"os"
"path/filepath"
"regexp"
"slices"
"sort"
"strings"
"sync"
"unicode"
"unicode/utf8"
"golang.org/x/net/publicsuffix"
)
// Options says what to mask.
type Options struct {
Secrets bool // API keys, private keys, tokens, passwords
Personal bool // emails, phone numbers, ID and bank card numbers
Words []string // the user's own: names, codenames, hosts
// WordsAnyCase matches Words in any case: Nightjar masks NIGHTJAR and
// nightjar too. Off, a word is matched as written.
WordsAnyCase bool
// WordsWhole matches Words only as whole words (acmilan_ on Discord):
// ABC masks "ABC IS" and 'ABC' but not XABCD, and 123456 masks no part
// of 91234567, 0.123456 or 123456.78 (wholeWord). Off, a word is found
// anywhere, inside a longer one too.
WordsWhole bool
Rules []Rule // the user's own rules, masked with the secrets
// Kinds turns each of the Categories of personal data on or off by its
// id, where the user chose; one not in it is as its Category.On says.
// None of them is masked while Personal is off.
Kinds map[string]bool
// Model, where set, finds more in the texts to mask (model.go)
Model Model
// Route names where a request masked for a vendor may go (the gateway
// sets it: the providers a model, its fallbacks or a routing group's
// members are served by). A text sent on it before goes on it again
// as it went (sentSpans); on another route it is masked afresh.
Route string
// MinScore is the score the user has Model's finds need (Settings →
// Privacy's slider), which Model's own finds are held to already. A
// value only the model found, masked before at a lower score, isn't
// masked again as known under it (knownSpans).
MinScore float64
// NamesByModel says Model finds Chinese names itself (AstrLink
// Guard): beside it only a nickname (小王, 老李) is found by the
// words around it (zhNameSpans)
NamesByModel bool
// Shape sends a key as a made-up key of its form, not a placeholder
// (shape.go)
Shape bool
// Actions is what is done with a value of a kind (a Hit's Kind) other
// than masking it, where the user chose: Warn sends it as written,
// Block masks it and the request is held back (the gateway's). A kind
// not in it is masked.
Actions map[string]string
// Allow are values the user lets go as written, whichever finder finds
// them: each a whole value, in any case.
Allow []string
// Seen, where set, is told of each value masked, warned of or blocked
Seen func(Hit)
// found are the spans Model found, by text, for one request's texts
found map[string][]Found
// sent is masking for a vendor (Mask, MaskJSON): a text masked
// before on Route has the model's finds and the known values it had
// then (sentSpans)
sent bool
}
// covers says o masks what a rule of category cat finds: "" are the
// secrets, catPersonal the personal data always masked with it, any other
// one of the Categories.
func (o Options) covers(cat string) bool {
switch cat {
case "":
return o.Secrets
case catPersonal:
return o.Personal
}
if !o.Personal {
return false
}
if on, ok := o.Kinds[cat]; ok {
return on
}
return categoryOn[cat]
}
// rule finds one kind of value. The match is group 1 when the pattern has
// one, else all of it; bound are the characters that may not touch it on
// either side (a key inside a longer word is no key); ok checks it further.
type rule struct {
kind string
re *regexp.Regexp
markers []string // one of these is in any text it can match
// fold are markers too, looked for in the text in lower case, for a
// pattern that matches its words in any case (Passport, PASSPORT)
fold []string
bound string
ok func(string) bool
// at checks the value at s[a:b] by what is around it: a dotted quad
// right after "version" is no address
at func(s string, a, b int) bool
// fix, where set, checks the value in place of ok and may shorten it:
// an IBAN printed in groups of four, followed by a word in capitals
fix func(s string, a, b int) (int, int, bool)
// back and ahead, where ahead is set, are how far before and after
// one of fold a match can be: the pattern is only run there, as one
// that takes its words in any case is slow over all of a long text
back, ahead int
// cat is what turns the rule on (Options.covers): "" the secrets
cat string
// yours is a rule the user wrote (customRules)
yours bool
// anyGroup takes the first group that matched, for a pattern with a
// group in each of its alternatives
anyGroup bool
// groups, where set, are the kinds of a match's groups, each masked as
// a value of its own (the user and the host of user@host), and groupOK
// checks each
groups []string
groupOK []func(string) bool
}
const (
alnum = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789"
tokenCh = alnum + "_-"
digits = "0123456789"
)
var rules = append([]rule{
{kind: "PRIVATE_KEY", re: regexp.MustCompile(`-----BEGIN (?:[A-Z0-9]+ )*PRIVATE KEY(?: BLOCK)?-----[\s\S]+?-----END (?:[A-Z0-9]+ )*PRIVATE KEY(?: BLOCK)?-----`), markers: []string{"PRIVATE KEY"}},
{kind: "API_KEY", re: regexp.MustCompile(`sk-(?:ant-|proj-|or-|svcacct-|admin-)?[A-Za-z0-9_-]{20,}`), markers: []string{"sk-"}, bound: tokenCh, ok: notASlug},
// a gateway's or a relay's own: oc_sk_…, or_sk_… (#195)
{kind: "API_KEY", re: regexp.MustCompile(`[a-z]{2,4}_sk_[A-Za-z0-9_-]{20,}`), markers: []string{"_sk_"}, bound: tokenCh, ok: secretValue},
{kind: "API_KEY", re: regexp.MustCompile(`(?:gh[pousr]_[A-Za-z0-9]{30,}|github_pat_[A-Za-z0-9_]{40,}|glpat-[A-Za-z0-9_-]{20,})`), markers: []string{"gh", "github_pat_", "glpat-"}, bound: tokenCh},
{kind: "API_KEY", re: regexp.MustCompile(`AIza[0-9A-Za-z_-]{35}`), markers: []string{"AIza"}, bound: tokenCh},
{kind: "API_KEY", re: regexp.MustCompile(`xox[abposr]-[0-9A-Za-z-]{10,}`), markers: []string{"xox"}, bound: tokenCh},
{kind: "API_KEY", re: regexp.MustCompile(`(?:sk|rk)_live_[0-9A-Za-z]{20,}`), markers: []string{"_live_"}, bound: tokenCh},
{kind: "API_KEY", re: regexp.MustCompile(`(?:AKIA|ASIA)[0-9A-Z]{16}`), markers: []string{"AKIA", "ASIA"}, bound: alnum},
{kind: "API_KEY", re: regexp.MustCompile(`(?:hf_[A-Za-z0-9]{30,}|gsk_[A-Za-z0-9]{40,}|xai-[A-Za-z0-9]{40,}|npm_[A-Za-z0-9]{36}|pypi-[A-Za-z0-9_-]{50,}|dop_v1_[a-f0-9]{64}|SG\.[A-Za-z0-9_-]{22}\.[A-Za-z0-9_-]{43})`),
markers: []string{"hf_", "gsk_", "xai-", "npm_", "pypi-", "dop_v1_", "SG."}, bound: tokenCh},
{kind: "TOKEN", re: regexp.MustCompile(`eyJ[A-Za-z0-9_-]{8,}\.eyJ[A-Za-z0-9_-]{8,}\.[A-Za-z0-9_-]{8,}`), markers: []string{"eyJ"}, bound: tokenCh},
// the password of user:password@host, or of :password@host as a Redis
// URL has it with no user
{kind: "PASSWORD", re: regexp.MustCompile(`[A-Za-z][A-Za-z0-9+.-]*://[^\s:@/?#"'<>]*:([^\s@/?#"'<>]{3,})@`), markers: []string{"://"}, ok: notAVariable, at: hiddenInURL},
// password = …, API_KEY: "…", as .env files and configs have them
{kind: "SECRET", re: regexp.MustCompile(`(?i)[A-Za-z0-9_.-]*(?:` + secretNames.pattern() + `)[A-Za-z0-9_.-]*` + quote + `[ \t]*[:=][ \t]*` + quote + `(` + secretCh + `{8,})`),
markers: secretNames.markers(), ok: secretValue, at: fieldAt},
// a field called key and nothing more, as providers.json has one: "key":
// "oc_sk_…" (#195). Only a long value of letters and digits both, since
// code calls anything a key: a map's "key": "value", sort_key, keyboard.
{kind: "SECRET", re: regexp.MustCompile(`(?i)(?:^|[^A-Za-z0-9_.-])` + quote + `key` + quote + `[ \t]*[:=][ \t]*` + quote + `(` + secretCh + `{16,})`),
markers: fieldNames{{"key", "key"}}.markers(), ok: secretValue, at: fieldAt},
// a password after its name in Chinese, Japanese or Korean, as people
// write one into a message: 密码:786kih, 密碼是 hunter2, パスワード: … .
// Unlike password = in a config, nothing in code is called 密码, so
// any value of four or more is taken, digits only too (CyberCaius on
// Discord); a sentence's closing stop is no part of it.
{kind: "PASSWORD", re: cjkNamed(cjkPasswordNames), fold: lowerAll(cjkPasswordNames), back: 64, ahead: 100, ok: notAVariable},
// a key, a secret or a token after its name the same way: 密钥为 …,
// 令牌是 …, 金鑰: … . A key of digits only is taken after 为, 是, :
// or = when it has six or more and counts nothing, so 密钥为12231241
// is masked and 密钥为 2048 位 is left.
{kind: "SECRET", re: cjkNamed(cjkKeyNames), fold: lowerAll(cjkKeyNames), back: 64, ahead: 100, at: keyAt},
// an English name for one before 为, 為, 是 or :, as a Chinese message
// has it: apikey 为 …, token 是 …, API Key:… (CyberCaius on Discord).
// password = … and API_KEY: … in a config are the SECRET rule's above.
{kind: "SECRET", re: regexp.MustCompile(`(?i)(?:` + secretNames.pattern() + `|pwd|api[ ]key|access[ ]key|secret[ ]key|key|\bpin(?:[ ]?code)?)` + quote +
`[ \t]*(?:[为為是][ \t]*[::]?|:)[ \t]*(?:\\?["'“「])?(` + secretCh + `{3,}[A-Za-z0-9_\-/+=~!@#%^&*])`),
fold: cjkCopulas, back: 24, ahead: 260, at: keyAt},
{kind: "EMAIL", re: regexp.MustCompile(`[A-Za-z0-9._%+-]+@[A-Za-z0-9-]+(?:\.[A-Za-z0-9-]+)*\.[A-Za-z]{2,}`), markers: []string{"@"}, bound: alnum + "_%+-", ok: realEmail, at: emailAt, cat: catPersonal},
{kind: "ID_CARD", re: regexp.MustCompile(`[1-9]\d{5}(?:18|19|20)\d{2}(?:0[1-9]|1[0-2])(?:0[1-9]|[12]\d|3[01])\d{3}[\dXx]`), bound: alnum, ok: chineseID, cat: catPersonal},
{kind: "PHONE", re: regexp.MustCompile(`(?:\+?86[- ]?)?1[3-9]\d{9}`), bound: alnum, at: phoneAt, cat: catPersonal},
{kind: "BANK_CARD", re: regexp.MustCompile(`[3-6]\d{3}(?:[ -]?\d{4}){2,3}(?:[ -]?\d{1,3})?`), bound: alnum, ok: luhn, cat: catPersonal},
}, moreRules...)
// cjkPasswordNames are what a password is called in Chinese, Japanese and
// Korean.
var cjkPasswordNames = []string{"密码", "密碼", "口令", "PIN码", "PIN碼", "PIN 码", "安全码", "安全碼", "パスワード", "暗証番号", "비밀번호", "암호", "비번"}
// cjkKeyNames are what a key, a secret or a token is called in Chinese,
// Simplified and Traditional, Japanese and Korean.
var cjkKeyNames = []string{
"密钥", "密鑰", "秘钥", "秘鑰", "私钥", "私鑰", "金钥", "金鑰", "令牌", "权杖", "權杖", "授权码", "授權碼",
"秘密鍵", "トークン", "シークレット", "APIキー", "アクセスキー",
"토큰", "비밀키", "API 키", "API키",
}
// cjkCopulas are what stands between a name and its value in a Chinese
// message: 为, 為, 是, a full-width colon.
var cjkCopulas = []string{"为", "為", "是", ":"}
// lowerAll is names as fold has them, in lower case.
func lowerAll(names []string) []string {
out := make([]string, len(names))
for i, n := range names {
out[i] = strings.ToLower(n)
}
return out
}
// cjkNamed is a value after one of names: after :, :, =, 为, 為, 是, は
// or 는, quoted or not, or after nothing but a space.
func cjkNamed(names []string) *regexp.Regexp {
return regexp.MustCompile(`(?i)(?:` + strings.Join(names, "|") + `)[ \t]*(?:[::=][ \t]*|[为為是][ \t]*[::]?[ \t]*|は[ \t]*|는[ \t]*)?(?:\\?["'“「])?(` + secretCh + `{3,63}[A-Za-z0-9_\-/+=~!@#%^&*])`)
}
// fieldNames are what a field that holds a secret is called: a pattern for
// each name, and text that any match of it has in it, so a rule's markers
// and the names its pattern takes come from one list and can't drift apart.
type fieldNames []struct{ re, marker string }
// secretNames are what a field holding a secret has in its name:
// DB_PASSWORD, apiKey, x-api-key, client_secret.
var secretNames = fieldNames{
{"password", "pass"}, {"passwd", "pass"},
{"secret", "secret"},
{"token", "token"},
{`api[_-]?key`, "key"}, {`access[_-]?key`, "key"}, {`private[_-]?key`, "key"},
{"credential", "credential"},
}
func (f fieldNames) pattern() string {
res := make([]string, len(f))
for i, n := range f {
res[i] = n.re
}
return strings.Join(res, "|")
}
// markers are each name's marker as a name is written in lower case, in
// upper case and capitalised: password, PASSWORD, Password, apiKey.
func (f fieldNames) markers() []string {
var out []string
for _, n := range f {
for _, m := range []string{n.marker, strings.ToUpper(n.marker), strings.ToUpper(n.marker[:1]) + n.marker[1:]} {
if !slices.Contains(out, m) {
out = append(out, m)
}
}
}
return out
}
const (
// quote is a field's or a value's quote, if it has one: " or ', or \"
// in JSON written inside a string
quote = `(?:\\?["'])?`
// secretCh are the characters of a secret's value
secretCh = `[A-Za-z0-9_\-./+=~!@#%^&*]`
)
// placeholderRe is a placeholder as Mask writes it.
var placeholderRe = regexp.MustCompile(`\{\{[A-Z][A-Z0-9_]*_[a-z2-7]{8}\}\}`)
// notAVariable turns away what stands in for a value: ${PASS}, <password>, ****.
func notAVariable(v string) bool {
if strings.ContainsAny(v[:1], "$%{<[*") {
return false
}
return strings.Trim(v, "*xX.") != ""
}
// keyValue is a key's or a token's value, not a word or a number: letters
// and digits both, or 16 characters or more that aren't all digits. token 是
// expired, key 是 value and 密钥为 2048 are left.
func keyValue(v string) bool {
if !notAVariable(v) {
return false
}
letter := strings.IndexFunc(v, func(r rune) bool { return r >= 'A' && r <= 'Z' || r >= 'a' && r <= 'z' }) >= 0
return letter && (strings.ContainsAny(v, digits) || len(v) >= 16)
}
// keyAt is keyValue for s[a:b], or six digits or more right after 为, 為,
// 是, :, :, =, は or 는 that count nothing: a key said in a message is
// often a number (密钥为12231241, token 是 20261025, CyberCaius on
// Discord), while 密钥为 2048 位 and 令牌 300000 毫秒 are a size and a
// time.
func keyAt(s string, a, b int) bool {
v := s[a:b]
if pathValue(v) || versionValue(v) {
// where a key is kept: 私钥 ~/.ssh/id_ed25519, or a release's
// version: token 是 0.15.0-rc1 (fieldvalue.go)
return false
}
if keyValue(v) {
return true
}
if len(v) < 6 || strings.Trim(v, digits) != "" {
return false
}
before := strings.TrimRight(s[:a], " \t\"'“「\\")
after := strings.TrimLeft(s[b:], " \t")
return slices.ContainsFunc(keySeparators, func(c string) bool { return strings.HasSuffix(before, c) }) &&
!slices.ContainsFunc(quantityUnits, func(u string) bool { return strings.HasPrefix(after, u) })
}
// keySeparators are what stands between a name and a key's value.
var keySeparators = []string{"为", "為", "是", ":", ":", "=", "は", "는"}
// quantityUnits are what a number counts when one follows it: bits, bytes,
// a time, a count.
var quantityUnits = []string{"位", "比特", "字节", "字節", "个", "個", "次", "秒", "毫秒", "分钟", "分鐘", "小时", "小時", "天", "年", "bit", "byte", "ms", "sec", "min", "ビット", "バイト", "비트", "바이트"}
// secretValue is a value that looks like a secret, not code: letters and
// digits both, and not a call, a reference or a placeholder of its own.
func secretValue(v string) bool {
if !notAVariable(v) || strings.HasPrefix(v, "process.env") || strings.HasPrefix(v, "os.") {
return false
}
return strings.ContainsAny(v, digits) && strings.IndexFunc(v, func(r rune) bool { return r >= 'A' && r <= 'Z' || r >= 'a' && r <= 'z' }) >= 0
}
// identPath is a name in code, or a field read through one: Uint8Array,
// details.2, opts.token2.
var identPath = regexp.MustCompile(`^[A-Za-z_][A-Za-z0-9_]*(?:\.[A-Za-z0-9_]+)*$`)
// notCode turns away a value at s[a:b] that is code naming a secret, not
// the secret: a call (privateKey: toBase64Url(…), secretKey =
// fromBase64Url(…)), or, unquoted after a colon, a type or a read that
// code goes on from (secretKey: Uint8Array;, private_key: details.2,).
// A secret written in code is a string, so in quotes; one after = or at a
// line's end (.env files, logs, YAML) is kept (Kieran Zhang on X, 10-25).
func notCode(s string, a, b int) bool {
if !identPath.MatchString(s[a:b]) {
return true
}
rest := strings.TrimLeft(s[b:], " \t")
if strings.HasPrefix(rest, "(") {
return false
}
if a > 0 && (s[a-1] == '"' || s[a-1] == '\'') || !strings.HasSuffix(strings.TrimRight(s[:a], " \t"), ":") {
return true
}
return rest == "" || !strings.ContainsRune(";,)}]|?&<>", rune(rest[0]))
}
// notUserinfo turns away what looks like an email at s[a:b] but is a
// URL's user:password@host or user@host: its @ ends the userinfo, so
// what is before it is a user name or a password and what is after it
// the host. postgres://app:[email protected]:5432/x went as
// app:{{EMAIL_…}}:5432, the password and the host one address, where the
// password is the PASSWORD rule's and the host stays (#3538). An address
// in a URL's path or query ([email protected]) and in mailto: is an
// email still.
func notUserinfo(s string, a, _ int) bool {
return !inUserinfo(s, a)
}
// emailAt is where an email at s[a:b] can be: not in a URL's userinfo
// (notUserinfo), and with no dot touching it that makes it part of a
// longer name: none before it, and none after it that a letter, digit,
// - or _ follows ([email protected] is no address). A dot after it that ends a
// sentence is the sentence's: "The user's email address is
// [email protected]." masked nothing while the dot was a bound character, and
// the address went to the vendor as written.
func emailAt(s string, a, b int) bool {
if a > 0 && s[a-1] == '.' {
return false
}
if b+1 < len(s) && s[b] == '.' && strings.IndexByte(tokenCh, s[b+1]) >= 0 {
return false
}
return notUserinfo(s, a, b)
}
// inUserinfo says s[a] is inside a URL's authority, after its scheme://
// and before the host: nothing from there back to the // is a /, ?, #,
// space, quote, < or > or @.
func inUserinfo(s string, a int) bool {
i := a
for i > 0 && !strings.ContainsRune(" \t\r\n/?#\"'<>@`", rune(s[i-1])) {
i--
}
if i < 3 || s[i-1] != '/' || s[i-2] != '/' || s[i-3] != ':' {
return false
}
j := i - 3 // the scheme ends at s[j], its colon
k := j
for k > 0 && (s[k-1] >= 'a' && s[k-1] <= 'z' || s[k-1] >= 'A' && s[k-1] <= 'Z' || s[k-1] >= '0' && s[k-1] <= '9' || strings.IndexByte("+.-", s[k-1]) >= 0) {
k--
}
for k < j && !(s[k] >= 'a' && s[k] <= 'z' || s[k] >= 'A' && s[k] <= 'Z') {
k++ // a scheme starts with a letter
}
return k < j
}
// hiddenInURL says the password at s[a:b], in a URL's user:password@,
// is hidden by masking it: it is not also written as it is elsewhere in
// the same URL, as its user name, a part of its host or of its path. A
// test database's URL often gives one word for all three:
// postgresql://geodatafusion:[email protected]:55432/geodatafusion
// is the PostGIS test of the geodatafusion repo. Its password was masked
// while the URL still had it twice as written, and once masked the word
// was known (knownSpans): the project's folders, its crate, its .toml
// and cargo test -p geodatafusion went to the vendor as a made-up word
// for the rest of the run, as a password (xuanwo on X, 10-29: "很多项目名
// 被识别成了 password", 26M masked in a day). postgres:postgres,
// root:root and guest:guest are the same: anyone who reads the user
// reads the password. A password not written elsewhere in its URL is
// masked as before, whatever it looks like: app:geodatafusion@ masks it
// there and, as known, wherever it is next.
func hiddenInURL(s string, a, b int) bool {
pw := s[a:b]
i := strings.LastIndex(s[:a], "//")
if i < 0 || s[a-1] != ':' {
return true
}
if s[i+2:a-1] == pw {
return false // the user name
}
// the host and the path, to the URL's end
end := b + 1
for end < len(s) && !strings.ContainsRune(" \t\r\n\"'<>`\\)]}", rune(s[end])) {
end++
}
if b+1 > end {
return true
}
for _, part := range strings.FieldsFunc(s[b+1:end], func(r rune) bool { return strings.ContainsRune("/:.?#&=;,@", r) }) {
if part == pw {
return false
}
}
return true
}
// realEmail is whether v, matched as an address, is someone's. git@ is
// the user every git host is reached by over SSH ([email protected]:o/r.git,
// [email protected], a company's own [email protected]), no one's mailbox, and
// was masked so a remote no longer read as one (jumy on Discord).
// notASlug is false for words in lower case joined by hyphens, as a
// page's address names an article about SK hynix
// (cnbc.com/…/sk-hynix-third-quarter-profit.html), which went to the model
// as a key. A key's characters are random: it has a capital, or a run of
// 16 or more with no hyphen (DeepSeek's sk- and 32 hex digits, OpenRouter's
// sk-or-v1- and 64), or hex digits alone, as a UUID's groups (sk- and a
// UUID in lower case) are, which no word but a few like "face" is.
func notASlug(v string) bool {
if strings.ContainsFunc(v, func(r rune) bool { return r >= 'A' && r <= 'Z' }) {
return true
}
if !strings.ContainsFunc(strings.TrimPrefix(v, "sk-"), func(r rune) bool {
return r != '-' && r != '_' && (r < '0' || r > '9') && (r < 'a' || r > 'f')
}) {
return true
}
for _, w := range strings.FieldsFunc(v, func(r rune) bool { return r == '-' || r == '_' }) {
if len(w) >= 16 {
return true
}
}
return false
}
func realEmail(v string) bool {
at := strings.LastIndexByte(v, '@')
if strings.EqualFold(v[:at], "git") {
return false
}
host := strings.ToLower(v[at+1:])
if !mailHost(v[:at], host) {
return false
}
if fileHost(host) || docFiles[host] || !mailTLD(host[strings.LastIndexByte(host, '.')+1:]) {
return false
}
for _, fake := range []string{"example.com", "example.org", "example.net", "localhost", ".test", ".invalid", ".example"} {
if host == strings.TrimPrefix(fake, ".") || strings.HasSuffix(host, fake) {
return false
}
}
return !strings.HasPrefix(host, "noreply") && !strings.Contains(v, "noreply@")
}
// mailHost is whether host, after local and an @, can be a mail domain:
// it has a dot, labels of letters, digits and hyphens (or a name's own
// script's: 例子.中国), and is not a
// linker's table; and local has no @ of its own. A disassembly's symbols
// read as addresses: system@plt, [email protected], a version after one @ or
// two (memcpy@GLIBC_2.14, foo@@GLIBC_2.2.5, foo@Base). .got is a
// top-level domain, and the local model took system@got for an email, so
// a reverse engineer had to turn masking off (ch35tnut_ on X, 10-28).
func mailHost(local, host string) bool {
if strings.HasSuffix(local, "@") || !strings.Contains(host, ".") || strings.Trim(host, ".") != host {
return false
}
linker := true
for _, l := range strings.Split(host, ".") {
if l == "" || strings.ContainsFunc(l, func(r rune) bool { return r < 0x80 && r != '-' && !(r >= 'a' && r <= 'z') && !(r >= '0' && r <= '9') }) {
return false
}
if !linkerWords[l] {
linker = false
}
}
return !linker
}
// linkerWords are the names of a binary's tables a symbol is reached
// through (system@plt, [email protected], [email protected]), no domain's labels.
var linkerWords = map[string]bool{"plt": true, "got": true, "sec": true, "iplt": true, "igot": true}
// docFiles are the files agents read by name, on a domain's ending
// (.md): "@AGENTS.md" after a word, as a prompt names one, is no address
// (ydzh on Discord: [email protected] was masked).
var docFiles = map[string]bool{
"agents.md": true, "claude.md": true, "gemini.md": true, "readme.md": true, "skill.md": true,
"changelog.md": true, "contributing.md": true, "license.md": true, "lessons.md": true, "todo.md": true,
"copilot-instructions.md": true, "crush.md": true, "qwen.md": true, "iflow.md": true, "warp.md": true,
}
// localTLDs are names an organisation's own mail can end in that no
// registry has: an address there is someone's all the same.
var localTLDs = map[string]bool{"local": true, "internal": true, "lan": true, "corp": true, "home": true, "intranet": true, "localdomain": true}
// mailTLD is whether tld, an address's last label, is one mail can be
// sent to: in the public suffix list's ICANN part, or an organisation's
// own. A decorator's or a module's attribute after a name
// ([email protected], ydzh on Discord) ends in a word no registry has.
func mailTLD(tld string) bool {
if localTLDs[tld] {
return true
}
_, icann := publicsuffix.PublicSuffix(tld)
return icann
}
// fileExt are file name extensions that are no top-level domain, so
// what ends in one is a file's name and not an address: an image a Mac
// saved at twice its size, [email protected], went to the model as
// {{EMAIL_…}} and Pi could no longer read the file (#3029).
// Other files are named so too ([email protected], [email protected]), and
// none of these is a domain either (IANA's list, checked 2026-10-21).
var fileExt = map[string]bool{
"jpg": true, "jpeg": true, "png": true, "gif": true, "webp": true, "svg": true, "heic": true, "heif": true,
"avif": true, "bmp": true, "tif": true, "tiff": true, "ico": true, "icns": true, "pdf": true,
"svgz": true, "apng": true, "jxl": true, "psd": true, "eps": true, "lottie": true,
"mp4": true, "m4v": true, "webm": true, "mkv": true, "avi": true, "mp3": true, "m4a": true, "wav": true,
"ogg": true, "flac": true, "aac": true, "ttf": true, "otf": true, "woff": true, "woff2": true, "eot": true,
"json": true, "yaml": true, "yml": true, "toml": true, "xml": true, "txt": true, "csv": true, "tsv": true,
"log": true, "html": true, "htm": true, "css": true, "scss": true, "less": true, "js": true, "mjs": true,
"cjs": true, "jsx": true, "ts": true, "tsx": true, "doc": true, "docx": true, "xls": true, "xlsx": true,
"ppt": true, "pptx": true, "gz": true, "tgz": true, "tar": true, "rar": true, "7z": true, "dmg": true,
}
// scaleExt are file extensions that are top-level domains too (.zip,
// .mov, .md, .map): a name ending in one is a file's only with a scale
// before it, [email protected], [email protected], while [email protected] is an
// address.
var scaleExt = map[string]bool{"zip": true, "mov": true, "md": true, "map": true}
var scaleRe = regexp.MustCompile(`^[0-9]+(?:\.[0-9]+)?x$`)
// fileHost says host, what follows the @ of what looks like an address,
// is the end of a file's name: an extension no domain has (fileExt), or a
// scale and an extension that is a domain too (scaleExt).
func fileHost(host string) bool {
dot := strings.LastIndexByte(host, '.')
ext := host[dot+1:]
return fileExt[ext] || dot > 0 && scaleExt[ext] && scaleRe.MatchString(host[:dot])
}
func chineseID(v string) bool {
w := []int{7, 9, 10, 5, 8, 4, 2, 1, 6, 3, 7, 9, 10, 5, 8, 4, 2}
sum := 0
for i := 0; i < 17; i++ {
sum += int(v[i]-'0') * w[i]
}
return strings.ToUpper(v[17:]) == string("10X98765432"[sum%11])
}
func luhn(v string) bool {
var ds []int
for _, r := range v {
if r >= '0' && r <= '9' {
ds = append(ds, int(r-'0'))
}
}
if len(ds) < 15 || len(ds) > 19 {
return false
}
sum := 0
for i := range ds {
d := ds[len(ds)-1-i]
if i%2 == 1 {
if d *= 2; d > 9 {
d -= 9
}
}
sum += d
}
return sum%10 == 0
}
// ---- placeholders ---------------------------------------------------------------
var (
mu sync.RWMutex
values = map[string]string{} // placeholder → value
kinds = map[string]bool{} // the kinds of values, for wordTail
keyOf []byte
keyPath string // guarded by mu, as keyOf is
)
// maxValues bounds what is held; past it the oldest half is let go, and a
// request that still has them brings them back.
const maxValues = 200_000
func key() []byte {
mu.RLock()
k := keyOf
mu.RUnlock()
if k != nil {
return k
}
mu.Lock()
defer mu.Unlock()
if keyOf != nil {
return keyOf
}
keyOf = loadKey()
return keyOf
}
// SetKeyPath chooses where the placeholder key is kept across restarts;
// "" keeps it in memory only. The gateway's owner sets it before requests
// arrive. Once the key is loaded, both it and its path stay fixed.
func SetKeyPath(path string) {
mu.Lock()
defer mu.Unlock()
if keyOf == nil {
keyPath = path
}
}
// loadKey is called with mu held.
func loadKey() []byte {
if keyPath != "" {
if b, err := os.ReadFile(keyPath); err == nil && len(b) >= 32 {
return b[:32]
}
}
b := make([]byte, 32)
_, _ = rand.Read(b)
if keyPath != "" {
_ = os.MkdirAll(filepath.Dir(keyPath), 0o700)
_ = os.WriteFile(keyPath, b, 0o600)
}
return b
}
var b32 = base32.StdEncoding.WithPadding(base32.NoPadding)
// tag is the placeholder v of kind goes as, made and not remembered: a
// value sent as written (Warn) is named by it in the record of what was
// found, and a preview names each value by it.
func tag(kind, v string) string {
h := hmac.New(sha256.New, key())
h.Write([]byte(kind + "\x00" + v))
return "{{" + kind + "_" + strings.ToLower(b32.EncodeToString(h.Sum(nil))[:8]) + "}}"
}
func placeholder(kind, v string) string {
p := tag(kind, v)
mu.Lock()
if _, ok := values[p]; !ok {
makeRoom()
values[p] = v
kinds[kind] = true
addKnown(p, v)
}
mu.Unlock()
return p
}
// makeRoom lets the oldest half of values go when they are full; called
// with mu held.
func makeRoom() {
if len(values) < maxValues {
return
}
n := 0
for k := range values {
delete(values, k)
delete(shapes, k)
if n++; n >= maxValues/2 {
break
}
}
reindexKnown()
}
// put is how o sends a value: as its placeholder, or as a made-up key of
// its form (Shape).
func (o Options) put() func(kind, v string) string {
if o.Shape {
return standIn(shaped)
}
return standIn(placeholder)
}
// ---- masking --------------------------------------------------------------------
type span struct {
start, end int
kind string
by string // which finder found it: one of the By constants
score float64 // the model's score for one it found, else 0
// bare: a whole bare value of JSON text (inStrings), whose
// placeholder goes in quoted
bare bool
}
// Which finder found a value (Hit.By): magpie's rules, the user's own
// rules, the user's words, the local model (with the Chinese names it
// brings, which run only with it), or a value masked before (known.go).
const (
ByRule = "rule"
ByYours = "yours"
ByWord = "word"
ByModel = "model"
ByKnown = "known"
)
// What is done with a value of a kind (Options.Actions) other than
// masking it.
const (
Warn = "warn" // sent as written, and recorded
Block = "block" // masked, and the request held back
)
// Hit is one value found: its kind, the finder that found it, the
// placeholder it went as (or would have, for one sent as written), what
// was done with it ("" masked, Warn, Block) and its Hint. It never holds
// the value itself.
type Hit struct {
Kind, By, Placeholder, Action string
Head string
Len int
}
// Value is the value behind placeholder p, while this run of magpie
// remembers it.
func Value(p string) (string, bool) {
mu.RLock()
defer mu.RUnlock()
v, ok := values[p]
return v, ok
}
// Hint is as much of v as may be kept where v itself may not: its first
// character (the first few of a key's, which name its kind: sk-, ghp_)
// and how many characters it has.
func Hint(kind, v string) (head string, n int) {
n = utf8.RuneCountInString(v)
if n < 2 {
return "", n
}
k := 1
if _, personal := kindCats[kind]; !personal && kind != "TERM" && n >= 16 {
// a key's prefix, up to where its random part starts
k = 0
for _, c := range v {
if k >= 4 || !strings.ContainsRune("abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ-_", c) {
break
}
k++
if c == '-' || c == '_' {
break
}
}
k = max(k, 1)
}
r := []rune(v)
return string(r[:k]), n
}
// anyCases are the patterns that find a user's word in any case, by word.
var anyCases sync.Map
// anyCase finds w as written but in any case (Unicode's simple folding, so
// a word whose case changes its length in bytes is still found whole).
func anyCase(w string) *regexp.Regexp {
if re, ok := anyCases.Load(w); ok {
return re.(*regexp.Regexp)
}
re := regexp.MustCompile(`(?i)` + regexp.QuoteMeta(w))
anyCases.Store(w, re)
return re
}
// wordRune says r is a letter, a digit or _ of a script that puts spaces
// between words. Chinese and Japanese don't, so a word of theirs is found
// inside their text, and a Latin word right beside a Chinese letter
// ("用ABC做") is still whole.
func wordRune(r rune) bool {
if r == '_' {
return true
}
if !unicode.IsLetter(r) && !unicode.IsDigit(r) {
return false
}
return !unicode.In(r, unicode.Han, unicode.Hiragana, unicode.Katakana)
}
// wholeWord says s[a:b], a word the user asked to mask, is a word of its
// own in s: where its first or last letter is a wordRune, the letter
// beside it in s isn't one. A number is one with its decimals, so a digit
// at its edge also has no "." and digit beside it: 123456 is not whole in
// 0.123456 or 123456.78, while "v1." ends a sentence.
func wholeWord(s string, a, b int) bool {
first, _ := utf8.DecodeRuneInString(s[a:b])
last, _ := utf8.DecodeLastRuneInString(s[a:b])
if wordRune(first) && a > 0 {
r, n := utf8.DecodeLastRuneInString(s[:a])
if wordRune(r) {
return false
}
if unicode.IsDigit(first) && r == '.' {
if p, _ := utf8.DecodeLastRuneInString(s[:a-n]); unicode.IsDigit(p) {
return false
}
}
}
if wordRune(last) && b < len(s) {
r, n := utf8.DecodeRuneInString(s[b:])
if wordRune(r) {
return false
}
if unicode.IsDigit(last) && r == '.' {
if f, _ := utf8.DecodeRuneInString(s[b+n:]); unicode.IsDigit(f) {
return false
}
}
}
return true
}
// Mask swaps what o covers in s for placeholders, and says how many.
func Mask(s string, o Options) (string, int) {
o.sent = true
return mask(s, withModel(o, []string{s}), o.put(), false)
}
// Preview is Mask with nothing remembered: each value goes as the
// placeholder, or with Options.Shape the made-up key, it would have (a
// user name its made-up name, an IP address its documentation one), which
// isn't kept for Restore or as a value seen before (known.go).
func Preview(s string, o Options) string {
put := tag
if o.Shape {
put = func(kind, v string) string {
name, k := splitName(v)
if f, ok := shapeOf(kind, k); ok {
return name + f
}
return tag(kind, v)
}
}
tagOrShape := put
put = func(kind, v string) string {
if kind == "USERNAME" {
return standInOf(v) // a user name's made-up name (shape.go)
}
if kind == "IP" {
if at, _, ok := ipStandIns(v); ok {
return at(0) // an address's documentation one (shape.go)
}
}
return tagOrShape(kind, v)
}
out, _ := mask(s, withModel(o, []string{s}), put, false)
return out
}
// mask swaps what o covers in s for what put makes of each value.
//
// escaped says s holds JSON text of its own (asJSON: a tool call's
// arguments), which the vendor reads after one more unescaping. There the
// finders read s as the vendor does (unescapedView): the \n before
// @mcp.tool is a newline, not an n that starts an address, and the \n
// before [email protected] is no letter of it either. No value starts or ends
// inside an escape (outsideEscapes), and what is kept to put back is the
// value as it reads unescaped, so Restore, which escapes it again in
// arguments, gives the agent what it wrote. Read as plain text, the n of
// a \n went as the first letter of [email protected], the backslash was left
// before the placeholder (\{{EMAIL_…}}), the arguments were JSON no
// longer, and Ollama Cloud turned away every turn that replayed the call
// with 400 "invalid tool call arguments" (#3022).
func mask(raw string, o Options, put func(kind, v string) string, escaped bool) (string, int) {
if len(raw) < 3 {
return raw, 0
}
s := raw // what the finders read: as long as raw, a place in one is the same in the other
if escaped {
s = unescapedView(raw)
put = unescapedPut(put)
}
var found []span
for _, w := range o.Words {
if w = strings.TrimSpace(w); len(w) < 2 {
continue
}
// a date is matched as that date, however it is written (dateword.go)
if spans, ok := dateSpans(s, w); ok {
found = append(found, spans...)
continue
}
// next finds w at or after i; one inside a longer word, where the
// user asked for whole words, is passed over a letter at a time, so
// "XABC ABC" still finds the second
next := func(i int) (int, int) {
if o.WordsAnyCase {
if m := anyCase(w).FindStringIndex(s[i:]); m != nil {
return i + m[0], i + m[1]
}
return -1, -1
}
if j := strings.Index(s[i:], w); j >= 0 {
return i + j, i + j + len(w)
}
return -1, -1
}
for i := 0; i <= len(s); {
a, b := next(i)
if a < 0 {
break
}
if o.WordsWhole && !wholeWord(s, a, b) {
_, n := utf8.DecodeRuneInString(s[a:])
i = a + max(n, 1)
continue
}
found = append(found, span{a, b, "TERM", ByWord, 0, false})
i = max(b, a+1)
}
}
all := rules
if o.Secrets && len(o.Rules) > 0 {
// the user's first: where one of theirs and one of magpie's find
// the same value, it goes by the name they gave it
all = append(customRules(o.Rules), rules...)
}
lower := ""
// split are matches a rule took apart into values of their own
// (rkato@box17 as a user and a host): what another rule found from
// the same place in them, an email, goes
var split []span
for _, r := range all {
if !o.covers(r.cat) {
continue
}
if len(r.markers) > 0 && !containsAny(s, r.markers) {
continue
}
if len(r.fold) > 0 {
if lower == "" {
lower = lowerASCII(s)
}
if !containsAny(lower, r.fold) {
continue
}
}
for _, m := range r.matches(s, lower) {
if len(r.groups) > 0 {
took := false
for g, kind := range r.groups {
if a, b := m[2*g+2], m[2*g+3]; a >= 0 && r.takes(s, a, b, r.groupOK[g]) {
found = append(found, span{a, b, kind, r.by(), 0, false})
took = true
}
}
if took {
split = append(split, span{m[2], m[1], "", "", 0, false})
}
continue
}
a, b := m[0], m[1]
for g := 1; 2*g+1 < len(m); g++ {
if m[2*g] >= 0 {
a, b = m[2*g], m[2*g+1]
break
}
if !r.anyGroup {
break
}
}
ok := r.ok
if r.fix != nil {
var good bool
if a, b, good = r.fix(s, a, b); !good {
continue
}
ok = nil
}
if r.takes(s, a, b, ok) {
found = append(found, span{a, b, r.kind, r.by(), 0, false})
}
}
}
for _, sp := range split {
found = slices.DeleteFunc(found, func(f span) bool {
return f.kind == "EMAIL" && f.start == sp.start && f.end <= sp.end
})
}
// after the rules', so where a rule finds the same value it keeps the
// kind it has always had
if o.sent {
// the model was asked about raw
now := modelSpans(raw, o)
kept := sentSpans(sentModel, raw, o, func() []span { return now })
learnLate(raw, now, kept, o, escaped)
found = append(found, kept...)
found = append(found, sentSpans(sentKnown, s, o, func() []span { return knownSpans(s, o) })...)
} else {
found = append(found, modelSpans(raw, o)...) // the model was asked about raw
found = append(found, knownSpans(s, o)...)
}
if escaped {
found = inStrings(raw, outsideEscapes(raw, found))
}
found = noNumbers(s, noIDs(s, found))
if o.sent {
noteScores(raw, found) // what is masked for a vendor, not a preview
}
return apply(raw, found, put, o)
}
// learnLate makes known what the model finds in raw now that it didn't
// the first time raw was sent on o.Route (now, beside kept): raw goes as
// it went, but a value in it the model has read since is masked in new
// text, as it would be had raw been masked with it (known.go). Nothing
// is told (o.Seen): it isn't masked here.
func learnLate(raw string, now, kept []span, o Options, escaped bool) {
var late []span
for _, f := range now {
if !slices.ContainsFunc(kept, func(k span) bool { return k.start == f.start && k.end == f.end && k.kind == f.kind }) {
late = append(late, f)
}
}
if len(late) == 0 {
return
}
if escaped {
late = outsideEscapes(raw, late)
}
noteScores(raw, late)
o.Seen = nil
put := o.put()
if escaped {
put = unescapedPut(put)
}
apply(raw, late, put, o)
}
// unescapedView is s, JSON text, as long as s and read as the vendor reads
// it once unescaped, as far as the finders tell: an escape that stands for
// a newline, a tab or another control character has its letter read as
// one (\n as \ and a newline), and a \u escape's u and digits as line
// breaks, so neither starts or ends a word. A backslash, a quote and a
// slash escaped stay as written: the rules read \" as a quote and C:\\Users
// as a Windows path.
func unescapedView(s string) string {
if strings.IndexByte(s, '\\') < 0 {
return s
}
b := []byte(s)
for i := 0; i+1 < len(b); i++ {
if b[i] != '\\' {
continue
}
i++
switch b[i] {
case 'n', 'b', 'f':
b[i] = '\n'
case 'r':
b[i] = '\r'
case 't':
b[i] = '\t'
case 'u':
for j := i; j < i+5 && j < len(b); j++ {
b[j] = '\n'
}
i += 4
}
}
return string(b)
}
// unescapedPut is put given each value unescaped, as the vendor and the
// agent read it: a private key's \n are newlines. A value that isn't a
// JSON string's text goes as it is.
func unescapedPut(put func(kind, v string) string) func(kind, v string) string {
return func(kind, v string) string {
if strings.IndexByte(v, '\\') >= 0 {
var u string
if json.Unmarshal([]byte(`"`+v+`"`), &u) == nil {
v = u
}
}
return put(kind, v)
}
}
// outsideEscapes is found with no value starting or ending inside one of
// the escapes of s, JSON text: after an escape's backslash, as the n of
// \n, or among a \u00e9's digits. Masked there, it would leave a
// backslash before the placeholder or cut one off from its letter, and s
// would be JSON no longer (#3022). A value is cut back to the escapes'
// edges instead, so what it holds is still masked: the [email protected] of a
// model's "[email protected]" read in \[email protected], a user's rule that
// reads up to the backslash of \". One left with nothing goes. Every
// finder's values go through here: the rules', the user's words, the
// model's and those known before.
func outsideEscapes(s string, found []span) []span {
if len(found) == 0 || strings.IndexByte(s, '\\') < 0 {
return found
}
// esc[i], for i inside an escape past its backslash: where the escape
// starts and ends, plus one each so a zero is outside one
type edges struct{ from, to int }
esc := make([]edges, len(s)+1)
for i := 0; i < len(s); i++ {
if s[i] != '\\' {
continue
}
n := 2
if i+1 < len(s) && s[i+1] == 'u' {
n = 6
}
end := min(i+n, len(s))
for j := i + 1; j < end; j++ {
esc[j] = edges{i + 1, end + 1}
}
i = end - 1
}
out := found[:0]
for _, f := range found {
if e := esc[f.start]; e.to > 0 {
f.start = e.to - 1
}
if e := esc[f.end]; e.from > 0 {
f.end = e.from - 1
}
if f.start < f.end {
out = append(out, f)
}
}
return out
}
// inStrings is found, for s JSON text (a tool call's arguments), with
// each value inside the one string it starts in: cut at that string's
// closing quote, or started at the first string inside it. A value that
// is a whole bare value (a number under "password":) is kept, and its
// placeholder goes in quoted (span.bare); one in nothing else, across a
// key's quotes, a colon or a brace, goes. Masked over the quote, the
// arguments were JSON no longer: the local model reads a password's ! and
// the "} after it as one token, and {"cmd":"export DB_PASSWORD=Hunt3r!"}
// went as {"cmd":"export DB_PASSWORD={{SECRET_…}}}; a user's rule
// tok-\S+ took the "} too. The vendor turned away every turn that
// replayed the call, and Codex reconnected over and over
// (xiaojiang091499 on X). Every finder's values go through here, as
// through outsideEscapes. Text that isn't JSON is left as it is.
func inStrings(s string, found []span) []span {
if len(found) == 0 || !json.Valid([]byte(s)) {
return found
}
// in[i] is 1 + which string literal byte i is the text of, 0 for one
// outside every string (its quotes among them)
in := make([]int32, len(s))
n := int32(0)
for i := 0; i < len(s); i++ {
if s[i] != '"' {
continue
}
n++
j := i + 1
for ; j < len(s) && s[j] != '"'; j++ {
in[j] = n
if s[j] == '\\' && j+1 < len(s) {
j++
in[j] = n
}
}
i = j
}
out := found[:0]
for _, f := range found {
a := f.start
for a < f.end && in[a] == 0 {
a++
}
if a == f.end {
if bareValue(s, f.start, f.end) {
f.bare = true
out = append(out, f)
}
continue
}
b := a
for b < f.end && in[b] == in[a] {
b++
}
f.start, f.end = a, b
out = append(out, f)
}
return out
}
// bareValue says s[a:b], JSON text outside its strings, is all of one
// bare value: an object's (after :) or an array's (after [ or ,), with
// the , } or ] that ends it after it.
func bareValue(s string, a, b int) bool {
if a >= b || strings.ContainsAny(s[a:b], " \t\r\n{}[]\",:") {
return false
}
i := a - 1
for i >= 0 && isSpace(s[i]) {
i--
}
k := b
for k < len(s) && isSpace(s[k]) {
k++
}
return i >= 0 && strings.IndexByte(":[,", s[i]) >= 0 && (k == len(s) || strings.IndexByte(",}]", s[k]) >= 0)
}
// noNumbers is found without what is a piece of a decimal number of s:
// digits right after the point of 5.13800138000, or right before the
// point of 13800138000.5. A number is judged whole, and the fraction of
// a height, a price or a coordinate is not a phone or a card number
// because its digits happen to check (willz on Discord). Nor is a number
// that stands bare as a JSON value: "timestamp": 17929036837 in a session
// file is a count or a time, and a phone, a card or an ID number in JSON
// is a string (id.null on Discord). Only personal data's kinds, whoever
// found them (a rule, the model, a value known before): a secret is
// masked wherever it is, and what the user chose (their words, their
// rules) stays theirs.
func noNumbers(s string, found []span) []span {
return slices.DeleteFunc(found, func(f span) bool {
_, personal := kindCats[f.kind]
return personal && f.by != ByWord && f.by != ByYours && (inDecimal(s, f.start, f.end) || jsonNumber(s, f.start, f.end))
})
}
// jsonNumber says s[a:b] is all of a bare JSON number, by where it sits:
// digits only, an object's value ("key": …) or an element of an array of
// numbers ([…, …]), and nothing after it but the , } or ] that ends a
// value. "phone": "13800138000" is a string, and phone:13800138000 and
// "call 13800138000, thanks" are text, so each is still masked.
func jsonNumber(s string, a, b int) bool {
if a >= b || strings.Trim(s[a:b], digits) != "" {
return false
}
space := func(c byte) bool { return c == ' ' || c == '\t' || c == '\n' || c == '\r' }
k := b
for k < len(s) && space(s[k]) {
k++
}
if k < len(s) && strings.IndexByte(",}]", s[k]) < 0 {
return false
}
i := a - 1
for i >= 0 && space(s[i]) {
i--
}
if i < 0 {
return false
}
switch s[i] {
case ':':
for i--; i >= 0 && space(s[i]); i-- {
}
return i >= 0 && s[i] == '"'
case '[', ',':
// back over the numbers before it to the array's [
for ; i >= 0; i-- {
if s[i] == '[' {
return true
}
if strings.IndexByte(digits+",.-+eE", s[i]) < 0 && !space(s[i]) {
return false
}
}
}
return false
}
// inDecimal says s[a:b] starts with a digit that is a decimal's fraction
// (a digit and a point before it) or ends with a digit that is its whole
// part (a point and a digit after it).
func inDecimal(s string, a, b int) bool {
isDigit := func(i int) bool { return i >= 0 && i < len(s) && s[i] >= '0' && s[i] <= '9' }
return isDigit(a) && a >= 2 && s[a-1] == '.' && isDigit(a-2) ||
isDigit(b-1) && b+1 < len(s) && s[b] == '.' && isDigit(b+1)
}
// by is the finder r is, for Hit.By.
func (r rule) by() string {
if r.yours {
return ByYours
}
return ByRule
}
// matches are r's matches in s, lower being lowerASCII(s), as
// FindAllStringSubmatchIndex has them: in all of s, or, for a rule with
// ahead set, only around its fold markers.
func (r rule) matches(s, lower string) [][]int {
if r.ahead == 0 {
return r.re.FindAllStringSubmatchIndex(s, -1)
}
var wins [][2]int
for _, f := range r.fold {
for i := 0; ; {
j := strings.Index(lower[i:], f)
if j < 0 {
break
}
at := i + j
i = at + len(f)
// from the line's start, or from where a word starts no further
// back than back: what the pattern takes for the text's start is
// where a line or a word starts in s too. The line's start is
// looked for no further back than that: a text of one long
// line, as a request body is, isn't read back to its start for
// each marker.
start := max(0, at-r.back)
if nl := strings.LastIndexByte(s[start:at], '\n'); nl >= 0 {
start += nl + 1
} else if start > 0 {
for start < at && (strings.IndexByte(alnum+"_.-", s[start-1]) >= 0 || s[start]&0xc0 == 0x80) {
start++
}
if start == at && strings.IndexByte(alnum+"_", s[at-1]) >= 0 {
continue // inside a longer word
}
}
wins = append(wins, [2]int{start, min(len(s), i+r.ahead)})
}
}
sort.Slice(wins, func(i, j int) bool { return wins[i][0] < wins[j][0] })
var out [][]int
for k := 0; k < len(wins); k++ {
w := wins[k]
for k+1 < len(wins) && wins[k+1][0] <= w[1] {
k++
w[1] = max(w[1], wins[k][1])
}
for _, m := range r.re.FindAllStringSubmatchIndex(s[w[0]:w[1]], -1) {
for x := range m {
if m[x] >= 0 {
m[x] += w[0]
}
}
out = append(out, m)
}
}
return out
}
// lowerASCII is s with its ASCII letters in lower case, as long as s, so a
// place in one is the same place in the other.
func lowerASCII(s string) string {
b := []byte(s)
for i, c := range b {
if 'A' <= c && c <= 'Z' {
b[i] = c + 'a' - 'A'
}
}
return string(b)
}
// takes says s[a:b] is a value r masks: nothing it may not touch beside
// it, and its checks pass.
func (r rule) takes(s string, a, b int, ok func(string) bool) bool {
if b <= a {
return false
}
if r.bound != "" && (a > 0 && strings.IndexByte(r.bound, s[a-1]) >= 0 || b < len(s) && strings.IndexByte(r.bound, s[b]) >= 0) {
return false
}
if ok != nil && !ok(s[a:b]) {
return false
}
return r.at == nil || r.at(s, a, b)
}
// apply swaps each of found in s for what put makes of its value, the
// first and longest where they overlap, nothing inside a placeholder. A
// value o allows is left as it is, and one of a kind o warns of is left
// and told. o.Seen, where set, is told of each.
func apply(s string, found []span, put func(kind, v string) string, o Options) (string, int) {
if len(found) == 0 {
return s, 0
}
// nothing inside a placeholder already there, or a made-up key
found = outsideShapes(s, found)
if strings.Contains(s, "{{") {
for _, p := range placeholderRe.FindAllStringIndex(s, -1) {
found = slices.DeleteFunc(found, func(f span) bool { return f.start < p[1] && f.end > p[0] })
}
if len(found) == 0 {
return s, 0
}
}
// the first to start, and of those the longest, wins where they overlap;
// of two the same, the rule found first, so a value's placeholder is the
// same every time (an API key in API_KEY=… is an API_KEY, not a SECRET)
sort.SliceStable(found, func(i, j int) bool {
if found[i].start != found[j].start {
return found[i].start < found[j].start
}
return found[i].end > found[j].end
})
var b strings.Builder
last, next, n := 0, 0, 0 // next: where the next value may start
for _, f := range found {
if f.start < next {
continue
}
next = f.end
v := s[f.start:f.end]
if o.allows(v) {
continue
}
act := o.Actions[f.kind]
if act != Warn && act != Block {
act = ""
}
h := Hit{Kind: f.kind, By: f.by, Action: act}
h.Head, h.Len = Hint(f.kind, v)
if act == Warn {
if o.Seen != nil {
h.Placeholder = tag(f.kind, v)
o.Seen(h)
}
continue
}
b.WriteString(s[last:f.start])
h.Placeholder = put(f.kind, v)
if f.bare {
b.WriteString(`"` + h.Placeholder + `"`)
} else {
b.WriteString(h.Placeholder)
}
if o.Seen != nil {
o.Seen(h)
}
last, n = f.end, n+1
}
b.WriteString(s[last:])
return b.String(), n
}
// allows says the user lets v go as written (Options.Allow).
func (o Options) allows(v string) bool {
for _, a := range o.Allow {
if strings.EqualFold(strings.TrimSpace(a), v) {
return true
}
}
return false
}
func containsAny(s string, subs []string) bool {
for _, x := range subs {
if strings.Contains(s, x) {
return true
}
}
return false
}
// ---- restoring ------------------------------------------------------------------
// placeholderBody is a placeholder without its braces, as a model may write
// one back: API_KEY_k3v9x2mq, phone_b5puzbfv, PHONE_B5PUZBFV.
var placeholderBody = regexp.MustCompile(`[A-Za-z][A-Za-z0-9_]*_[A-Za-z2-7]{8}`)
// Restore puts the values back in s. escaped is for text that is itself
// JSON (a tool call's arguments), where a value goes in escaped.
//
// A model doesn't always write a placeholder back as it was given: it
// writes {{ PHONE_b5puzbfv }} as a template has it, drops a brace, or the
// braces, or changes the case. Any of those is put back too when what is
// left names a placeholder this magpie made: its kind and the 8 of its
// hash, which no other text has by chance (shensmart on Discord).
func Restore(s string, escaped bool) string { return restore(s, escaped, nil) }
// restore is Restore, counting in put, when it isn't nil, each
// placeholder it put a value back for.
func restore(s string, escaped bool, put map[string]int) string {
mu.RLock()
defer mu.RUnlock()
if len(values) == 0 {
return s
}
s = restoreShapes(s, put)
if !strings.Contains(s, "_") {
return s
}
var b strings.Builder
last := 0
for _, m := range placeholderBody.FindAllStringIndex(s, -1) {
a, z := m[0], m[1]
if a < last || a > 0 && isWordByte(s[a-1]) || z < len(s) && isWordByte(s[z]) {
continue
}
p := canonical(s[a:z])
v, ok := values[p]
if !ok {
continue
}
a, z = braces(s, a, z)
if a < last {
continue
}
if put != nil {
put[p]++
}
b.WriteString(s[last:a])
if escaped {
v = jsonEscape(v)
}
b.WriteString(v)
last = z
}
if last == 0 {
return s
}
b.WriteString(s[last:])
return b.String()
}
// canonical is the placeholder Mask wrote for body, whatever its case.
func canonical(body string) string {
cut := len(body) - 8
return "{{" + strings.ToUpper(body[:cut]) + strings.ToLower(body[cut:]) + "}}"
}
func isWordByte(c byte) bool {
return c >= 'A' && c <= 'Z' || c >= 'a' && c <= 'z' || c >= '0' && c <= '9' || c == '_'
}
// braces widens s[a:z], a placeholder's body, over the braces the model
// wrote around it, and the spaces inside them: both of {{ and }}, or one
// of them dropped ({{X} or {X}}), or just {{ or just }}. A single brace
// on one side alone is the text's own ({X as in code) and stays.
func braces(s string, a, z int) (int, int) {
la, l := a, 0
for la > 0 && (s[la-1] == ' ' || s[la-1] == '\t') {
la--
}
for l < 2 && la > 0 && s[la-1] == '{' {
la--
l++
}
rz, r := z, 0
for rz < len(s) && (s[rz] == ' ' || s[rz] == '\t') {
rz++
}
for r < 2 && rz < len(s) && s[rz] == '}' {
rz++
r++
}
if l == 2 || l == 1 && r >= 1 {
a = la
}
if r == 2 || r == 1 && l >= 1 {
z = rz
}
return a, z
}
// maxPlaceholder is the longest placeholder Mask writes: {{, a kind (a
// user's rule may name one up to maxKind long), _, the 8 of the hash, }}.
const maxPlaceholder = len("{{") + maxKind + len("_") + 8 + len("}}")
// partialTail is how much of the end of s may be the start of a
// placeholder the next piece of a stream finishes, in any of the ways
// Restore takes one back: {{API_, {{ phone_b5, PHONE_b5puzbfv} — or, with
// no brace before it, a last word that starts as a placeholder this magpie
// made does (PHO, PHONE_b5pu), or is one whose }} may still come.
func partialTail(s string) int {
if n := braceTail(s); n > 0 {
return n
}
return max(wordTail(s), shapeTail(s))
}
// braceTail is a tail from {, or {{, on: then spaces, a placeholder's body
// or the start of one, then spaces and one } at most.
func braceTail(s string) int {
i := strings.LastIndexByte(s, '{')
if i < 0 {
return 0
}
if i > 0 && s[i-1] == '{' {
i--
}
t := s[i:]
if len(t) >= maxPlaceholder+4 || strings.Contains(t, "}}") {
return 0
}
body := strings.TrimLeft(t, "{")
body = strings.TrimLeft(body, " \t")
if body == "" {
return len(t) // {, {{, {{␠
}
if c := body[0]; !(c >= 'A' && c <= 'Z' || c >= 'a' && c <= 'z') {
return 0
}
j := 0
for j < len(body) && isWordByte(body[j]) {
j++
}
rest := strings.TrimLeft(body[j:], " \t")
if rest != "" && rest != "}" {
return 0
}
if j < len(body) && placeholderBody.FindString(body[:j]) != body[:j] {
return 0 // a space or a brace after something that is no placeholder
}
return len(t)
}
// wordTail is the last word of s, where it may be a placeholder's body as
// a model writes one without braces: the start of one of the kinds this
// magpie has made placeholders of, up to the 8 of its hash. A word of
// text that only starts like a kind is held for an event and then goes
// on, unchanged.
func wordTail(s string) int {
i := len(s)
for i > 0 && isWordByte(s[i-1]) {
i--
}
w := s[i:]
if w == "" || len(w) > maxKind+9 || !(w[0] >= 'A' && w[0] <= 'Z') {
return 0
}
u := strings.ToUpper(w)
mu.RLock()
defer mu.RUnlock()
for k := range kinds {
k += "_"
if strings.HasPrefix(k, u) {
return len(w)
}
if strings.HasPrefix(u, k) && len(u)-len(k) <= 8 && strings.Trim(strings.ToLower(u[len(k):]), "abcdefghijklmnopqrstuvwxyz234567") == "" {
return len(w)
}
}
return 0
}