Skip to content

The cross-harness, self-improving memory layer for AI agents. Join our community: https://discord.com/invite/zdNChS2fBu

perf(redaction): hold needle and needle-set ids in a two-word idSet (#1246)

kyle-elliott-asymptote38c8d9ff0f76
diff --git a/pkg/asymptoteobserve/privacy.go b/pkg/asymptoteobserve/privacy.go
index c4f9eeda..fe304ca1 100644
--- a/pkg/asymptoteobserve/privacy.go
+++ b/pkg/asymptoteobserve/privacy.go
@@ -175,7 +175,7 @@ func (rule redactionRule) pattern() *regexp.Regexp {
// text before it; both read the text as it is, so they must not depend on the case of its letters.
// within, when set, is a broader set the text must also hold.
type redactNeedles struct {
- id uint
+ id int // its index in redactEngine.setMasks and the gate's sets
folded bool
needles []string
followedBy func(rest string) bool
@@ -183,14 +183,14 @@ type redactNeedles struct {
within *redactNeedles
}
-var redactNeedleSets uint
+var redactNeedleSets int
func newRedactNeedles(folded bool, needles ...string) *redactNeedles {
if len(needles) == 0 {
panic("asymptoteobserve: a redaction rule needs at least one needle")
}
- if redactNeedleSets >= 64 {
- panic("asymptoteobserve: redactGate tracks at most 64 needle sets")
+ if redactNeedleSets >= idSetCap {
+ panic("asymptoteobserve: more redaction needle sets than an idSet holds; lengthen idSet")
}
set := &redactNeedles{id: redactNeedleSets, folded: folded, needles: needles}
redactNeedleSets++
@@ -495,7 +495,7 @@ var redactionRules = concatRules([]redactionRule{
// Model providers and developer platforms whose keys agents handle all day. Each length and
// alphabet is gitleaks' rule of the same name, or trufflehog's detector where gitleaks has none.
// Where prefixes differ in a letter or two, the rules share one needle (shp, lsv2_, _v1_): the
- // engine tracks at most 64 distinct needles, and each new one spends a slot.
+ // engine tracks at most idSetCap distinct needles, and each new one spends a slot.
shapeRule(`\b(hv[sbr]\.)[A-Za-z0-9_-]{24,}`, "${1}"+redactedValue, "hvs.", "hvb.", "hvr."), // HashiCorp Vault
shapeRule(`\b(gsk_)[A-Za-z0-9]{52}\b`, "${1}"+redactedValue, "gsk_"), // Groq
shapeRule(`\b(pplx-)[A-Za-z0-9]{48}\b`, "${1}"+redactedValue, "pplx-"), // Perplexity
diff --git a/pkg/asymptoteobserve/redact_adjacent.go b/pkg/asymptoteobserve/redact_adjacent.go
index 6a68fd6a..55c38071 100644
--- a/pkg/asymptoteobserve/redact_adjacent.go
+++ b/pkg/asymptoteobserve/redact_adjacent.go
@@ -58,8 +58,8 @@ const (
// redactShape is what the adjacency checks know of one shape rule's pattern.
type redactShape struct {
rule int
- needles uint64 // scanner needle bits
- offset int // the rule's offset: how many bytes before its needle a match starts
+ needles idSet // scanner needle ids
+ offset int // the rule's offset: how many bytes before its needle a match starts
// tail, when set, is the class of the open-ended repeat the pattern ends in, at least tailMin
// long, after at most head bytes. Without it the pattern matches at most bound-1 bytes.
tail *[256]bool
@@ -83,7 +83,7 @@ type redactShape struct {
// start of a word that a replacement made.
type redactFollower struct {
rule int
- needles uint64
+ needles idSet
kind redactFollowerKind
// charset is every byte the pattern can match, for followRun.
charset [256]bool
@@ -176,7 +176,7 @@ func leadPattern(rule redactionRule) *regexp.Regexp {
// shapeOf reads a shape rule's pattern: a concatenation that ends, before any trailing assertion,
// either in an open-ended repeat of one class or in something of bounded length.
-func shapeOf(index int, rule redactionRule, parsed *syntax.Regexp, needles uint64) redactShape {
+func shapeOf(index int, rule redactionRule, parsed *syntax.Regexp, needles idSet) redactShape {
s := redactShape{rule: index, needles: needles, offset: rule.offset, jwt: rule.shape == shapeJWT}
parts := []*syntax.Regexp{parsed}
if parsed.Op == syntax.OpConcat {
@@ -416,9 +416,9 @@ func hitsFrom(hits []redactHit, pos int) int {
}
// hasNeedleAt reports whether one of needles occurs at pos.
-func (c *redactChain) hasNeedleAt(needles uint64, pos int) bool {
+func (c *redactChain) hasNeedleAt(needles *idSet, pos int) bool {
for i := hitsFrom(c.hits, pos); i < len(c.hits) && c.hits[i].start == pos; i++ {
- if needles&(1<<c.hits[i].id) != 0 {
+ if needles.has(c.hits[i].id) {
return true
}
}
@@ -469,7 +469,7 @@ func (c *redactChain) rescue(si, q int) []int {
h := c.hits[i]
for sj := range c.adj.shapes {
x := h.start - c.adj.shapes[sj].offset
- if c.adj.shapes[sj].needles&(1<<h.id) == 0 || x <= q || x >= q+s.bound {
+ if !c.adj.shapes[sj].needles.has(h.id) || x <= q || x >= q+s.bound {
continue
}
if m := c.find(s.rule, q, x); m != nil && m[1] == x {
@@ -494,7 +494,7 @@ func (c *redactChain) rescue(si, q int) []int {
func (c *redactChain) redactedAt(x int) bool {
for si := range c.adj.shapes {
s := &c.adj.shapes[si]
- if !c.hasNeedleAt(s.needles, x+s.offset) {
+ if !c.hasNeedleAt(&s.needles, x+s.offset) {
continue
}
end := min(len(c.t), x+s.prefix+len(redactedValue))
@@ -515,7 +515,7 @@ func (c *redactChain) rescueBefore(si, pos, limit int) []int {
}
for i := hitsFrom(c.hits, pos+s.offset); i < len(c.hits) && c.hits[i].start-s.offset < limit; i++ {
q := c.hits[i].start - s.offset
- if s.needles&(1<<c.hits[i].id) == 0 || !wordBoundary(c.t, q) {
+ if !s.needles.has(c.hits[i].id) || !wordBoundary(c.t, q) {
continue
}
if m := c.rescue(si, q); m != nil {
@@ -536,7 +536,7 @@ func (c *redactChain) secretAt(p int) (int, []int) {
shape, match := -1, []int(nil)
for si := range c.adj.shapes {
s := &c.adj.shapes[si]
- if !c.hasNeedleAt(s.needles, p+s.offset) {
+ if !c.hasNeedleAt(&s.needles, p+s.offset) {
continue
}
if m, open := c.shapeAt(si, p); m != nil {
@@ -804,7 +804,7 @@ func (c *redactChain) crossing(si int, m []int) (int, int, []int) {
if q >= best {
break
}
- if s.needles&(1<<h.id) == 0 || q < lo {
+ if !s.needles.has(h.id) || q < lo {
continue
}
if s.jwt {
@@ -908,7 +908,8 @@ func (e *redactEngine) redactAfterMarkers(t string) (string, bool) {
if p < 0 {
return t, false
}
- hits, _ := e.scanner.scan(t, nil)
+ var found idSet
+ hits := e.scanner.scan(t, nil, &found)
c := newRedactChain(e, t, hits)
var out []byte
last := 0
@@ -1071,7 +1072,7 @@ func (c *redactChain) followerCrossing(m []int) (int, int) {
}
for _, needle := range redactionRules[f.rule].needles.needles {
q := e - len(needle)
- if q <= start || !c.hasNeedleAt(f.needles, q) {
+ if q <= start || !c.hasNeedleAt(&f.needles, q) {
continue
}
if f.kind == followLine {
@@ -1133,7 +1134,7 @@ func (c *redactChain) lineFollowerAt(q int) []int {
}
}
for k, f := range c.adj.followers {
- if f.kind != followLine || !c.hasNeedleAt(f.needles, q) {
+ if f.kind != followLine || !c.hasNeedleAt(&f.needles, q) {
continue
}
// The pattern opens with the needle and \b: where a word byte follows the needle it finds
@@ -1206,7 +1207,7 @@ func (c *redactChain) followerAt(p int) ([]int, redactUse) {
m = c.schemeFollowerAt(f, p)
c.asked[k] = redactAsked{p, m}
default:
- if !c.hasNeedleAt(f.needles, p) {
+ if !c.hasNeedleAt(&f.needles, p) {
continue
}
// The word, the blanks after it, and a run of the charset without blanks. A run
diff --git a/pkg/asymptoteobserve/redact_engine.go b/pkg/asymptoteobserve/redact_engine.go
index 3e581c38..f6dd159e 100644
--- a/pkg/asymptoteobserve/redact_engine.go
+++ b/pkg/asymptoteobserve/redact_engine.go
@@ -167,7 +167,7 @@ var (
// cost every hook run its allocations whether it redacts or not (TestPackageInitStaysCheap).
type redactEngine struct {
scanner needleScanner
- setMasks [64]uint64 // redactNeedles id -> the needles of the set, as scanner needle bits
+ setMasks [idSetCap]idSet // redactNeedles id -> the needles of the set, as scanner needle ids
rules []redactSlot
assigned redactSlot
// adjacent is what redact_adjacent.go knows of the rules, built the first time it is needed.
@@ -194,7 +194,7 @@ var redactEngineFor = sync.OnceValue(func() *redactEngine {
ids[n] = id
needles = append(needles, n)
}
- e.setMasks[set.id] |= 1 << id
+ e.setMasks[set.id].add(id)
}
}
}
@@ -204,11 +204,11 @@ var redactEngineFor = sync.OnceValue(func() *redactEngine {
register(redactAssignedRule().needles)
// Not needles of any rule: they mark text a case-insensitive pattern can match without one.
for _, text := range []string{"\u017f", "\u212a"} {
- e.scanner.unfoldable |= 1 << len(needles)
+ e.scanner.unfoldable.add(len(needles))
needles = append(needles, redactNeedle{text: text})
}
- if len(needles) > 64 {
- panic("asymptoteobserve: redaction tracks at most 64 needles")
+ if len(needles) > idSetCap {
+ panic("asymptoteobserve: more redaction needles than an idSet holds; lengthen idSet")
}
e.scanner.build(needles)
return e
@@ -235,10 +235,10 @@ type redactGate struct {
engine *redactEngine
scanned bool
hits []redactHit
- found uint64 // scanner needle bits
- // known and present are bit sets over redactNeedles ids: which sets were looked for in this
- // version of the text, and which were found.
- known, present uint64
+ found idSet // scanner needle ids
+ // known and present are sets of redactNeedles ids: which sets were looked for in this version
+ // of the text, and which were found.
+ known, present idSet
// hidden is set when a match may hide another from this pass: a value a rule keeps, or the
// capture takes, that is itself a label and its separator (keptLabel), inside a match that
// neither looks into; a copy a token ran on over, as replacing a copy may (copyHidesMatch); or a
@@ -251,7 +251,7 @@ type redactGate struct {
func (g *redactGate) set(value string) {
if value != g.value {
- g.value, g.scanned, g.known, g.present = value, false, 0, 0
+ g.value, g.scanned, g.known, g.present = value, false, idSet{}, idSet{}
if g.copies != nil {
g.copies.lo, g.copies.at = 0, g.copies.at[:0]
}
@@ -267,7 +267,7 @@ func (g *redactGate) e() *redactEngine {
func (g *redactGate) scan() {
if !g.scanned {
- g.hits, g.found = g.e().scanner.scan(g.value, g.hits[:0])
+ g.hits = g.e().scanner.scan(g.value, g.hits[:0], &g.found)
g.scanned = true
}
}
@@ -276,14 +276,14 @@ func (g *redactGate) scan() {
// equates with s and k: such a pattern can match "ſecret", which holds no needle "secret".
func (g *redactGate) unfoldable() bool {
g.scan()
- return g.found&g.e().scanner.unfoldable != 0
+ return g.found.meets(&g.e().scanner.unfoldable)
}
// holdsNothing reports whether the text holds no needle at all, and so no set: no rule can match
// it.
func (g *redactGate) holdsNothing() bool {
g.scan()
- return g.enabled && g.found == 0
+ return g.enabled && g.found.empty()
}
func (g *redactGate) holds(set *redactNeedles) bool {
@@ -293,23 +293,22 @@ func (g *redactGate) holds(set *redactNeedles) bool {
if set.within != nil && !g.holds(set.within) {
return false
}
- bit := uint64(1) << set.id
- if g.known&bit != 0 {
- return g.present&bit != 0
+ if g.known.has(set.id) {
+ return g.present.has(set.id)
}
- g.known |= bit
+ g.known.add(set.id)
if set.folded && g.unfoldable() {
- g.present |= bit
+ g.present.add(set.id)
return true
}
g.scan()
- mask := g.e().setMasks[set.id]
- if g.found&mask == 0 {
+ mask := &g.e().setMasks[set.id]
+ if !g.found.meets(mask) {
return false
}
for _, h := range g.hits {
- if mask&(1<<h.id) != 0 && set.passes(g.e(), g.value, h, false) {
- g.present |= bit
+ if mask.has(h.id) && set.passes(g.e(), g.value, h, false) {
+ g.present.add(set.id)
return true
}
}
@@ -327,7 +326,7 @@ func (g *redactGate) holds(set *redactNeedles) bool {
func (set *redactNeedles) passes(engine *redactEngine, t string, h redactHit, within bool) bool {
masks := &engine.setMasks
for s := set; s != nil; s = s.within {
- if masks[s.id]&(1<<h.id) != 0 && (s.followedBy != nil && !s.followedBy(t[h.end:]) || s.precededBy != nil && !s.precededBy(t[:h.start])) {
+ if masks[s.id].has(h.id) && (s.followedBy != nil && !s.followedBy(t[h.end:]) || s.precededBy != nil && !s.precededBy(t[:h.start])) {
return false
}
if !within {
@@ -428,8 +427,8 @@ func (rule redactionRule) matches(text *redactGate, slot *redactSlot) iter.Seq2[
}
text.scan()
engine := text.e()
- mask := engine.setMasks[rule.needles.id]
- if text.found&mask == 0 {
+ mask := &engine.setMasks[rule.needles.id]
+ if !text.found.meets(mask) {
return
}
if rule.start == startLine {
@@ -443,7 +442,7 @@ func (rule redactionRule) matches(text *redactGate, slot *redactSlot) iter.Seq2[
first, handed, last := -1, 0, -1
budget := redactAnchoredPerByte*len(t) + redactAnchoredBase
for _, h := range text.hits {
- if mask&(1<<h.id) == 0 || !rule.needles.passes(engine, t, h, true) {
+ if !mask.has(h.id) || !rule.needles.passes(engine, t, h, true) {
continue
}
q := rule.startFor(t, h, 0, slot.boundary)
@@ -483,7 +482,7 @@ func (rule redactionRule) matches(text *redactGate, slot *redactSlot) iter.Seq2[
}
pos, failed := 0, -1
for _, h := range text.hits {
- if mask&(1<<h.id) == 0 || h.start < pos || !rule.needles.passes(engine, t, h, true) {
+ if !mask.has(h.id) || h.start < pos || !rule.needles.passes(engine, t, h, true) {
continue
}
q := rule.startFor(t, h, pos, slot.boundary)
@@ -583,10 +582,10 @@ func (rule redactionRule) opensParam(t string, q int) bool {
// lineStart returns the start of a line at or before every place a startLine match holding one of
// the needles can begin: before the needle, an optional `set-`, a quote, whitespace across lines,
// and a few bytes (`=`, `--header`) more.
-func lineStart(engine *redactEngine, t string, hits []redactHit, mask uint64, set *redactNeedles) (int, bool) {
+func lineStart(engine *redactEngine, t string, hits []redactHit, mask *idSet, set *redactNeedles) (int, bool) {
from, ok := len(t), false
for _, h := range hits {
- if mask&(1<<h.id) == 0 || !set.passes(engine, t, h, true) {
+ if !mask.has(h.id) || !set.passes(engine, t, h, true) {
continue
}
b := h.start
@@ -638,6 +637,58 @@ var labelClass, schemeClass = func() (label, scheme [256]bool) {
return label, scheme
}()
+// ---- id sets ----
+
+// idSet is a set of small ids: the scanner's needle ids, or redactNeedles ids. What the gate asks
+// of one (does the text hold one of this set's needles, is this hit's needle one of them) is a few
+// ANDs, with nothing allocated. Its methods take pointers: the sets sit in arrays and structs, and
+// Go copies an array of more than one word for a value receiver and passes it in memory.
+//
+// It holds idSetCap ids. To hold more, lengthen the array, to a power of two: idBit masks an id's
+// word index with len-1, which the compiler can prove in range, so has and add carry no bounds
+// check. The engine panics at build time before an id reaches idSetCap.
+type idSet [2]uint64
+
+// idSetCap is how many ids an idSet holds.
+const idSetCap = len(idSet{}) * 64
+
+// An idSet's length must be a power of two (idBit); this constant overflows, and the package
+// fails to compile, when it is not.
+const _ = -uint(len(idSet{}) & (len(idSet{}) - 1))
+
+// idBit returns the index of id's word in an idSet and its bit in that word.
+func idBit(id int) (int, uint64) {
+ return int(uint(id)>>6) & (len(idSet{}) - 1), 1 << (uint(id) & 63)
+}
+
+func (s *idSet) add(id int) {
+ w, bit := idBit(id)
+ s[w] |= bit
+}
+
+func (s *idSet) has(id int) bool {
+ w, bit := idBit(id)
+ return s[w]&bit != 0
+}
+
+func (s *idSet) empty() bool { return *s == idSet{} }
+
+// meets reports whether s and o hold an id in common.
+func (s *idSet) meets(o *idSet) bool {
+ var common uint64
+ for i := range s {
+ common |= s[i] & o[i]
+ }
+ return common != 0
+}
+
+// union adds every id of o to s.
+func (s *idSet) union(o *idSet) {
+ for i := range s {
+ s[i] |= o[i]
+ }
+}
+
// ---- the needle scanner ----
// redactNeedle is one literal some rule's pattern cannot match without; folded needles are lower
@@ -662,11 +713,11 @@ type needleScanner struct {
needles []redactNeedle
class [256]uint8 // byte -> input class; 0 for bytes no needle holds
delta []uint32 // state + class -> next state
- out []uint64 // (state-firstOut)/nc -> the needles ending there
+ out []idSet // (state-firstOut)/nc -> the needles ending there
nc int // number of classes, the length of a row
firstOut int
- longest int // the longest needle's length
- unfoldable uint64 // the needles that are U+017F and U+212A
+ longest int // the longest needle's length
+ unfoldable idSet // the needles that are U+017F and U+212A
}
func asciiLower(c byte) byte {
@@ -695,19 +746,19 @@ func (sc *needleScanner) build(needles []redactNeedle) {
}
// The trie.
next := [][]int{make([]int, n)}
- out := []uint64{0}
+ out := []idSet{{}}
for id, nd := range needles {
s := 0
for i := range len(nd.text) {
c := sc.class[nd.text[i]]
if next[s][c] == 0 {
next = append(next, make([]int, n))
- out = append(out, 0)
+ out = append(out, idSet{})
next[s][c] = len(next) - 1
}
s = next[s][c]
}
- out[s] |= 1 << id
+ out[s].add(id)
}
// Failure links, breadth first, folded into a complete transition table. Class 0 never
// continues a needle, so it always returns to the root.
@@ -723,7 +774,7 @@ func (sc *needleScanner) build(needles []redactNeedle) {
for len(queue) > 0 {
s := queue[0]
queue = queue[1:]
- out[s] |= out[fail[s]]
+ out[s].union(&out[fail[s]])
for c := 1; c < n; c++ {
if t := next[s][c]; t != 0 {
fail[t] = delta[fail[s]*n+c]
@@ -737,13 +788,13 @@ func (sc *needleScanner) build(needles []redactNeedle) {
// Renumber: the root first, then the states no needle ends in, then those some do.
order := make([]int, 0, len(next))
for s := range next {
- if out[s] == 0 {
+ if out[s].empty() {
order = append(order, s)
}
}
sc.firstOut = len(order) * n
for s := range next {
- if out[s] != 0 {
+ if !out[s].empty() {
order = append(order, s)
sc.out = append(sc.out, out[s])
}
@@ -764,10 +815,11 @@ func (sc *needleScanner) build(needles []redactNeedle) {
// dependent loads; independent ones overlap.
const scanLanes = 4
-// scan appends every needle occurrence in t to hits, ordered by start, and returns them with the
-// set of needles found.
-func (sc *needleScanner) scan(t string, hits []redactHit) ([]redactHit, uint64) {
- var found uint64
+// scan appends every needle occurrence in t to hits, ordered by start, returns them, and sets
+// found to the needles that occur. found is written in place rather than returned: Go passes an
+// array of more than one word in memory, so returning one cost a copy on every text.
+func (sc *needleScanner) scan(t string, hits []redactHit, found *idSet) []redactHit {
+ *found = idSet{}
delta, class, first := sc.delta, &sc.class, uint32(sc.firstOut)
s, from := uint32(0), 0
if seg := len(t) / scanLanes; seg >= 64 {
@@ -788,7 +840,7 @@ func (sc *needleScanner) scan(t string, hits []redactHit) ([]redactHit, uint64)
if s0 >= first || s1 >= first || s2 >= first || s3 >= first {
for k, st := range [scanLanes]uint32{s0, s1, s2, s3} {
if end := k*seg - overlap*min(k, 1) + j + 1; st >= first && end > k*seg && end <= (k+1)*seg {
- hits = sc.record(t, st, end, hits, &found)
+ hits = sc.record(t, st, end, hits, found)
}
}
}
@@ -798,27 +850,36 @@ func (sc *needleScanner) scan(t string, hits []redactHit) ([]redactHit, uint64)
}
for i := from; i < len(t); i++ {
if s = delta[s+uint32(class[t[i]])]; s >= first {
- hits = sc.record(t, s, i+1, hits, &found)
+ hits = sc.record(t, s, i+1, hits, found)
}
}
// The automaton finds occurrences by where they end, lane by lane; the rules want them by
// where they start.
slices.SortStableFunc(hits, func(a, b redactHit) int { return a.start - b.start })
- return hits, found
+ return hits
}
// record appends the occurrences ending at end, where the automaton is in state s. A needle that
// is not folded must match the text byte for byte.
-func (sc *needleScanner) record(t string, s uint32, end int, hits []redactHit, found *uint64) []redactHit {
- for m := sc.out[(int(s)-sc.firstOut)/sc.nc]; m != 0; m &= m - 1 {
- id := bits.TrailingZeros64(m)
- nd := &sc.needles[id]
- start := end - len(nd.text)
- if !nd.folded && t[start:end] != nd.text {
- continue
+func (sc *needleScanner) record(t string, s uint32, end int, hits []redactHit, found *idSet) []redactHit {
+ // The words are read here, and each word's needles found gathered in a register and stored
+ // once, rather than through an iterator and idSet.add: a store per occurrence into found made
+ // the scan of 8 KiB of tool output about 8% slower.
+ out := &sc.out[(int(s)-sc.firstOut)/sc.nc]
+ for w, m := range out {
+ var got uint64
+ for ; m != 0; m &= m - 1 {
+ bit := bits.TrailingZeros64(m)
+ id := w<<6 | bit
+ nd := &sc.needles[id]
+ start := end - len(nd.text)
+ if !nd.folded && t[start:end] != nd.text {
+ continue
+ }
+ got |= 1 << bit
+ hits = append(hits, redactHit{start: start, end: end, id: id})
}
- *found |= 1 << id
- hits = append(hits, redactHit{start: start, end: end, id: id})
+ found[w] |= got
}
return hits
}
diff --git a/pkg/asymptoteobserve/redact_engine_test.go b/pkg/asymptoteobserve/redact_engine_test.go
index ac9d6344..75421283 100644
--- a/pkg/asymptoteobserve/redact_engine_test.go
+++ b/pkg/asymptoteobserve/redact_engine_test.go
@@ -44,7 +44,8 @@ func TestNeedleScannerFindsEveryOccurrence(t *testing.T) {
b.WriteString(pieces[r.IntN(len(pieces))])
}
text := b.String()
- got, found := sc.scan(text, nil)
+ var found idSet
+ got := sc.scan(text, nil, &found)
expected := want(text)
key := func(a, b redactHit) int {
if a.start != b.start {
@@ -57,12 +58,46 @@ func TestNeedleScannerFindsEveryOccurrence(t *testing.T) {
if !slices.Equal(got, expected) {
t.Fatalf("scan(%q)\n = %v\nwant %v", text, got, expected)
}
- var mask uint64
+ var mask idSet
for _, h := range expected {
- mask |= 1 << h.id
+ mask.add(h.id)
}
if found != mask {
- t.Fatalf("scan(%q) found %b, want %b", text, found, mask)
+ t.Fatalf("scan(%q) found %x, want %x", text, found, mask)
+ }
+ }
+}
+
+// The rules use fewer needles and sets than one word of an idSet holds, so nothing else reaches
+// the ids past 63. A scanner over idSetCap needles must report each one, in either word, and the
+// set of those found must hold exactly them.
+func TestNeedleScannerReportsIDsInEveryWord(t *testing.T) {
+ var sc needleScanner
+ needles := make([]redactNeedle, idSetCap)
+ for id := range needles {
+ needles[id] = redactNeedle{text: fmt.Sprintf("<n%d>", id)}
+ }
+ sc.build(needles)
+ for _, ids := range [][]int{{0}, {63}, {64}, {idSetCap - 1}, {1, 63, 64, 65, idSetCap - 1}} {
+ var text strings.Builder
+ var want idSet
+ for _, id := range ids {
+ text.WriteString("x " + needles[id].text)
+ want.add(id)
+ }
+ var found idSet
+ hits := sc.scan(text.String(), nil, &found)
+ var got []int
+ for _, h := range hits {
+ got = append(got, h.id)
+ }
+ if !slices.Equal(got, ids) || found != want {
+ t.Errorf("scan(%q): ids %v, found %x; want ids %v, found %x", text.String(), got, found, ids, want)
+ }
+ for id := range idSetCap {
+ if found.has(id) != slices.Contains(ids, id) {
+ t.Errorf("found.has(%d) = %t for ids %v", id, found.has(id), ids)
+ }
}
}
}