Skip to content

RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.

perf(tokenizer): memoize term lookups and precompile per-token regexes (#20610)

qinling0210966cd36ef60f
diff --git a/internal/service/nlp/term_weight.go b/internal/service/nlp/term_weight.go
index 2579726dbb..f99c4cdd91 100644
--- a/internal/service/nlp/term_weight.go
+++ b/internal/service/nlp/term_weight.go
@@ -23,6 +23,7 @@ import (
"regexp"
"strconv"
"strings"
+ "sync"
"unicode"
"ragflow/internal/tokenizer"
@@ -30,6 +31,47 @@ import (
"go.uber.org/zap"
)
+// Hoisted out of Weights' per-token loops: MatchString and MustCompile compile
+// the pattern on every call.
+var (
+ rePretokenDigit = regexp.MustCompile(`^[0-9]$`)
+ reShortAlnum = regexp.MustCompile(`^[0-9a-z]{1,2}$`)
+ reLeadingAlnum = regexp.MustCompile(`^[0-9a-zA-Z]`)
+ reSpaceRun = regexp.MustCompile(`[ \t]+`)
+ reEndsWithLetter = regexp.MustCompile(`.*[a-zA-Z]$`)
+ reNumLike = regexp.MustCompile(`^[0-9,.]{2,}$`)
+ reShortLetters = regexp.MustCompile(`^[a-z]{1,2}$`)
+ reNumSpace = regexp.MustCompile(`^[0-9. -]{2,}$`)
+ reTagNumeric = regexp.MustCompile(`^[0-9-]+`)
+)
+
+// Does not parse: the trailing \] escapes the closing bracket. MatchString used
+// to swallow that error and report "no match", so Pretoken's "#" substitution
+// has never fired. Compiling once keeps that behaviour.
+const pretokenPunctPattern = `[~—\t @#%!<>,\.\?":;'\{\}\[\]_=\(\)\|,。?》•●○↓《;':""【¥ 】…¥!、·()×\` + "`" + `&/「」\]`
+
+// Compiling here keeps the pattern off the per-token path, but the error cannot
+// be reported here: package-level variables are initialised before
+// common.InitLogger installs the global logger, so common.Warn would silently
+// drop it. The error is carried to warnPunctPatternOnce, which runs once the
+// logger is live. rePretokenPunct is nil while the pattern stays unparseable.
+var (
+ rePretokenPunct, pretokenPunctErr = regexp.Compile(pretokenPunctPattern)
+ pretokenPunctWarnOnce sync.Once
+)
+
+// warnPunctPatternOnce reports the unparseable punct pattern on the first call
+// to Pretoken. That runs on the request path, after InitLogger has installed the
+// global logger, which is where the warning is actually visible.
+func warnPunctPatternOnce() {
+ pretokenPunctWarnOnce.Do(func() {
+ if pretokenPunctErr != nil {
+ common.Warn("invalid regular expression; matching is skipped",
+ zap.String("pattern", pretokenPunctPattern), zap.Error(pretokenPunctErr))
+ }
+ })
+}
+
// TermWeightDealer calculates term weights for text processing
// Reference: rag/nlp/term_weight.py
type TermWeightDealer struct {
@@ -152,8 +194,7 @@ func loadDict(fnm string) map[string]int {
// Pretoken preprocesses and tokenizes text
// Reference: term_weight.py L92-114
func (d *TermWeightDealer) Pretoken(txt string, num bool, stpwd bool) []string {
- patt := `[~—\t @#%!<>,\.\?":;'\{\}\[\]_=\(\)\|,。?》•●○↓《;':""【¥ 】…¥!、·()×\` + "`" + `&/「」\]`
-
+ warnPunctPatternOnce()
res := []string{}
tokenized, err := tokenizer.Tokenize(txt)
if err != nil {
@@ -170,11 +211,11 @@ func (d *TermWeightDealer) Pretoken(txt string, num bool, stpwd bool) []string {
}
}
// Check single digit (unless num is true)
- if matched, _ := regexp.MatchString("^[0-9]$", tk); matched && !num {
+ if rePretokenDigit.MatchString(tk) && !num {
continue
}
- // Check patterns
- if matched, _ := regexp.MatchString(patt, t); matched {
+ // rePretokenPunct is nil while pretokenPunctPattern stays unparseable.
+ if rePretokenPunct != nil && rePretokenPunct.MatchString(t) {
tk = "#"
}
if tk != "#" && tk != "" {
@@ -194,8 +235,7 @@ func (d *TermWeightDealer) TokenMerge(tks []string) []string {
return true
}
// Match 1-2 alphanumeric characters
- matched, _ := regexp.MatchString("^[0-9a-z]{1,2}$", t)
- return matched
+ return reShortAlnum.MatchString(t)
}
if len(tks) == 0 {
@@ -209,7 +249,7 @@ func (d *TermWeightDealer) TokenMerge(tks []string) []string {
if i == 0 && len(tks) > 1 && oneTerm(tks[i]) {
nextLen := len([]rune(tks[i+1]))
isNextMultiChar := nextLen > 1
- isNextNotAlnum, _ := regexp.MatchString("^[0-9a-zA-Z]", tks[i+1])
+ isNextNotAlnum := reLeadingAlnum.MatchString(tks[i+1])
if isNextMultiChar && !isNextNotAlnum {
res = append(res, tks[0]+" "+tks[1])
i = 2
@@ -282,7 +322,7 @@ func (d *TermWeightDealer) Split(txt string) []string {
tks := []string{}
// Normalize spaces (tabs and multiple spaces -> single space)
- txt = regexp.MustCompile("[ \\t]+").ReplaceAllString(txt, " ")
+ txt = reSpaceRun.ReplaceAllString(txt, " ")
txt = strings.TrimSpace(txt)
for t := range strings.SplitSeq(txt, " ") {
@@ -291,8 +331,8 @@ func (d *TermWeightDealer) Split(txt string) []string {
continue
}
if len(tks) > 0 {
- prevEndsWithLetter, _ := regexp.MatchString(".*[a-zA-Z]$", tks[len(tks)-1])
- currEndsWithLetter, _ := regexp.MatchString(".*[a-zA-Z]$", t)
+ prevEndsWithLetter := reEndsWithLetter.MatchString(tks[len(tks)-1])
+ currEndsWithLetter := reEndsWithLetter.MatchString(t)
prevNE := d.ne[tks[len(tks)-1]]
currNE := d.ne[t]
if prevEndsWithLetter && currEndsWithLetter &&
@@ -309,16 +349,31 @@ func (d *TermWeightDealer) Split(txt string) []string {
// Weights calculates weights for tokens
// Reference: term_weight.py L163-246
func (d *TermWeightDealer) Weights(tks []string, preprocess bool) []TermWeight {
- numPattern := regexp.MustCompile("^[0-9,.]{2,}$")
- shortLetterPattern := regexp.MustCompile("^[a-z]{1,2}$")
- numSpacePattern := regexp.MustCompile("^[0-9. -]{2,}$")
+ // One dictionary lookup per distinct term. Both halves come from the same
+ // entry, tks repeats terms (title x2, important x5, question x6), and the
+ // consumers below are pure functions of the term, so caching the pair for the
+ // duration of this call cannot change any weight.
+ type freqTag struct {
+ freq int32
+ tag string
+ }
+ lookups := make(map[string]freqTag, len(tks))
+ lookup := func(t string) freqTag {
+ if v, ok := lookups[t]; ok {
+ return v
+ }
+ freq, tag := tokenizer.GetTermFreqAndTag(t)
+ v := freqTag{freq: freq, tag: tag}
+ lookups[t] = v
+ return v
+ }
// ner weight function
nerWeight := func(t string) float64 {
- if numPattern.MatchString(t) {
+ if reNumLike.MatchString(t) {
return 2
}
- if shortLetterPattern.MatchString(t) {
+ if reShortLetters.MatchString(t) {
return 0.01
}
if d.ne == nil {
@@ -338,7 +393,7 @@ func (d *TermWeightDealer) Weights(tks []string, preprocess bool) []TermWeight {
// postag weight function using real POS tagger
postagWeight := func(t string) float64 {
- tag := tokenizer.GetTermTag(t)
+ tag := lookup(t).tag
// Map POS tags to weights (matching Python implementation)
if tag == "r" || tag == "c" || tag == "d" {
return 0.3
@@ -350,7 +405,7 @@ func (d *TermWeightDealer) Weights(tks []string, preprocess bool) []TermWeight {
return 2
}
// Fallback to heuristic for terms without tags
- if matched, _ := regexp.MatchString("^[0-9-]+", tag); matched {
+ if reTagNumeric.MatchString(tag) {
return 2
}
return 1
@@ -359,11 +414,11 @@ func (d *TermWeightDealer) Weights(tks []string, preprocess bool) []TermWeight {
// freq function using real frequency dictionary
var freq func(t string) float64
freq = func(t string) float64 {
- if numSpacePattern.MatchString(t) {
+ if reNumSpace.MatchString(t) {
return 3
}
- // Use tokenizer's freq function
- s := tokenizer.GetTermFreq(t)
+ // Use the entry resolved for this term earlier in the call.
+ s := lookup(t).freq
if s == 0 {
if oovFrequency, ok := alphabeticOOVFrequency(t); ok {
return oovFrequency
@@ -407,7 +462,7 @@ func (d *TermWeightDealer) Weights(tks []string, preprocess bool) []TermWeight {
// df function
var df func(t string) float64
df = func(t string) float64 {
- if numSpacePattern.MatchString(t) {
+ if reNumSpace.MatchString(t) {
return 5
}
if v, ok := d.df[t]; ok {
diff --git a/internal/service/nlp/term_weight_test.go b/internal/service/nlp/term_weight_test.go
index c3007af6b2..6282726545 100644
--- a/internal/service/nlp/term_weight_test.go
+++ b/internal/service/nlp/term_weight_test.go
@@ -16,6 +16,7 @@ package nlp
import (
"encoding/json"
+ "math"
"os"
"path/filepath"
"reflect"
@@ -893,3 +894,47 @@ func TestSplitEdgeCases(t *testing.T) {
}
})
}
+
+// A term's weight must depend only on the term, never on where it sits in tks or
+// on how often it repeats. Weights relies on that to resolve one dictionary
+// lookup per distinct term instead of one per position, and the duplicate-token
+// multipliers (title x2, important x5, question x6) rely on it too.
+//
+// Weights normalizes by the sum, so compare a ratio between two terms: the
+// normalization cancels and only the underlying weights remain.
+func TestWeightsDependOnlyOnTheTerm(t *testing.T) {
+ d := NewTermWeightDealer("")
+
+ ratio := func(tks []string, i, j int) float64 {
+ weights := d.Weights(tks, false)
+ if len(weights) != len(tks) {
+ t.Fatalf("got %d weights for %d terms", len(weights), len(tks))
+ }
+ if weights[j].Weight == 0 {
+ t.Fatalf("weight of %q is zero, cannot form a ratio", tks[j])
+ }
+ return weights[i].Weight / weights[j].Weight
+ }
+
+ // Use two terms with visibly different weights: "a" takes the short-letter
+ // ner penalty while "hello" takes the out-of-vocabulary prior. With two
+ // similar terms the ratio would be 1 and every check below would hold even
+ // if the weights were position-dependent, so assert it really discriminates.
+ want := ratio([]string{"a", "hello"}, 0, 1)
+ if math.Abs(want-1) < 0.01 {
+ t.Fatalf("a/hello = %v: the terms are indistinguishable, so this test proves nothing", want)
+ }
+
+ if got := ratio([]string{"hello", "a"}, 1, 0); math.Abs(got-want) > 1e-12 {
+ t.Errorf("a/hello = %v when a leads, %v when it trails", want, got)
+ }
+ if got := ratio([]string{"a", "a", "hello"}, 0, 2); math.Abs(got-want) > 1e-12 {
+ t.Errorf("a/hello = %v alone, %v when a repeats first", want, got)
+ }
+ if got := ratio([]string{"a", "hello", "a"}, 0, 1); math.Abs(got-want) > 1e-12 {
+ t.Errorf("a/hello = %v adjacent, %v when separated", want, got)
+ }
+ if got := ratio([]string{"hello", "a", "a"}, 2, 0); math.Abs(got-want) > 1e-12 {
+ t.Errorf("a/hello = %v alone, %v when a repeats last", want, got)
+ }
+}
diff --git a/internal/tokenizer/term_lookup_cache_test.go b/internal/tokenizer/term_lookup_cache_test.go
new file mode 100644
index 0000000000..4e38b70a67
--- /dev/null
+++ b/internal/tokenizer/term_lookup_cache_test.go
@@ -0,0 +1,141 @@
+//
+// Copyright 2026 The InfiniFlow Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+
+package tokenizer
+
+import (
+ "fmt"
+ "sync"
+ "sync/atomic"
+ "testing"
+)
+
+// A full memo has to rotate, not freeze. Freezing lets the first
+// termLookupCacheMax distinct terms occupy the budget permanently, so a term
+// that only became hot later misses on every lookup with no way back; rotating
+// keeps the memo representing recent traffic.
+//
+// This test drives admitTerm directly with a small limit so it needs no
+// dictionary, and therefore runs without the "manual" build tag.
+func TestAdmitTermRotatesWhenFull(t *testing.T) {
+ const limit = 4
+ var (
+ cache sync.Map
+ counter atomic.Int64
+ )
+
+ for i, term := range []string{"a", "b", "c", "d"} {
+ admitTerm(&cache, &counter, limit, term, int32(i))
+ }
+ if got := counter.Load(); got != limit {
+ t.Fatalf("counter = %d after filling the memo, want %d", got, limit)
+ }
+
+ // The term that triggers the rotation was just looked up, so it has to end
+ // up in the new generation rather than being recomputed.
+ admitTerm(&cache, &counter, limit, "e", int32(9))
+ got, ok := cache.Load("e")
+ if !ok || got.(int32) != 9 {
+ t.Fatalf("term admitted at the cap = %v, present = %v; want 9, true", got, ok)
+ }
+ if n := counter.Load(); n != 1 {
+ t.Fatalf("counter = %d after rotation, want 1", n)
+ }
+ if _, ok := cache.Load("a"); ok {
+ t.Error("rotation kept an entry from the previous generation")
+ }
+
+ // Capacity must be usable again: the whole point is that terms arriving
+ // after the cap was reached still get memoized.
+ for _, term := range []string{"f", "g", "h"} {
+ admitTerm(&cache, &counter, limit, term, int32(0))
+ }
+ if n := counter.Load(); n != limit {
+ t.Fatalf("counter = %d after refilling, want %d", n, limit)
+ }
+ for _, term := range []string{"e", "f", "g", "h"} {
+ if _, ok := cache.Load(term); !ok {
+ t.Errorf("%q is missing after refill", term)
+ }
+ }
+}
+
+// A term that arrived before the rotation must be re-admitted afterwards: that
+// is what self-healing means. If this regressed to "insert only until full",
+// "old" would stay out forever.
+func TestAdmitTermReadmitsAfterRotation(t *testing.T) {
+ const limit = 2
+ var (
+ cache sync.Map
+ counter atomic.Int64
+ )
+
+ admitTerm(&cache, &counter, limit, "old", int32(1))
+ admitTerm(&cache, &counter, limit, "filler", int32(2))
+
+ // Fill past the cap so the generation containing "old" is dropped.
+ admitTerm(&cache, &counter, limit, "trigger", int32(3))
+ if _, ok := cache.Load("old"); ok {
+ t.Fatal("expected the previous generation to be gone")
+ }
+
+ // "old" comes back: the rotated memo must accept it again.
+ admitTerm(&cache, &counter, limit, "old", int32(1))
+ if v, ok := cache.Load("old"); !ok || v.(int32) != 1 {
+ t.Fatalf("re-admitted term = %v, present = %v; want 1, true", v, ok)
+ }
+}
+
+// Rotation clears the memo and resets the counter on the same path that
+// concurrent inserts use, so exercise it under contention. The value of this
+// test is being run with -race; the bound below also pins the documented
+// "limit plus writers in flight" overshoot.
+func TestAdmitTermIsRaceFreeUnderContention(t *testing.T) {
+ const (
+ limit = 4
+ workers = 8
+ perWorker = 64
+ )
+
+ var (
+ cache sync.Map
+ counter atomic.Int64
+ wg sync.WaitGroup
+ )
+
+ for w := 0; w < workers; w++ {
+ wg.Add(1)
+ go func(w int) {
+ defer wg.Done()
+ for i := 0; i < perWorker; i++ {
+ admitTerm(&cache, &counter, limit, fmt.Sprintf("w%d-%d", w, i), int32(i))
+ }
+ }(w)
+ }
+ wg.Wait()
+
+ entries := 0
+ cache.Range(func(_, _ any) bool {
+ entries++
+ return true
+ })
+ if entries == 0 {
+ t.Fatal("memo is empty after concurrent admission")
+ }
+ if entries > limit+workers {
+ t.Fatalf("memo holds %d entries, want at most limit+workers = %d", entries, limit+workers)
+ }
+}
diff --git a/internal/tokenizer/tokenizer.go b/internal/tokenizer/tokenizer.go
index 98446812ae..3124415d6b 100644
--- a/internal/tokenizer/tokenizer.go
+++ b/internal/tokenizer/tokenizer.go
@@ -115,6 +115,9 @@ func Init(cfg *PoolConfig) error {
cfg.AcquireTimeout = 10 * time.Second
}
+ // The dictionaries are reloaded below, so stale lookups must not survive.
+ resetTermLookupCaches()
+
common.Info("Initializing analyzer pool",
zap.String("dict_path", cfg.DictPath),
zap.Int("min_size", cfg.MinSize),
@@ -529,22 +532,100 @@ func IsInitialized() bool {
return globalPool != nil && globalPool.initialized
}
+// termLookupCacheMax caps each memo below. Reaching it rotates the memo instead
+// of freezing it: freezing lets whatever happened to arrive first occupy the
+// budget permanently, so a term that only became hot later would miss forever,
+// with no way back. Rotating keeps the memo representative of recent traffic and
+// costs one re-warm of the (small) hot set per rotation. This mirrors
+// bpeModel.memoPut in bpe.go, for the same reason: the memo is an optimisation,
+// not a correctness structure, so clearing it beats tracking recency.
+const termLookupCacheMax = 1 << 16
+
+// termCacheRotateMu serialises rotation so a burst of writers past the cap does
+// not clear the memo repeatedly. It is only taken on the rotation path, never on
+// a hit.
+var termCacheRotateMu sync.Mutex
+
+// Memoized because every lookup acquires a pooled analyzer and crosses cgo,
+// while Weights issues one freq plus one tag per token per retrieved chunk. A
+// result depends only on the term and on the dictionaries Init loads, so the
+// term alone is a valid key. Only successful lookups are cached, so a call made
+// before Init cannot pin a zero.
+//
+// The analyzer resolves both halves from the same packed trie entry, so they
+// share one memo rather than two maps keyed by the same term: that halves the
+// key storage, and a miss resolves the pair with a single analyzer checkout.
+var (
+ termLookupCache sync.Map // term -> termLookup
+ termLookupCached atomic.Int64
+)
+
+// termLookup is one dictionary entry: the pair the analyzer resolves together.
+type termLookup struct {
+ freq int32
+ tag string
+}
+
+func resetTermLookupCaches() {
+ termLookupCache.Clear()
+ termLookupCached.Store(0)
+}
+
+// admitTerm memoizes key -> val in cache, rotating the whole memo once it has
+// reached limit. Rotation runs before the insert, so the value just looked up
+// survives into the new generation instead of being recomputed.
+//
+// Admission is a check-then-act: concurrent writers can overshoot limit by the
+// number of writers in flight, and a rotation can race an in-flight insert (that
+// entry is dropped, its count lost with it). Both leave the memo bounded and
+// approximately counted, which is all this policy needs.
+func admitTerm[V any](cache *sync.Map, counter *atomic.Int64, limit int64, key string, val V) {
+ if counter.Load() >= limit {
+ termCacheRotateMu.Lock()
+ if counter.Load() >= limit {
+ cache.Clear()
+ counter.Store(0)
+ common.Info("term lookup memo rotated", zap.Int64("limit", limit))
+ }
+ termCacheRotateMu.Unlock()
+ }
+ if _, loaded := cache.LoadOrStore(key, val); !loaded {
+ counter.Add(1)
+ }
+}
+
+// GetTermFreqAndTag returns the frequency and the POS tag of a term, resolving
+// both in one analyzer checkout. They come from the same packed trie entry, so
+// asking for them separately pays for two checkouts and two cgo round trips.
+// Returns: the frequency, or 0 if the term is not in the dictionary; and the POS
+// tag, or "" if it has none.
+func GetTermFreqAndTag(term string) (int32, string) {
+ if v, ok := termLookupCache.Load(term); ok {
+ entry := v.(termLookup)
+ return entry.freq, entry.tag
+ }
+ entry, err := withAnalyzerResult("", func(a *rag.Analyzer) (termLookup, error) {
+ return termLookup{freq: a.GetTermFreq(term), tag: a.GetTermTag(term)}, nil
+ })
+ if err != nil {
+ return entry.freq, entry.tag
+ }
+ admitTerm(&termLookupCache, &termLookupCached, termLookupCacheMax, term, entry)
+ return entry.freq, entry.tag
+}
+
// GetTermFreq returns the frequency of a term (matching Python rag_tokenizer.freq)
// Returns: frequency value, or 0 if term not found
func GetTermFreq(term string) int32 {
- result, _ := withAnalyzerResult("", func(a *rag.Analyzer) (int32, error) {
- return a.GetTermFreq(term), nil
- })
- return result
+ freq, _ := GetTermFreqAndTag(term)
+ return freq
}
// GetTermTag returns the POS tag of a term (matching Python rag_tokenizer.tag)
// Returns: POS tag string (e.g., "n", "v", "ns"), or empty string if term not found or no tag
func GetTermTag(term string) string {
- result, _ := withAnalyzerResult("", func(a *rag.Analyzer) (string, error) {
- return a.GetTermTag(term), nil
- })
- return result
+ _, tag := GetTermFreqAndTag(term)
+ return tag
}
var cl100kEncoder struct {
diff --git a/internal/tokenizer/tokenizer_test.go b/internal/tokenizer/tokenizer_test.go
index 380805b577..669353cde4 100644
--- a/internal/tokenizer/tokenizer_test.go
+++ b/internal/tokenizer/tokenizer_test.go
@@ -230,6 +230,100 @@ func TestGetTermTag_PoolNotInitialized(t *testing.T) {
}
}
+// The memo must hold the analyzer's answer (never a pre-Init zero), cost exactly
+// one entry per term whichever accessor is used, not grow on a repeat lookup, and
+// be dropped when the dictionaries are reloaded.
+func TestTermLookupCacheMemoizesPerTermAndDoesNotGrowOnHit(t *testing.T) {
+ if err := Init(&PoolConfig{
+ DictPath: "",
+ MinSize: 1,
+ MaxSize: 1,
+ IdleTimeout: 5 * time.Second,
+ AcquireTimeout: 5 * time.Second,
+ }); err != nil {
+ t.Fatalf("Failed to initialize pool: %v", err)
+ }
+ defer Close()
+ resetTermLookupCaches()
+
+ const term = "hello"
+ freq, tag := GetTermFreqAndTag(term)
+
+ cached, ok := termLookupCache.Load(term)
+ if !ok {
+ t.Fatalf("GetTermFreqAndTag(%q) did not memoize its result", term)
+ }
+ if entry := cached.(termLookup); entry.freq != freq || entry.tag != tag {
+ t.Fatalf("memoized %+v, want {freq:%d tag:%q}", entry, freq, tag)
+ }
+
+ if gotFreq, gotTag := GetTermFreqAndTag(term); gotFreq != freq || gotTag != tag {
+ t.Fatalf("second lookup = (%d, %q), want (%d, %q)", gotFreq, gotTag, freq, tag)
+ }
+ // The single-value accessors read the same memo rather than keeping their
+ // own: one term occupies one entry however it is asked for.
+ if got := GetTermFreq(term); got != freq {
+ t.Fatalf("GetTermFreq(%q) = %d, want %d", term, got, freq)
+ }
+ if got := GetTermTag(term); got != tag {
+ t.Fatalf("GetTermTag(%q) = %q, want %q", term, got, tag)
+ }
+ if got := termLookupCached.Load(); got != 1 {
+ t.Fatalf("memo holds %d entries after one term, want 1", got)
+ }
+
+ // A miss resolves both halves in one analyzer checkout, so asking for just
+ // the frequency still leaves a complete entry behind for the tag.
+ resetTermLookupCaches()
+ if got := GetTermFreq(term); got != freq {
+ t.Fatalf("GetTermFreq(%q) after reset = %d, want %d", term, got, freq)
+ }
+ if entry, ok := termLookupCache.Load(term); !ok || entry.(termLookup).tag != tag {
+ t.Fatalf("a frequency-only lookup stored %v, want a full entry with tag %q", entry, tag)
+ }
+
+ resetTermLookupCaches()
+ if _, ok := termLookupCache.Load(term); ok {
+ t.Fatal("resetTermLookupCaches left an entry behind")
+ }
+ if got := termLookupCached.Load(); got != 0 {
+ t.Fatalf("memo counter = %d after reset, want 0", got)
+ }
+}
+
+// The three accessors have to agree, and the combined one has to be stable on a
+// repeat. The memo stores whatever the combined lookup returns, so a divergence
+// here would silently change weights. The terms mix in-dictionary entries (so
+// the tag is non-empty) with out-of-vocabulary ones.
+func TestGetTermFreqAndTagAgreesWithSingleValueAccessors(t *testing.T) {
+ if err := Init(&PoolConfig{
+ DictPath: "",
+ MinSize: 1,
+ MaxSize: 1,
+ IdleTimeout: 5 * time.Second,
+ AcquireTimeout: 5 * time.Second,
+ }); err != nil {
+ t.Fatalf("Failed to initialize pool: %v", err)
+ }
+ defer Close()
+ resetTermLookupCaches()
+
+ for _, term := range []string{"hello", "文档", "北京", "xyzzy", "ab"} {
+ freq, tag := GetTermFreqAndTag(term)
+
+ if got := GetTermFreq(term); got != freq {
+ t.Errorf("GetTermFreq(%q) = %d, want %d", term, got, freq)
+ }
+ if got := GetTermTag(term); got != tag {
+ t.Errorf("GetTermTag(%q) = %q, want %q", term, got, tag)
+ }
+ if freq2, tag2 := GetTermFreqAndTag(term); freq2 != freq || tag2 != tag {
+ t.Errorf("repeat lookup for %q = (%d, %q), want (%d, %q)",
+ term, freq2, tag2, freq, tag)
+ }
+ }
+}
+
func TestTokenize_DefaultLanguageResetsAnalyzerState(t *testing.T) {
restore := saveEngineType()
defer restore()