RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.
Go agent-harnessagent-native-databaseagentic-aiagentic-retrievalagentic-searchaiai-agentscontext-enginecontext-engineeringcontext-managementharness-engineeringknowledge-compilationragretrieval-augmented-generationretrieval-harnesssearch-harness
perf(tokenizer): memoize term lookups and precompile per-token regexes (#20610)
qinling0210966cd36ef60f
diff --git a/internal/service/nlp/term_weight.go b/internal/service/nlp/term_weight.goindex 2579726dbb..f99c4cdd91 100644--- a/internal/service/nlp/term_weight.go+++ b/internal/service/nlp/term_weight.go@@ -23,6 +23,7 @@ import ("regexp""strconv""strings"+ "sync""unicode""ragflow/internal/tokenizer"@@ -30,6 +31,47 @@ import ("go.uber.org/zap")+// Hoisted out of Weights' per-token loops: MatchString and MustCompile compile+// the pattern on every call.+var (+ rePretokenDigit = regexp.MustCompile(`^[0-9]$`)+ reShortAlnum = regexp.MustCompile(`^[0-9a-z]{1,2}$`)+ reLeadingAlnum = regexp.MustCompile(`^[0-9a-zA-Z]`)+ reSpaceRun = regexp.MustCompile(`[ \t]+`)+ reEndsWithLetter = regexp.MustCompile(`.*[a-zA-Z]$`)+ reNumLike = regexp.MustCompile(`^[0-9,.]{2,}$`)+ reShortLetters = regexp.MustCompile(`^[a-z]{1,2}$`)+ reNumSpace = regexp.MustCompile(`^[0-9. -]{2,}$`)+ reTagNumeric = regexp.MustCompile(`^[0-9-]+`)+)++// Does not parse: the trailing \] escapes the closing bracket. MatchString used+// to swallow that error and report "no match", so Pretoken's "#" substitution+// has never fired. Compiling once keeps that behaviour.+const pretokenPunctPattern = `[~—\t @#%!<>,\.\?":;'\{\}\[\]_=\(\)\|,。?》•●○↓《;':""【¥ 】…¥!、·()×\` + "`" + `&/「」\]`++// Compiling here keeps the pattern off the per-token path, but the error cannot+// be reported here: package-level variables are initialised before+// common.InitLogger installs the global logger, so common.Warn would silently+// drop it. The error is carried to warnPunctPatternOnce, which runs once the+// logger is live. rePretokenPunct is nil while the pattern stays unparseable.+var (+ rePretokenPunct, pretokenPunctErr = regexp.Compile(pretokenPunctPattern)+ pretokenPunctWarnOnce sync.Once+)++// warnPunctPatternOnce reports the unparseable punct pattern on the first call+// to Pretoken. That runs on the request path, after InitLogger has installed the+// global logger, which is where the warning is actually visible.+func warnPunctPatternOnce() {+ pretokenPunctWarnOnce.Do(func() {+ if pretokenPunctErr != nil {+ common.Warn("invalid regular expression; matching is skipped",+ zap.String("pattern", pretokenPunctPattern), zap.Error(pretokenPunctErr))+ }+ })+}+// TermWeightDealer calculates term weights for text processing// Reference: rag/nlp/term_weight.pytype TermWeightDealer struct {@@ -152,8 +194,7 @@ func loadDict(fnm string) map[string]int {// Pretoken preprocesses and tokenizes text// Reference: term_weight.py L92-114func (d *TermWeightDealer) Pretoken(txt string, num bool, stpwd bool) []string {- patt := `[~—\t @#%!<>,\.\?":;'\{\}\[\]_=\(\)\|,。?》•●○↓《;':""【¥ 】…¥!、·()×\` + "`" + `&/「」\]`-+ warnPunctPatternOnce()res := []string{}tokenized, err := tokenizer.Tokenize(txt)if err != nil {@@ -170,11 +211,11 @@ func (d *TermWeightDealer) Pretoken(txt string, num bool, stpwd bool) []string {}}// Check single digit (unless num is true)- if matched, _ := regexp.MatchString("^[0-9]$", tk); matched && !num {+ if rePretokenDigit.MatchString(tk) && !num {continue}- // Check patterns- if matched, _ := regexp.MatchString(patt, t); matched {+ // rePretokenPunct is nil while pretokenPunctPattern stays unparseable.+ if rePretokenPunct != nil && rePretokenPunct.MatchString(t) {tk = "#"}if tk != "#" && tk != "" {@@ -194,8 +235,7 @@ func (d *TermWeightDealer) TokenMerge(tks []string) []string {return true}// Match 1-2 alphanumeric characters- matched, _ := regexp.MatchString("^[0-9a-z]{1,2}$", t)- return matched+ return reShortAlnum.MatchString(t)}if len(tks) == 0 {@@ -209,7 +249,7 @@ func (d *TermWeightDealer) TokenMerge(tks []string) []string {if i == 0 && len(tks) > 1 && oneTerm(tks[i]) {nextLen := len([]rune(tks[i+1]))isNextMultiChar := nextLen > 1- isNextNotAlnum, _ := regexp.MatchString("^[0-9a-zA-Z]", tks[i+1])+ isNextNotAlnum := reLeadingAlnum.MatchString(tks[i+1])if isNextMultiChar && !isNextNotAlnum {res = append(res, tks[0]+" "+tks[1])i = 2@@ -282,7 +322,7 @@ func (d *TermWeightDealer) Split(txt string) []string {tks := []string{}// Normalize spaces (tabs and multiple spaces -> single space)- txt = regexp.MustCompile("[ \\t]+").ReplaceAllString(txt, " ")+ txt = reSpaceRun.ReplaceAllString(txt, " ")txt = strings.TrimSpace(txt)for t := range strings.SplitSeq(txt, " ") {@@ -291,8 +331,8 @@ func (d *TermWeightDealer) Split(txt string) []string {continue}if len(tks) > 0 {- prevEndsWithLetter, _ := regexp.MatchString(".*[a-zA-Z]$", tks[len(tks)-1])- currEndsWithLetter, _ := regexp.MatchString(".*[a-zA-Z]$", t)+ prevEndsWithLetter := reEndsWithLetter.MatchString(tks[len(tks)-1])+ currEndsWithLetter := reEndsWithLetter.MatchString(t)prevNE := d.ne[tks[len(tks)-1]]currNE := d.ne[t]if prevEndsWithLetter && currEndsWithLetter &&@@ -309,16 +349,31 @@ func (d *TermWeightDealer) Split(txt string) []string {// Weights calculates weights for tokens// Reference: term_weight.py L163-246func (d *TermWeightDealer) Weights(tks []string, preprocess bool) []TermWeight {- numPattern := regexp.MustCompile("^[0-9,.]{2,}$")- shortLetterPattern := regexp.MustCompile("^[a-z]{1,2}$")- numSpacePattern := regexp.MustCompile("^[0-9. -]{2,}$")+ // One dictionary lookup per distinct term. Both halves come from the same+ // entry, tks repeats terms (title x2, important x5, question x6), and the+ // consumers below are pure functions of the term, so caching the pair for the+ // duration of this call cannot change any weight.+ type freqTag struct {+ freq int32+ tag string+ }+ lookups := make(map[string]freqTag, len(tks))+ lookup := func(t string) freqTag {+ if v, ok := lookups[t]; ok {+ return v+ }+ freq, tag := tokenizer.GetTermFreqAndTag(t)+ v := freqTag{freq: freq, tag: tag}+ lookups[t] = v+ return v+ }// ner weight functionnerWeight := func(t string) float64 {- if numPattern.MatchString(t) {+ if reNumLike.MatchString(t) {return 2}- if shortLetterPattern.MatchString(t) {+ if reShortLetters.MatchString(t) {return 0.01}if d.ne == nil {@@ -338,7 +393,7 @@ func (d *TermWeightDealer) Weights(tks []string, preprocess bool) []TermWeight {// postag weight function using real POS taggerpostagWeight := func(t string) float64 {- tag := tokenizer.GetTermTag(t)+ tag := lookup(t).tag// Map POS tags to weights (matching Python implementation)if tag == "r" || tag == "c" || tag == "d" {return 0.3@@ -350,7 +405,7 @@ func (d *TermWeightDealer) Weights(tks []string, preprocess bool) []TermWeight {return 2}// Fallback to heuristic for terms without tags- if matched, _ := regexp.MatchString("^[0-9-]+", tag); matched {+ if reTagNumeric.MatchString(tag) {return 2}return 1@@ -359,11 +414,11 @@ func (d *TermWeightDealer) Weights(tks []string, preprocess bool) []TermWeight {// freq function using real frequency dictionaryvar freq func(t string) float64freq = func(t string) float64 {- if numSpacePattern.MatchString(t) {+ if reNumSpace.MatchString(t) {return 3}- // Use tokenizer's freq function- s := tokenizer.GetTermFreq(t)+ // Use the entry resolved for this term earlier in the call.+ s := lookup(t).freqif s == 0 {if oovFrequency, ok := alphabeticOOVFrequency(t); ok {return oovFrequency@@ -407,7 +462,7 @@ func (d *TermWeightDealer) Weights(tks []string, preprocess bool) []TermWeight {// df functionvar df func(t string) float64df = func(t string) float64 {- if numSpacePattern.MatchString(t) {+ if reNumSpace.MatchString(t) {return 5}if v, ok := d.df[t]; ok {diff --git a/internal/service/nlp/term_weight_test.go b/internal/service/nlp/term_weight_test.goindex c3007af6b2..6282726545 100644--- a/internal/service/nlp/term_weight_test.go+++ b/internal/service/nlp/term_weight_test.go@@ -16,6 +16,7 @@ package nlpimport ("encoding/json"+ "math""os""path/filepath""reflect"@@ -893,3 +894,47 @@ func TestSplitEdgeCases(t *testing.T) {}})}++// A term's weight must depend only on the term, never on where it sits in tks or+// on how often it repeats. Weights relies on that to resolve one dictionary+// lookup per distinct term instead of one per position, and the duplicate-token+// multipliers (title x2, important x5, question x6) rely on it too.+//+// Weights normalizes by the sum, so compare a ratio between two terms: the+// normalization cancels and only the underlying weights remain.+func TestWeightsDependOnlyOnTheTerm(t *testing.T) {+ d := NewTermWeightDealer("")++ ratio := func(tks []string, i, j int) float64 {+ weights := d.Weights(tks, false)+ if len(weights) != len(tks) {+ t.Fatalf("got %d weights for %d terms", len(weights), len(tks))+ }+ if weights[j].Weight == 0 {+ t.Fatalf("weight of %q is zero, cannot form a ratio", tks[j])+ }+ return weights[i].Weight / weights[j].Weight+ }++ // Use two terms with visibly different weights: "a" takes the short-letter+ // ner penalty while "hello" takes the out-of-vocabulary prior. With two+ // similar terms the ratio would be 1 and every check below would hold even+ // if the weights were position-dependent, so assert it really discriminates.+ want := ratio([]string{"a", "hello"}, 0, 1)+ if math.Abs(want-1) < 0.01 {+ t.Fatalf("a/hello = %v: the terms are indistinguishable, so this test proves nothing", want)+ }++ if got := ratio([]string{"hello", "a"}, 1, 0); math.Abs(got-want) > 1e-12 {+ t.Errorf("a/hello = %v when a leads, %v when it trails", want, got)+ }+ if got := ratio([]string{"a", "a", "hello"}, 0, 2); math.Abs(got-want) > 1e-12 {+ t.Errorf("a/hello = %v alone, %v when a repeats first", want, got)+ }+ if got := ratio([]string{"a", "hello", "a"}, 0, 1); math.Abs(got-want) > 1e-12 {+ t.Errorf("a/hello = %v adjacent, %v when separated", want, got)+ }+ if got := ratio([]string{"hello", "a", "a"}, 2, 0); math.Abs(got-want) > 1e-12 {+ t.Errorf("a/hello = %v alone, %v when a repeats last", want, got)+ }+}diff --git a/internal/tokenizer/term_lookup_cache_test.go b/internal/tokenizer/term_lookup_cache_test.gonew file mode 100644index 0000000000..4e38b70a67--- /dev/null+++ b/internal/tokenizer/term_lookup_cache_test.go@@ -0,0 +1,141 @@+//+// Copyright 2026 The InfiniFlow Authors. All Rights Reserved.+//+// Licensed under the Apache License, Version 2.0 (the "License");+// you may not use this file except in compliance with the License.+// You may obtain a copy of the License at+//+// http://www.apache.org/licenses/LICENSE-2.0+//+// Unless required by applicable law or agreed to in writing, software+// distributed under the License is distributed on an "AS IS" BASIS,+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+// See the License for the specific language governing permissions and+// limitations under the License.+//++package tokenizer++import (+ "fmt"+ "sync"+ "sync/atomic"+ "testing"+)++// A full memo has to rotate, not freeze. Freezing lets the first+// termLookupCacheMax distinct terms occupy the budget permanently, so a term+// that only became hot later misses on every lookup with no way back; rotating+// keeps the memo representing recent traffic.+//+// This test drives admitTerm directly with a small limit so it needs no+// dictionary, and therefore runs without the "manual" build tag.+func TestAdmitTermRotatesWhenFull(t *testing.T) {+ const limit = 4+ var (+ cache sync.Map+ counter atomic.Int64+ )++ for i, term := range []string{"a", "b", "c", "d"} {+ admitTerm(&cache, &counter, limit, term, int32(i))+ }+ if got := counter.Load(); got != limit {+ t.Fatalf("counter = %d after filling the memo, want %d", got, limit)+ }++ // The term that triggers the rotation was just looked up, so it has to end+ // up in the new generation rather than being recomputed.+ admitTerm(&cache, &counter, limit, "e", int32(9))+ got, ok := cache.Load("e")+ if !ok || got.(int32) != 9 {+ t.Fatalf("term admitted at the cap = %v, present = %v; want 9, true", got, ok)+ }+ if n := counter.Load(); n != 1 {+ t.Fatalf("counter = %d after rotation, want 1", n)+ }+ if _, ok := cache.Load("a"); ok {+ t.Error("rotation kept an entry from the previous generation")+ }++ // Capacity must be usable again: the whole point is that terms arriving+ // after the cap was reached still get memoized.+ for _, term := range []string{"f", "g", "h"} {+ admitTerm(&cache, &counter, limit, term, int32(0))+ }+ if n := counter.Load(); n != limit {+ t.Fatalf("counter = %d after refilling, want %d", n, limit)+ }+ for _, term := range []string{"e", "f", "g", "h"} {+ if _, ok := cache.Load(term); !ok {+ t.Errorf("%q is missing after refill", term)+ }+ }+}++// A term that arrived before the rotation must be re-admitted afterwards: that+// is what self-healing means. If this regressed to "insert only until full",+// "old" would stay out forever.+func TestAdmitTermReadmitsAfterRotation(t *testing.T) {+ const limit = 2+ var (+ cache sync.Map+ counter atomic.Int64+ )++ admitTerm(&cache, &counter, limit, "old", int32(1))+ admitTerm(&cache, &counter, limit, "filler", int32(2))++ // Fill past the cap so the generation containing "old" is dropped.+ admitTerm(&cache, &counter, limit, "trigger", int32(3))+ if _, ok := cache.Load("old"); ok {+ t.Fatal("expected the previous generation to be gone")+ }++ // "old" comes back: the rotated memo must accept it again.+ admitTerm(&cache, &counter, limit, "old", int32(1))+ if v, ok := cache.Load("old"); !ok || v.(int32) != 1 {+ t.Fatalf("re-admitted term = %v, present = %v; want 1, true", v, ok)+ }+}++// Rotation clears the memo and resets the counter on the same path that+// concurrent inserts use, so exercise it under contention. The value of this+// test is being run with -race; the bound below also pins the documented+// "limit plus writers in flight" overshoot.+func TestAdmitTermIsRaceFreeUnderContention(t *testing.T) {+ const (+ limit = 4+ workers = 8+ perWorker = 64+ )++ var (+ cache sync.Map+ counter atomic.Int64+ wg sync.WaitGroup+ )++ for w := 0; w < workers; w++ {+ wg.Add(1)+ go func(w int) {+ defer wg.Done()+ for i := 0; i < perWorker; i++ {+ admitTerm(&cache, &counter, limit, fmt.Sprintf("w%d-%d", w, i), int32(i))+ }+ }(w)+ }+ wg.Wait()++ entries := 0+ cache.Range(func(_, _ any) bool {+ entries+++ return true+ })+ if entries == 0 {+ t.Fatal("memo is empty after concurrent admission")+ }+ if entries > limit+workers {+ t.Fatalf("memo holds %d entries, want at most limit+workers = %d", entries, limit+workers)+ }+}diff --git a/internal/tokenizer/tokenizer.go b/internal/tokenizer/tokenizer.goindex 98446812ae..3124415d6b 100644--- a/internal/tokenizer/tokenizer.go+++ b/internal/tokenizer/tokenizer.go@@ -115,6 +115,9 @@ func Init(cfg *PoolConfig) error {cfg.AcquireTimeout = 10 * time.Second}+ // The dictionaries are reloaded below, so stale lookups must not survive.+ resetTermLookupCaches()+common.Info("Initializing analyzer pool",zap.String("dict_path", cfg.DictPath),zap.Int("min_size", cfg.MinSize),@@ -529,22 +532,100 @@ func IsInitialized() bool {return globalPool != nil && globalPool.initialized}+// termLookupCacheMax caps each memo below. Reaching it rotates the memo instead+// of freezing it: freezing lets whatever happened to arrive first occupy the+// budget permanently, so a term that only became hot later would miss forever,+// with no way back. Rotating keeps the memo representative of recent traffic and+// costs one re-warm of the (small) hot set per rotation. This mirrors+// bpeModel.memoPut in bpe.go, for the same reason: the memo is an optimisation,+// not a correctness structure, so clearing it beats tracking recency.+const termLookupCacheMax = 1 << 16++// termCacheRotateMu serialises rotation so a burst of writers past the cap does+// not clear the memo repeatedly. It is only taken on the rotation path, never on+// a hit.+var termCacheRotateMu sync.Mutex++// Memoized because every lookup acquires a pooled analyzer and crosses cgo,+// while Weights issues one freq plus one tag per token per retrieved chunk. A+// result depends only on the term and on the dictionaries Init loads, so the+// term alone is a valid key. Only successful lookups are cached, so a call made+// before Init cannot pin a zero.+//+// The analyzer resolves both halves from the same packed trie entry, so they+// share one memo rather than two maps keyed by the same term: that halves the+// key storage, and a miss resolves the pair with a single analyzer checkout.+var (+ termLookupCache sync.Map // term -> termLookup+ termLookupCached atomic.Int64+)++// termLookup is one dictionary entry: the pair the analyzer resolves together.+type termLookup struct {+ freq int32+ tag string+}++func resetTermLookupCaches() {+ termLookupCache.Clear()+ termLookupCached.Store(0)+}++// admitTerm memoizes key -> val in cache, rotating the whole memo once it has+// reached limit. Rotation runs before the insert, so the value just looked up+// survives into the new generation instead of being recomputed.+//+// Admission is a check-then-act: concurrent writers can overshoot limit by the+// number of writers in flight, and a rotation can race an in-flight insert (that+// entry is dropped, its count lost with it). Both leave the memo bounded and+// approximately counted, which is all this policy needs.+func admitTerm[V any](cache *sync.Map, counter *atomic.Int64, limit int64, key string, val V) {+ if counter.Load() >= limit {+ termCacheRotateMu.Lock()+ if counter.Load() >= limit {+ cache.Clear()+ counter.Store(0)+ common.Info("term lookup memo rotated", zap.Int64("limit", limit))+ }+ termCacheRotateMu.Unlock()+ }+ if _, loaded := cache.LoadOrStore(key, val); !loaded {+ counter.Add(1)+ }+}++// GetTermFreqAndTag returns the frequency and the POS tag of a term, resolving+// both in one analyzer checkout. They come from the same packed trie entry, so+// asking for them separately pays for two checkouts and two cgo round trips.+// Returns: the frequency, or 0 if the term is not in the dictionary; and the POS+// tag, or "" if it has none.+func GetTermFreqAndTag(term string) (int32, string) {+ if v, ok := termLookupCache.Load(term); ok {+ entry := v.(termLookup)+ return entry.freq, entry.tag+ }+ entry, err := withAnalyzerResult("", func(a *rag.Analyzer) (termLookup, error) {+ return termLookup{freq: a.GetTermFreq(term), tag: a.GetTermTag(term)}, nil+ })+ if err != nil {+ return entry.freq, entry.tag+ }+ admitTerm(&termLookupCache, &termLookupCached, termLookupCacheMax, term, entry)+ return entry.freq, entry.tag+}+// GetTermFreq returns the frequency of a term (matching Python rag_tokenizer.freq)// Returns: frequency value, or 0 if term not foundfunc GetTermFreq(term string) int32 {- result, _ := withAnalyzerResult("", func(a *rag.Analyzer) (int32, error) {- return a.GetTermFreq(term), nil- })- return result+ freq, _ := GetTermFreqAndTag(term)+ return freq}// GetTermTag returns the POS tag of a term (matching Python rag_tokenizer.tag)// Returns: POS tag string (e.g., "n", "v", "ns"), or empty string if term not found or no tagfunc GetTermTag(term string) string {- result, _ := withAnalyzerResult("", func(a *rag.Analyzer) (string, error) {- return a.GetTermTag(term), nil- })- return result+ _, tag := GetTermFreqAndTag(term)+ return tag}var cl100kEncoder struct {diff --git a/internal/tokenizer/tokenizer_test.go b/internal/tokenizer/tokenizer_test.goindex 380805b577..669353cde4 100644--- a/internal/tokenizer/tokenizer_test.go+++ b/internal/tokenizer/tokenizer_test.go@@ -230,6 +230,100 @@ func TestGetTermTag_PoolNotInitialized(t *testing.T) {}}+// The memo must hold the analyzer's answer (never a pre-Init zero), cost exactly+// one entry per term whichever accessor is used, not grow on a repeat lookup, and+// be dropped when the dictionaries are reloaded.+func TestTermLookupCacheMemoizesPerTermAndDoesNotGrowOnHit(t *testing.T) {+ if err := Init(&PoolConfig{+ DictPath: "",+ MinSize: 1,+ MaxSize: 1,+ IdleTimeout: 5 * time.Second,+ AcquireTimeout: 5 * time.Second,+ }); err != nil {+ t.Fatalf("Failed to initialize pool: %v", err)+ }+ defer Close()+ resetTermLookupCaches()++ const term = "hello"+ freq, tag := GetTermFreqAndTag(term)++ cached, ok := termLookupCache.Load(term)+ if !ok {+ t.Fatalf("GetTermFreqAndTag(%q) did not memoize its result", term)+ }+ if entry := cached.(termLookup); entry.freq != freq || entry.tag != tag {+ t.Fatalf("memoized %+v, want {freq:%d tag:%q}", entry, freq, tag)+ }++ if gotFreq, gotTag := GetTermFreqAndTag(term); gotFreq != freq || gotTag != tag {+ t.Fatalf("second lookup = (%d, %q), want (%d, %q)", gotFreq, gotTag, freq, tag)+ }+ // The single-value accessors read the same memo rather than keeping their+ // own: one term occupies one entry however it is asked for.+ if got := GetTermFreq(term); got != freq {+ t.Fatalf("GetTermFreq(%q) = %d, want %d", term, got, freq)+ }+ if got := GetTermTag(term); got != tag {+ t.Fatalf("GetTermTag(%q) = %q, want %q", term, got, tag)+ }+ if got := termLookupCached.Load(); got != 1 {+ t.Fatalf("memo holds %d entries after one term, want 1", got)+ }++ // A miss resolves both halves in one analyzer checkout, so asking for just+ // the frequency still leaves a complete entry behind for the tag.+ resetTermLookupCaches()+ if got := GetTermFreq(term); got != freq {+ t.Fatalf("GetTermFreq(%q) after reset = %d, want %d", term, got, freq)+ }+ if entry, ok := termLookupCache.Load(term); !ok || entry.(termLookup).tag != tag {+ t.Fatalf("a frequency-only lookup stored %v, want a full entry with tag %q", entry, tag)+ }++ resetTermLookupCaches()+ if _, ok := termLookupCache.Load(term); ok {+ t.Fatal("resetTermLookupCaches left an entry behind")+ }+ if got := termLookupCached.Load(); got != 0 {+ t.Fatalf("memo counter = %d after reset, want 0", got)+ }+}++// The three accessors have to agree, and the combined one has to be stable on a+// repeat. The memo stores whatever the combined lookup returns, so a divergence+// here would silently change weights. The terms mix in-dictionary entries (so+// the tag is non-empty) with out-of-vocabulary ones.+func TestGetTermFreqAndTagAgreesWithSingleValueAccessors(t *testing.T) {+ if err := Init(&PoolConfig{+ DictPath: "",+ MinSize: 1,+ MaxSize: 1,+ IdleTimeout: 5 * time.Second,+ AcquireTimeout: 5 * time.Second,+ }); err != nil {+ t.Fatalf("Failed to initialize pool: %v", err)+ }+ defer Close()+ resetTermLookupCaches()++ for _, term := range []string{"hello", "文档", "北京", "xyzzy", "ab"} {+ freq, tag := GetTermFreqAndTag(term)++ if got := GetTermFreq(term); got != freq {+ t.Errorf("GetTermFreq(%q) = %d, want %d", term, got, freq)+ }+ if got := GetTermTag(term); got != tag {+ t.Errorf("GetTermTag(%q) = %q, want %q", term, got, tag)+ }+ if freq2, tag2 := GetTermFreqAndTag(term); freq2 != freq || tag2 != tag {+ t.Errorf("repeat lookup for %q = (%d, %q), want (%d, %q)",+ term, freq2, tag2, freq, tag)+ }+ }+}+func TestTokenize_DefaultLanguageResetsAnalyzerState(t *testing.T) {restore := saveEngineType()defer restore()