RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.
Go agent-harnessagent-native-databaseagentic-aiagentic-retrievalagentic-searchaiai-agentscontext-enginecontext-engineeringcontext-managementharness-engineeringknowledge-compilationragretrieval-augmented-generationretrieval-harnesssearch-harness
Fix:tokenizer pool race condition (#20903)
ciel7d1490c28431
diff --git a/internal/tokenizer/tokenizer.go b/internal/tokenizer/tokenizer.goindex 3124415d6b..6cf3b11749 100644--- a/internal/tokenizer/tokenizer.go+++ b/internal/tokenizer/tokenizer.go@@ -18,6 +18,7 @@ package tokenizerimport ("context"+ "errors""fmt""ragflow/internal/common""runtime"@@ -53,15 +54,23 @@ type poolInstance struct {lastUsedAt time.Time}-// analyzerPool is the elastic pool for analyzer instances+// analyzerPool is the elastic pool for analyzer instances.+//+// stateMu guards the lifecycle state: initialized, baseAnalyzer, and the+// instances channel. acquire/release/shrink hold it for read across their whole+// body; Close takes it for write to close the channel and destroy the+// analyzers. currentSize is atomic and sits outside that pair so the hot paths+// keep their existing read-modify-write.type analyzerPool struct {config PoolConfig- baseAnalyzer *rag.Analyzer // Original analyzer used as template for copying- instances chan *poolInstance // Channel-based pool for available instances- currentSize int32 // Current number of instances (atomic)- initialized bool- mu sync.RWMutex- stopCh chan struct{}+ baseAnalyzer *rag.Analyzer // guarded by stateMu+ instances chan *poolInstance // send/recv and close guarded by stateMu+ currentSize int32 // atomic+ initialized bool // guarded by stateMu+ stateMu sync.RWMutex+ stopCh chan struct{} // closed once by stopOnce to wake waiters+ stopOnce sync.Once+ borrowers sync.WaitGroup // checked-out instances not yet releasedwg sync.WaitGroup}@@ -71,110 +80,159 @@ type analyzerPool struct {// language override.const defaultLanguage = "English"+// initMu serializes Init so a single pool is built and published at a time.+// globalPool carries only fully constructed pools. The hot path loads it once+// and pins that pointer for the whole acquire/release pair, so a concurrent+// Init can never make a caller borrow from one pool and return to another.var (- globalPool *analyzerPool- poolOnce sync.Once- poolInitError error+ initMu sync.Mutex+ globalPool atomic.Pointer[analyzerPool])-// Init initializes the elastic analyzer pool with the given configuration-// Can be called multiple times if the pool was previously closed+// errPoolClosed is returned by acquire when Close is in progress and the pool+// can no longer hand out instances.+var errPoolClosed = errors.New("analyzer pool closed")++// isInitialized reports the live state under the same lock Close uses to clear+// it.+func (p *analyzerPool) isInitialized() bool {+ p.stateMu.RLock()+ defer p.stateMu.RUnlock()+ return p.initialized+}++// Init initializes the elastic analyzer pool with the given configuration. It+// is idempotent while a pool is live and builds a fresh pool after Close.func Init(cfg *PoolConfig) error {- // Check if we need to reset poolOnce (for testing or re-initialization)- if globalPool != nil && !globalPool.initialized {- // Pool was closed, reset poolOnce for re-initialization- poolOnce = sync.Once{}+ initMu.Lock()+ defer initMu.Unlock()++ if current := globalPool.Load(); current != nil && current.isInitialized() {+ return nil}- poolOnce.Do(func() {- if cfg == nil {- cfg = &PoolConfig{}- }+ if cfg == nil {+ cfg = &PoolConfig{}+ }- // Set default values- if cfg.DictPath == "" {- if env := common.GetEnv(common.EnvRAGFlowDictPath); env != "" {- cfg.DictPath = env- } else {- cfg.DictPath = "/usr/share/infinity/resource"- }- }- if cfg.MinSize <= 0 {- cfg.MinSize = runtime.NumCPU() * 2- }- if cfg.MaxSize <= 0 {- cfg.MaxSize = runtime.NumCPU() * 16- }- if cfg.MinSize > cfg.MaxSize {- cfg.MinSize = cfg.MaxSize- }- if cfg.IdleTimeout <= 0 {- cfg.IdleTimeout = 5 * time.Minute- }- if cfg.AcquireTimeout <= 0 {- cfg.AcquireTimeout = 10 * time.Second+ // Set default values+ if cfg.DictPath == "" {+ if env := common.GetEnv(common.EnvRAGFlowDictPath); env != "" {+ cfg.DictPath = env+ } else {+ cfg.DictPath = "/usr/share/infinity/resource"}+ }+ if cfg.MinSize <= 0 {+ cfg.MinSize = runtime.NumCPU() * 2+ }+ if cfg.MaxSize <= 0 {+ cfg.MaxSize = runtime.NumCPU() * 16+ }+ if cfg.MinSize > cfg.MaxSize {+ cfg.MinSize = cfg.MaxSize+ }+ if cfg.IdleTimeout <= 0 {+ cfg.IdleTimeout = 5 * time.Minute+ }+ if cfg.AcquireTimeout <= 0 {+ cfg.AcquireTimeout = 10 * time.Second+ }- // The dictionaries are reloaded below, so stale lookups must not survive.- resetTermLookupCaches()+ // The dictionaries are reloaded below, so stale lookups must not survive.+ resetTermLookupCaches()- common.Info("Initializing analyzer pool",- zap.String("dict_path", cfg.DictPath),- zap.Int("min_size", cfg.MinSize),- zap.Int("max_size", cfg.MaxSize),- zap.Duration("idle_timeout", cfg.IdleTimeout),- zap.Duration("acquire_timeout", cfg.AcquireTimeout))+ common.Info("Initializing analyzer pool",+ zap.String("dict_path", cfg.DictPath),+ zap.Int("min_size", cfg.MinSize),+ zap.Int("max_size", cfg.MaxSize),+ zap.Duration("idle_timeout", cfg.IdleTimeout),+ zap.Duration("acquire_timeout", cfg.AcquireTimeout))- globalPool = &analyzerPool{- config: *cfg,- instances: make(chan *poolInstance, cfg.MaxSize),- stopCh: make(chan struct{}),- }+ p := &analyzerPool{+ config: *cfg,+ instances: make(chan *poolInstance, cfg.MaxSize),+ stopCh: make(chan struct{}),+ }++ // Create the base analyzer as template+ baseAnalyzer, err := rag.NewAnalyzer(cfg.DictPath)+ if err != nil {+ wrapped := fmt.Errorf("failed to create base analyzer: %w", err)+ common.Error("Failed to create base analyzer", wrapped)+ return wrapped+ }++ if err = baseAnalyzer.Load(); err != nil {+ wrapped := fmt.Errorf("failed to load base analyzer: %w", err)+ common.Error("Failed to load base analyzer", wrapped)+ baseAnalyzer.Close()+ return wrapped+ }++ p.baseAnalyzer = baseAnalyzer- // Create the base analyzer as template- baseAnalyzer, err := rag.NewAnalyzer(cfg.DictPath)+ // Pre-warm minSize instances+ for i := 0; i < cfg.MinSize; i++ {+ instance, err := p.createInstance()if err != nil {- poolInitError = fmt.Errorf("failed to create base analyzer: %w", err)- common.Error("Failed to create base analyzer", poolInitError)- return+ wrapped := fmt.Errorf("failed to create instance %d: %w", i, err)+ common.Error("Failed to create pool instance", wrapped)+ p.discardUnpublished()+ return wrapped}+ p.instances <- instance+ atomic.AddInt32(&p.currentSize, 1)+ }- if err = baseAnalyzer.Load(); err != nil {- poolInitError = fmt.Errorf("failed to load base analyzer: %w", err)- common.Error("Failed to load base analyzer", poolInitError)- baseAnalyzer.Close()- return- }+ p.initialized = true- globalPool.baseAnalyzer = baseAnalyzer+ // Start the shrink loop before publishing, so no observer can see a pool+ // without its shrinker.+ p.wg.Add(1)+ go p.shrinkLoop()- // Pre-warm minSize instances- for i := 0; i < cfg.MinSize; i++ {- instance, err := globalPool.createInstance()- if err != nil {- poolInitError = fmt.Errorf("failed to create instance %d: %w", i, err)- common.Error("Failed to create pool instance", poolInitError)- globalPool.Close()- return- }- globalPool.instances <- instance- atomic.AddInt32(&globalPool.currentSize, 1)- }+ // Publish only a fully constructed pool.+ globalPool.Store(p)- globalPool.initialized = true- common.Info("Analyzer pool initialized successfully",- zap.Int("pre_warmed", cfg.MinSize),- zap.Int32("current_size", atomic.LoadInt32(&globalPool.currentSize)))+ common.Info("Analyzer pool initialized successfully",+ zap.Int("pre_warmed", cfg.MinSize),+ zap.Int32("current_size", atomic.LoadInt32(&p.currentSize)))- // Start the shrink loop for idle instance cleanup- globalPool.wg.Add(1)- go globalPool.shrinkLoop()- })+ return nil+}++// discardUnpublished releases the analyzers of a pool whose Init failed before+// the pool was ever published. initialized is still false on this path, so+// Close's idempotency guard would return before freeing anything.+func (p *analyzerPool) discardUnpublished() {+ p.drainInstances()+ if p.baseAnalyzer != nil {+ p.baseAnalyzer.Close()+ p.baseAnalyzer = nil+ }+}- return poolInitError+// drainInstances closes every analyzer currently resting in the channel. It+// drains without blocking, so it is safe whether or not the channel is closed.+func (p *analyzerPool) drainInstances() {+ for {+ select {+ case instance, ok := <-p.instances:+ if !ok {+ return+ }+ if instance != nil && instance.analyzer != nil {+ instance.analyzer.Close()+ }+ default:+ return+ }+ }}-// createInstance creates a new analyzer instance by copying the base analyzer+// createInstance creates a new analyzer instance by copying the base analyzer.+// Callers must hold stateMu for read, or be the sole initializer (Init).func (p *analyzerPool) createInstance() (*poolInstance, error) {if p.baseAnalyzer == nil {return nil, fmt.Errorf("base analyzer is nil")@@ -195,6 +253,12 @@ func (p *analyzerPool) createInstance() (*poolInstance, error) {// acquire gets an analyzer instance from the pool// If pool is empty and below max size, creates a new instance dynamicallyfunc (p *analyzerPool) acquire() (*poolInstance, error) {+ // The whole body runs under the read lock: the initialized check, the+ // channel receive, and the dynamic expansion must not interleave with+ // Close's write section, which flips initialized and closes the channel.+ p.stateMu.RLock()+ defer p.stateMu.RUnlock()+if !p.initialized {return nil, fmt.Errorf("pool not initialized")}@@ -203,6 +267,7 @@ func (p *analyzerPool) acquire() (*poolInstance, error) {select {case instance := <-p.instances:instance.lastUsedAt = time.Now()+ p.borrowers.Add(1)return instance, nildefault:}@@ -222,6 +287,7 @@ func (p *analyzerPool) acquire() (*poolInstance, error) {zap.Int32("previous_size", current),zap.Int32("new_size", current+1),zap.Int("max_size", p.config.MaxSize))+ p.borrowers.Add(1)return instance, nil}// CAS failed, another goroutine created an instance, fall through to wait@@ -234,18 +300,27 @@ func (p *analyzerPool) acquire() (*poolInstance, error) {select {case instance := <-p.instances:instance.lastUsedAt = time.Now()+ p.borrowers.Add(1)return instance, nil+ case <-p.stopCh:+ return nil, errPoolClosedcase <-ctx.Done():return nil, fmt.Errorf("timeout waiting for analyzer instance (current_size=%d, max=%d)",atomic.LoadInt32(&p.currentSize), p.config.MaxSize)}}-// release returns an analyzer instance to the pool+// release returns an analyzer instance to the pool.func (p *analyzerPool) release(instance *poolInstance) {if instance == nil || instance.analyzer == nil {return}+ defer p.borrowers.Done()++ // The initialized check and the channel send share one critical section:+ // otherwise Close could close the channel between them.+ p.stateMu.RLock()+ defer p.stateMu.RUnlock()if !p.initialized {instance.analyzer.Close()@@ -284,6 +359,9 @@ func (p *analyzerPool) shrinkLoop() {// shrink removes idle instances that have exceeded the idle timeout// while keeping at least MinSize instancesfunc (p *analyzerPool) shrink() {+ p.stateMu.RLock()+ defer p.stateMu.RUnlock()+if !p.initialized {return}@@ -336,78 +414,102 @@ func (p *analyzerPool) shrink() {}}-// Close closes the pool and releases all resources+// Close closes the pool and releases all resources. It is idempotent and safe+// to call concurrently with acquire/release.func (p *analyzerPool) Close() {if p == nil {return}- p.mu.Lock()+ // Wake blocked acquirers before requesting the write lock. Once a writer is+ // queued, RWMutex blocks new readers, so an acquire parked on <-p.instances+ // could never be fed by a release and would hold its read lock until the+ // acquire timeout. Closing stopCh makes those waiters return immediately.+ p.stopOnce.Do(func() { close(p.stopCh) })++ // Phase 1: flip initialized under the write lock. acquire/release/shrink+ // hold the read lock across their whole body, so after this no caller can+ // observe the pool as open or touch the channel.+ p.stateMu.Lock()if !p.initialized {- p.mu.Unlock()+ p.stateMu.Unlock()return}p.initialized = false- p.mu.Unlock()+ p.stateMu.Unlock()- // Signal shrink loop to stop- close(p.stopCh)+ // shrinkLoop takes stateMu.RLock inside shrink, so this wait stays outside+ // the lock; stopCh is already closed, so it returns promptly.p.wg.Wait()- // Close all instances in pool+ // A copied analyzer references the base analyzer's loaded dictionaries+ // (RAGAnalyzer_Copy shares trie_/pos_table_/wordnet_lemma_ and does not own+ // them), so the base must outlive every checked-out instance. Wait for the+ // borrowers to return before destroying it.+ p.borrowers.Wait()++ // Phase 2: close the channel and release the analyzers. The read lock keeps+ // a late release from sending here; such a release sees initialized == false+ // and closes its own instance instead. drainInstances then collects whatever+ // is still buffered.+ p.stateMu.Lock()close(p.instances)- for instance := range p.instances {- if instance != nil && instance.analyzer != nil {- instance.analyzer.Close()- }- }-- // Close base analyzer+ p.drainInstances()if p.baseAnalyzer != nil {p.baseAnalyzer.Close()p.baseAnalyzer = nil}+ p.stateMu.Unlock()common.Info(fmt.Sprintf("Analyzer pool closed, final_size: %d", atomic.LoadInt32(&p.currentSize)))}// GetPoolStats returns current pool statisticsfunc GetPoolStats() map[string]interface{} {- if globalPool == nil {+ p := globalPool.Load()+ if p == nil {return map[string]interface{}{"initialized": false,}}+ p.stateMu.RLock()+ initialized := p.initialized+ available := len(p.instances)+ p.stateMu.RUnlock()+return map[string]interface{}{- "initialized": globalPool.initialized,- "current_size": atomic.LoadInt32(&globalPool.currentSize),- "min_size": globalPool.config.MinSize,- "max_size": globalPool.config.MaxSize,- "idle_timeout": globalPool.config.IdleTimeout.String(),- "instances_available": len(globalPool.instances),+ "initialized": initialized,+ "current_size": atomic.LoadInt32(&p.currentSize),+ "min_size": p.config.MinSize,+ "max_size": p.config.MaxSize,+ "idle_timeout": p.config.IdleTimeout.String(),+ "instances_available": available,}}// Close closes the global poolfunc Close() {- if globalPool != nil {- globalPool.Close()+ if p := globalPool.Load(); p != nil {+ p.Close()}}// withAnalyzer acquires an analyzer instance, applies the given language// (with "" mapping to defaultLanguage), and executes fn.func withAnalyzer(lang string, fn func(*rag.Analyzer) error) error {- if globalPool == nil {+ // Load the pool once and use that pointer for both acquire and release, so a+ // concurrent Init cannot swap the global pool between the two.+ pool := globalPool.Load()+ if pool == nil {return fmt.Errorf("tokenizer pool not initialized")}- instance, err := globalPool.acquire()+ instance, err := pool.acquire()if err != nil {return err}- defer globalPool.release(instance)+ defer pool.release(instance)if lang == "" {lang = defaultLanguage@@ -420,15 +522,16 @@ func withAnalyzer(lang string, fn func(*rag.Analyzer) error) error {// withAnalyzerResult is the result-returning variant of withAnalyzer.func withAnalyzerResult[T any](lang string, fn func(*rag.Analyzer) (T, error)) (T, error) {var result T- if globalPool == nil {+ pool := globalPool.Load()+ if pool == nil {return result, fmt.Errorf("tokenizer pool not initialized")}- instance, err := globalPool.acquire()+ instance, err := pool.acquire()if err != nil {return result, err}- defer globalPool.release(instance)+ defer pool.release(instance)if lang == "" {lang = defaultLanguage@@ -529,7 +632,8 @@ func SetEnablePosition(enablePosition bool) {// IsInitialized checks whether the tokenizer pool has been initializedfunc IsInitialized() bool {- return globalPool != nil && globalPool.initialized+ p := globalPool.Load()+ return p != nil && p.isInitialized()}// termLookupCacheMax caps each memo below. Reaching it rotates the memo insteaddiff --git a/internal/tokenizer/tokenizer_close_race_test.go b/internal/tokenizer/tokenizer_close_race_test.gonew file mode 100644index 0000000000..e53445c491--- /dev/null+++ b/internal/tokenizer/tokenizer_close_race_test.go@@ -0,0 +1,299 @@+//+// Copyright 2026 The InfiniFlow Authors. All Rights Reserved.+//+// Licensed under the Apache License, Version 2.0 (the "License");+// you may not use this file except in compliance with the License.+// You may obtain a copy of the License at+//+// http://www.apache.org/licenses/LICENSE-2.0+//+// Unless required by applicable law or agreed to in writing, software+// distributed under the License is distributed on an "AS IS" BASIS,+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+// See the License for the specific language governing permissions and+// limitations under the License.+//++package tokenizer++import (+ "os"+ "path/filepath"+ "sync"+ "testing"+ "time"++ "ragflow/internal/common"+)++// prepareCloseRaceTest pins the engine type for the test, restores it+// afterwards, and skips when the analyzer dictionary is not present. The+// dictionary is a local asset rather than a service, but the unit tier must+// still run on a machine that has not downloaded it.+func prepareCloseRaceTest(t *testing.T) {+ t.Helper()++ original := engineType+ engineType = ""+ t.Cleanup(func() { engineType = original })++ dir := os.Getenv(common.EnvRAGFlowDictPath)+ if dir == "" {+ dir = "/usr/share/infinity/resource"+ }+ if _, err := os.Stat(filepath.Join(dir, "rag", "huqie.trie")); err != nil {+ t.Skipf("analyzer dictionary not present at %s (%v); run ragflow_deps/download_deps.py", dir, err)+ }+}++// smallPoolConfig keeps the pool at one pre-warmed slot with a generous acquire+// timeout, so an acquire that cannot be served parks on the blocking path+// instead of timing out.+func smallPoolConfig() *PoolConfig {+ return &PoolConfig{+ DictPath: "",+ MinSize: 1,+ MaxSize: 1,+ IdleTimeout: 5 * time.Second,+ AcquireTimeout: 30 * time.Second,+ }+}++// waitPoolClosed blocks until p reports not initialized, so a caller can release+// a checked-out instance after Close has flipped the pool but while Close is+// still waiting for that release.+func waitPoolClosed(t *testing.T, p *analyzerPool) {+ t.Helper()++ deadline := time.Now().Add(2 * time.Second)+ for p.isInitialized() {+ if time.Now().After(deadline) {+ t.Fatal("Close did not flip the pool to closed within 2s")+ }+ time.Sleep(time.Millisecond)+ }+}++// A blocked acquire must wake up and return an error when Close runs, Close must+// wait for the checked-out instances, and it must return promptly once the last+// one is released. On the unsynchronized pool, close(instances) turns the parked+// receive into a nil instance and panics on the next field write, and the late+// release sends on the closed channel.+func TestAcquireUnblocksOnCloseCompletion(t *testing.T) {+ prepareCloseRaceTest(t)++ if err := Init(smallPoolConfig()); err != nil {+ t.Fatalf("Init: %v", err)+ }+ defer Close()++ p := globalPool.Load()+ if p == nil {+ t.Fatal("globalPool is nil after Init")+ }++ // Hold the only slot so the next acquire is forced onto the blocking path.+ held, err := p.acquire()+ if err != nil {+ t.Fatalf("first acquire: %v", err)+ }++ done := make(chan error, 1)+ started := make(chan struct{})+ go func() {+ close(started)+ _, err := p.acquire()+ done <- err+ }()++ <-started+ // The pool is empty and at MaxSize, so the worker exhausts the fast path and+ // the dynamic-expansion branch and parks in the timeout select. Give it a+ // moment to get there before Close.+ time.Sleep(50 * time.Millisecond)++ closeDone := make(chan time.Duration, 1)+ go func() {+ begin := time.Now()+ p.Close()+ closeDone <- time.Since(begin)+ }()++ select {+ case err := <-done:+ if err == nil {+ t.Fatal("parked acquire returned a nil error after Close")+ }+ case <-time.After(2 * time.Second):+ t.Fatal("parked acquire did not wake within 2s of Close")+ }++ // Close is waiting for this instance; releasing it must not send on the+ // closed channel and must let Close finish.+ p.release(held)++ select {+ case elapsed := <-closeDone:+ if elapsed > 2*time.Second {+ t.Fatalf("Close took %v after the last borrower was released", elapsed)+ }+ case <-time.After(2 * time.Second):+ t.Fatal("Close did not return after the last borrower was released")+ }+}++// Releasing a checked-out instance after Close has flipped the pool must close+// that instance rather than send on the closed channel.+func TestReleaseAfterCloseSelfClosesInstance(t *testing.T) {+ prepareCloseRaceTest(t)++ if err := Init(smallPoolConfig()); err != nil {+ t.Fatalf("Init: %v", err)+ }++ p := globalPool.Load()+ held, err := p.acquire()+ if err != nil {+ t.Fatalf("acquire: %v", err)+ }++ closeDone := make(chan struct{})+ go func() {+ p.Close()+ close(closeDone)+ }()++ waitPoolClosed(t, p)++ p.release(held) // must not panic+ <-closeDone+}++// Concurrent Close calls must close the underlying channel exactly once and all+// return once the last borrower is released.+func TestConcurrentCloseIsIdempotent(t *testing.T) {+ prepareCloseRaceTest(t)++ if err := Init(&PoolConfig{+ DictPath: "",+ MinSize: 1,+ MaxSize: 4,+ IdleTimeout: 5 * time.Second,+ AcquireTimeout: 5 * time.Second,+ }); err != nil {+ t.Fatalf("Init: %v", err)+ }++ p := globalPool.Load()+ held, err := p.acquire()+ if err != nil {+ t.Fatalf("acquire: %v", err)+ }++ const closers = 8+ var wg sync.WaitGroup+ wg.Add(closers)+ for i := 0; i < closers; i++ {+ go func() {+ defer wg.Done()+ defer func() {+ if r := recover(); r != nil {+ t.Errorf("concurrent Close panicked: %v", r)+ }+ }()+ p.Close()+ }()+ }++ waitPoolClosed(t, p)+ p.release(held)+ wg.Wait()++ if IsInitialized() {+ t.Error("pool still reports initialized after concurrent Close")+ }+}++// Init must be able to rebuild a pool that a previous Close shut down.+func TestInitAfterCloseRebuildsPool(t *testing.T) {+ prepareCloseRaceTest(t)++ if err := Init(smallPoolConfig()); err != nil {+ t.Fatalf("Init: %v", err)+ }+ if !IsInitialized() {+ t.Fatal("pool not initialized after Init")+ }++ Close()+ if IsInitialized() {+ t.Fatal("pool still reports initialized after Close")+ }++ if err := Init(smallPoolConfig()); err != nil {+ t.Fatalf("re-Init after Close: %v", err)+ }+ defer Close()++ if !IsInitialized() {+ t.Fatal("pool not initialized after re-Init")+ }+ if _, err := New("").Tokenize("hello world"); err != nil {+ t.Fatalf("Tokenize after re-Init: %v", err)+ }+}++// Hammer tokenization from many goroutines while Close runs. Every error is+// tolerated; the assertions are that nothing panics and, under -race, that the+// lifecycle state is not read and written concurrently. Close must also wait for+// the in-flight tokenizations before it destroys the shared dictionaries.+func TestTokenizeCloseRaceStress(t *testing.T) {+ prepareCloseRaceTest(t)++ if err := Init(&PoolConfig{+ DictPath: "",+ MinSize: 2,+ MaxSize: 8,+ IdleTimeout: 50 * time.Millisecond,+ AcquireTimeout: 100 * time.Millisecond,+ }); err != nil {+ t.Fatalf("Init: %v", err)+ }++ stop := make(chan struct{})+ var wg sync.WaitGroup+ const workers = 16+ wg.Add(workers)+ for i := 0; i < workers; i++ {+ go func() {+ defer wg.Done()+ defer func() {+ if r := recover(); r != nil {+ t.Errorf("Tokenize panicked during Close: %v", r)+ }+ }()+ for {+ select {+ case <-stop:+ return+ default:+ }+ _, _ = New("").Tokenize("hello world 测试")+ }+ }()+ }++ time.Sleep(20 * time.Millisecond)+ // Close while the workers are still hammering the pool. Close must wait for+ // the in-flight tokenizations to return their instances before it destroys+ // the base analyzer whose dictionaries they share.+ Close()+ close(stop)+ wg.Wait()++ // The pool must be usable again after the close storm.+ if err := Init(smallPoolConfig()); err != nil {+ t.Fatalf("re-Init after close storm: %v", err)+ }+ Close()+}