Skip to content

RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.

Fix:tokenizer pool race condition (#20903)

ciel7d1490c28431
diff --git a/internal/tokenizer/tokenizer.go b/internal/tokenizer/tokenizer.go
index 3124415d6b..6cf3b11749 100644
--- a/internal/tokenizer/tokenizer.go
+++ b/internal/tokenizer/tokenizer.go
@@ -18,6 +18,7 @@ package tokenizer
import (
"context"
+ "errors"
"fmt"
"ragflow/internal/common"
"runtime"
@@ -53,15 +54,23 @@ type poolInstance struct {
lastUsedAt time.Time
}
-// analyzerPool is the elastic pool for analyzer instances
+// analyzerPool is the elastic pool for analyzer instances.
+//
+// stateMu guards the lifecycle state: initialized, baseAnalyzer, and the
+// instances channel. acquire/release/shrink hold it for read across their whole
+// body; Close takes it for write to close the channel and destroy the
+// analyzers. currentSize is atomic and sits outside that pair so the hot paths
+// keep their existing read-modify-write.
type analyzerPool struct {
config PoolConfig
- baseAnalyzer *rag.Analyzer // Original analyzer used as template for copying
- instances chan *poolInstance // Channel-based pool for available instances
- currentSize int32 // Current number of instances (atomic)
- initialized bool
- mu sync.RWMutex
- stopCh chan struct{}
+ baseAnalyzer *rag.Analyzer // guarded by stateMu
+ instances chan *poolInstance // send/recv and close guarded by stateMu
+ currentSize int32 // atomic
+ initialized bool // guarded by stateMu
+ stateMu sync.RWMutex
+ stopCh chan struct{} // closed once by stopOnce to wake waiters
+ stopOnce sync.Once
+ borrowers sync.WaitGroup // checked-out instances not yet released
wg sync.WaitGroup
}
@@ -71,110 +80,159 @@ type analyzerPool struct {
// language override.
const defaultLanguage = "English"
+// initMu serializes Init so a single pool is built and published at a time.
+// globalPool carries only fully constructed pools. The hot path loads it once
+// and pins that pointer for the whole acquire/release pair, so a concurrent
+// Init can never make a caller borrow from one pool and return to another.
var (
- globalPool *analyzerPool
- poolOnce sync.Once
- poolInitError error
+ initMu sync.Mutex
+ globalPool atomic.Pointer[analyzerPool]
)
-// Init initializes the elastic analyzer pool with the given configuration
-// Can be called multiple times if the pool was previously closed
+// errPoolClosed is returned by acquire when Close is in progress and the pool
+// can no longer hand out instances.
+var errPoolClosed = errors.New("analyzer pool closed")
+
+// isInitialized reports the live state under the same lock Close uses to clear
+// it.
+func (p *analyzerPool) isInitialized() bool {
+ p.stateMu.RLock()
+ defer p.stateMu.RUnlock()
+ return p.initialized
+}
+
+// Init initializes the elastic analyzer pool with the given configuration. It
+// is idempotent while a pool is live and builds a fresh pool after Close.
func Init(cfg *PoolConfig) error {
- // Check if we need to reset poolOnce (for testing or re-initialization)
- if globalPool != nil && !globalPool.initialized {
- // Pool was closed, reset poolOnce for re-initialization
- poolOnce = sync.Once{}
+ initMu.Lock()
+ defer initMu.Unlock()
+
+ if current := globalPool.Load(); current != nil && current.isInitialized() {
+ return nil
}
- poolOnce.Do(func() {
- if cfg == nil {
- cfg = &PoolConfig{}
- }
+ if cfg == nil {
+ cfg = &PoolConfig{}
+ }
- // Set default values
- if cfg.DictPath == "" {
- if env := common.GetEnv(common.EnvRAGFlowDictPath); env != "" {
- cfg.DictPath = env
- } else {
- cfg.DictPath = "/usr/share/infinity/resource"
- }
- }
- if cfg.MinSize <= 0 {
- cfg.MinSize = runtime.NumCPU() * 2
- }
- if cfg.MaxSize <= 0 {
- cfg.MaxSize = runtime.NumCPU() * 16
- }
- if cfg.MinSize > cfg.MaxSize {
- cfg.MinSize = cfg.MaxSize
- }
- if cfg.IdleTimeout <= 0 {
- cfg.IdleTimeout = 5 * time.Minute
- }
- if cfg.AcquireTimeout <= 0 {
- cfg.AcquireTimeout = 10 * time.Second
+ // Set default values
+ if cfg.DictPath == "" {
+ if env := common.GetEnv(common.EnvRAGFlowDictPath); env != "" {
+ cfg.DictPath = env
+ } else {
+ cfg.DictPath = "/usr/share/infinity/resource"
}
+ }
+ if cfg.MinSize <= 0 {
+ cfg.MinSize = runtime.NumCPU() * 2
+ }
+ if cfg.MaxSize <= 0 {
+ cfg.MaxSize = runtime.NumCPU() * 16
+ }
+ if cfg.MinSize > cfg.MaxSize {
+ cfg.MinSize = cfg.MaxSize
+ }
+ if cfg.IdleTimeout <= 0 {
+ cfg.IdleTimeout = 5 * time.Minute
+ }
+ if cfg.AcquireTimeout <= 0 {
+ cfg.AcquireTimeout = 10 * time.Second
+ }
- // The dictionaries are reloaded below, so stale lookups must not survive.
- resetTermLookupCaches()
+ // The dictionaries are reloaded below, so stale lookups must not survive.
+ resetTermLookupCaches()
- common.Info("Initializing analyzer pool",
- zap.String("dict_path", cfg.DictPath),
- zap.Int("min_size", cfg.MinSize),
- zap.Int("max_size", cfg.MaxSize),
- zap.Duration("idle_timeout", cfg.IdleTimeout),
- zap.Duration("acquire_timeout", cfg.AcquireTimeout))
+ common.Info("Initializing analyzer pool",
+ zap.String("dict_path", cfg.DictPath),
+ zap.Int("min_size", cfg.MinSize),
+ zap.Int("max_size", cfg.MaxSize),
+ zap.Duration("idle_timeout", cfg.IdleTimeout),
+ zap.Duration("acquire_timeout", cfg.AcquireTimeout))
- globalPool = &analyzerPool{
- config: *cfg,
- instances: make(chan *poolInstance, cfg.MaxSize),
- stopCh: make(chan struct{}),
- }
+ p := &analyzerPool{
+ config: *cfg,
+ instances: make(chan *poolInstance, cfg.MaxSize),
+ stopCh: make(chan struct{}),
+ }
+
+ // Create the base analyzer as template
+ baseAnalyzer, err := rag.NewAnalyzer(cfg.DictPath)
+ if err != nil {
+ wrapped := fmt.Errorf("failed to create base analyzer: %w", err)
+ common.Error("Failed to create base analyzer", wrapped)
+ return wrapped
+ }
+
+ if err = baseAnalyzer.Load(); err != nil {
+ wrapped := fmt.Errorf("failed to load base analyzer: %w", err)
+ common.Error("Failed to load base analyzer", wrapped)
+ baseAnalyzer.Close()
+ return wrapped
+ }
+
+ p.baseAnalyzer = baseAnalyzer
- // Create the base analyzer as template
- baseAnalyzer, err := rag.NewAnalyzer(cfg.DictPath)
+ // Pre-warm minSize instances
+ for i := 0; i < cfg.MinSize; i++ {
+ instance, err := p.createInstance()
if err != nil {
- poolInitError = fmt.Errorf("failed to create base analyzer: %w", err)
- common.Error("Failed to create base analyzer", poolInitError)
- return
+ wrapped := fmt.Errorf("failed to create instance %d: %w", i, err)
+ common.Error("Failed to create pool instance", wrapped)
+ p.discardUnpublished()
+ return wrapped
}
+ p.instances <- instance
+ atomic.AddInt32(&p.currentSize, 1)
+ }
- if err = baseAnalyzer.Load(); err != nil {
- poolInitError = fmt.Errorf("failed to load base analyzer: %w", err)
- common.Error("Failed to load base analyzer", poolInitError)
- baseAnalyzer.Close()
- return
- }
+ p.initialized = true
- globalPool.baseAnalyzer = baseAnalyzer
+ // Start the shrink loop before publishing, so no observer can see a pool
+ // without its shrinker.
+ p.wg.Add(1)
+ go p.shrinkLoop()
- // Pre-warm minSize instances
- for i := 0; i < cfg.MinSize; i++ {
- instance, err := globalPool.createInstance()
- if err != nil {
- poolInitError = fmt.Errorf("failed to create instance %d: %w", i, err)
- common.Error("Failed to create pool instance", poolInitError)
- globalPool.Close()
- return
- }
- globalPool.instances <- instance
- atomic.AddInt32(&globalPool.currentSize, 1)
- }
+ // Publish only a fully constructed pool.
+ globalPool.Store(p)
- globalPool.initialized = true
- common.Info("Analyzer pool initialized successfully",
- zap.Int("pre_warmed", cfg.MinSize),
- zap.Int32("current_size", atomic.LoadInt32(&globalPool.currentSize)))
+ common.Info("Analyzer pool initialized successfully",
+ zap.Int("pre_warmed", cfg.MinSize),
+ zap.Int32("current_size", atomic.LoadInt32(&p.currentSize)))
- // Start the shrink loop for idle instance cleanup
- globalPool.wg.Add(1)
- go globalPool.shrinkLoop()
- })
+ return nil
+}
+
+// discardUnpublished releases the analyzers of a pool whose Init failed before
+// the pool was ever published. initialized is still false on this path, so
+// Close's idempotency guard would return before freeing anything.
+func (p *analyzerPool) discardUnpublished() {
+ p.drainInstances()
+ if p.baseAnalyzer != nil {
+ p.baseAnalyzer.Close()
+ p.baseAnalyzer = nil
+ }
+}
- return poolInitError
+// drainInstances closes every analyzer currently resting in the channel. It
+// drains without blocking, so it is safe whether or not the channel is closed.
+func (p *analyzerPool) drainInstances() {
+ for {
+ select {
+ case instance, ok := <-p.instances:
+ if !ok {
+ return
+ }
+ if instance != nil && instance.analyzer != nil {
+ instance.analyzer.Close()
+ }
+ default:
+ return
+ }
+ }
}
-// createInstance creates a new analyzer instance by copying the base analyzer
+// createInstance creates a new analyzer instance by copying the base analyzer.
+// Callers must hold stateMu for read, or be the sole initializer (Init).
func (p *analyzerPool) createInstance() (*poolInstance, error) {
if p.baseAnalyzer == nil {
return nil, fmt.Errorf("base analyzer is nil")
@@ -195,6 +253,12 @@ func (p *analyzerPool) createInstance() (*poolInstance, error) {
// acquire gets an analyzer instance from the pool
// If pool is empty and below max size, creates a new instance dynamically
func (p *analyzerPool) acquire() (*poolInstance, error) {
+ // The whole body runs under the read lock: the initialized check, the
+ // channel receive, and the dynamic expansion must not interleave with
+ // Close's write section, which flips initialized and closes the channel.
+ p.stateMu.RLock()
+ defer p.stateMu.RUnlock()
+
if !p.initialized {
return nil, fmt.Errorf("pool not initialized")
}
@@ -203,6 +267,7 @@ func (p *analyzerPool) acquire() (*poolInstance, error) {
select {
case instance := <-p.instances:
instance.lastUsedAt = time.Now()
+ p.borrowers.Add(1)
return instance, nil
default:
}
@@ -222,6 +287,7 @@ func (p *analyzerPool) acquire() (*poolInstance, error) {
zap.Int32("previous_size", current),
zap.Int32("new_size", current+1),
zap.Int("max_size", p.config.MaxSize))
+ p.borrowers.Add(1)
return instance, nil
}
// CAS failed, another goroutine created an instance, fall through to wait
@@ -234,18 +300,27 @@ func (p *analyzerPool) acquire() (*poolInstance, error) {
select {
case instance := <-p.instances:
instance.lastUsedAt = time.Now()
+ p.borrowers.Add(1)
return instance, nil
+ case <-p.stopCh:
+ return nil, errPoolClosed
case <-ctx.Done():
return nil, fmt.Errorf("timeout waiting for analyzer instance (current_size=%d, max=%d)",
atomic.LoadInt32(&p.currentSize), p.config.MaxSize)
}
}
-// release returns an analyzer instance to the pool
+// release returns an analyzer instance to the pool.
func (p *analyzerPool) release(instance *poolInstance) {
if instance == nil || instance.analyzer == nil {
return
}
+ defer p.borrowers.Done()
+
+ // The initialized check and the channel send share one critical section:
+ // otherwise Close could close the channel between them.
+ p.stateMu.RLock()
+ defer p.stateMu.RUnlock()
if !p.initialized {
instance.analyzer.Close()
@@ -284,6 +359,9 @@ func (p *analyzerPool) shrinkLoop() {
// shrink removes idle instances that have exceeded the idle timeout
// while keeping at least MinSize instances
func (p *analyzerPool) shrink() {
+ p.stateMu.RLock()
+ defer p.stateMu.RUnlock()
+
if !p.initialized {
return
}
@@ -336,78 +414,102 @@ func (p *analyzerPool) shrink() {
}
}
-// Close closes the pool and releases all resources
+// Close closes the pool and releases all resources. It is idempotent and safe
+// to call concurrently with acquire/release.
func (p *analyzerPool) Close() {
if p == nil {
return
}
- p.mu.Lock()
+ // Wake blocked acquirers before requesting the write lock. Once a writer is
+ // queued, RWMutex blocks new readers, so an acquire parked on <-p.instances
+ // could never be fed by a release and would hold its read lock until the
+ // acquire timeout. Closing stopCh makes those waiters return immediately.
+ p.stopOnce.Do(func() { close(p.stopCh) })
+
+ // Phase 1: flip initialized under the write lock. acquire/release/shrink
+ // hold the read lock across their whole body, so after this no caller can
+ // observe the pool as open or touch the channel.
+ p.stateMu.Lock()
if !p.initialized {
- p.mu.Unlock()
+ p.stateMu.Unlock()
return
}
p.initialized = false
- p.mu.Unlock()
+ p.stateMu.Unlock()
- // Signal shrink loop to stop
- close(p.stopCh)
+ // shrinkLoop takes stateMu.RLock inside shrink, so this wait stays outside
+ // the lock; stopCh is already closed, so it returns promptly.
p.wg.Wait()
- // Close all instances in pool
+ // A copied analyzer references the base analyzer's loaded dictionaries
+ // (RAGAnalyzer_Copy shares trie_/pos_table_/wordnet_lemma_ and does not own
+ // them), so the base must outlive every checked-out instance. Wait for the
+ // borrowers to return before destroying it.
+ p.borrowers.Wait()
+
+ // Phase 2: close the channel and release the analyzers. The read lock keeps
+ // a late release from sending here; such a release sees initialized == false
+ // and closes its own instance instead. drainInstances then collects whatever
+ // is still buffered.
+ p.stateMu.Lock()
close(p.instances)
- for instance := range p.instances {
- if instance != nil && instance.analyzer != nil {
- instance.analyzer.Close()
- }
- }
-
- // Close base analyzer
+ p.drainInstances()
if p.baseAnalyzer != nil {
p.baseAnalyzer.Close()
p.baseAnalyzer = nil
}
+ p.stateMu.Unlock()
common.Info(fmt.Sprintf("Analyzer pool closed, final_size: %d", atomic.LoadInt32(&p.currentSize)))
}
// GetPoolStats returns current pool statistics
func GetPoolStats() map[string]interface{} {
- if globalPool == nil {
+ p := globalPool.Load()
+ if p == nil {
return map[string]interface{}{
"initialized": false,
}
}
+ p.stateMu.RLock()
+ initialized := p.initialized
+ available := len(p.instances)
+ p.stateMu.RUnlock()
+
return map[string]interface{}{
- "initialized": globalPool.initialized,
- "current_size": atomic.LoadInt32(&globalPool.currentSize),
- "min_size": globalPool.config.MinSize,
- "max_size": globalPool.config.MaxSize,
- "idle_timeout": globalPool.config.IdleTimeout.String(),
- "instances_available": len(globalPool.instances),
+ "initialized": initialized,
+ "current_size": atomic.LoadInt32(&p.currentSize),
+ "min_size": p.config.MinSize,
+ "max_size": p.config.MaxSize,
+ "idle_timeout": p.config.IdleTimeout.String(),
+ "instances_available": available,
}
}
// Close closes the global pool
func Close() {
- if globalPool != nil {
- globalPool.Close()
+ if p := globalPool.Load(); p != nil {
+ p.Close()
}
}
// withAnalyzer acquires an analyzer instance, applies the given language
// (with "" mapping to defaultLanguage), and executes fn.
func withAnalyzer(lang string, fn func(*rag.Analyzer) error) error {
- if globalPool == nil {
+ // Load the pool once and use that pointer for both acquire and release, so a
+ // concurrent Init cannot swap the global pool between the two.
+ pool := globalPool.Load()
+ if pool == nil {
return fmt.Errorf("tokenizer pool not initialized")
}
- instance, err := globalPool.acquire()
+ instance, err := pool.acquire()
if err != nil {
return err
}
- defer globalPool.release(instance)
+ defer pool.release(instance)
if lang == "" {
lang = defaultLanguage
@@ -420,15 +522,16 @@ func withAnalyzer(lang string, fn func(*rag.Analyzer) error) error {
// withAnalyzerResult is the result-returning variant of withAnalyzer.
func withAnalyzerResult[T any](lang string, fn func(*rag.Analyzer) (T, error)) (T, error) {
var result T
- if globalPool == nil {
+ pool := globalPool.Load()
+ if pool == nil {
return result, fmt.Errorf("tokenizer pool not initialized")
}
- instance, err := globalPool.acquire()
+ instance, err := pool.acquire()
if err != nil {
return result, err
}
- defer globalPool.release(instance)
+ defer pool.release(instance)
if lang == "" {
lang = defaultLanguage
@@ -529,7 +632,8 @@ func SetEnablePosition(enablePosition bool) {
// IsInitialized checks whether the tokenizer pool has been initialized
func IsInitialized() bool {
- return globalPool != nil && globalPool.initialized
+ p := globalPool.Load()
+ return p != nil && p.isInitialized()
}
// termLookupCacheMax caps each memo below. Reaching it rotates the memo instead
diff --git a/internal/tokenizer/tokenizer_close_race_test.go b/internal/tokenizer/tokenizer_close_race_test.go
new file mode 100644
index 0000000000..e53445c491
--- /dev/null
+++ b/internal/tokenizer/tokenizer_close_race_test.go
@@ -0,0 +1,299 @@
+//
+// Copyright 2026 The InfiniFlow Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+
+package tokenizer
+
+import (
+ "os"
+ "path/filepath"
+ "sync"
+ "testing"
+ "time"
+
+ "ragflow/internal/common"
+)
+
+// prepareCloseRaceTest pins the engine type for the test, restores it
+// afterwards, and skips when the analyzer dictionary is not present. The
+// dictionary is a local asset rather than a service, but the unit tier must
+// still run on a machine that has not downloaded it.
+func prepareCloseRaceTest(t *testing.T) {
+ t.Helper()
+
+ original := engineType
+ engineType = ""
+ t.Cleanup(func() { engineType = original })
+
+ dir := os.Getenv(common.EnvRAGFlowDictPath)
+ if dir == "" {
+ dir = "/usr/share/infinity/resource"
+ }
+ if _, err := os.Stat(filepath.Join(dir, "rag", "huqie.trie")); err != nil {
+ t.Skipf("analyzer dictionary not present at %s (%v); run ragflow_deps/download_deps.py", dir, err)
+ }
+}
+
+// smallPoolConfig keeps the pool at one pre-warmed slot with a generous acquire
+// timeout, so an acquire that cannot be served parks on the blocking path
+// instead of timing out.
+func smallPoolConfig() *PoolConfig {
+ return &PoolConfig{
+ DictPath: "",
+ MinSize: 1,
+ MaxSize: 1,
+ IdleTimeout: 5 * time.Second,
+ AcquireTimeout: 30 * time.Second,
+ }
+}
+
+// waitPoolClosed blocks until p reports not initialized, so a caller can release
+// a checked-out instance after Close has flipped the pool but while Close is
+// still waiting for that release.
+func waitPoolClosed(t *testing.T, p *analyzerPool) {
+ t.Helper()
+
+ deadline := time.Now().Add(2 * time.Second)
+ for p.isInitialized() {
+ if time.Now().After(deadline) {
+ t.Fatal("Close did not flip the pool to closed within 2s")
+ }
+ time.Sleep(time.Millisecond)
+ }
+}
+
+// A blocked acquire must wake up and return an error when Close runs, Close must
+// wait for the checked-out instances, and it must return promptly once the last
+// one is released. On the unsynchronized pool, close(instances) turns the parked
+// receive into a nil instance and panics on the next field write, and the late
+// release sends on the closed channel.
+func TestAcquireUnblocksOnCloseCompletion(t *testing.T) {
+ prepareCloseRaceTest(t)
+
+ if err := Init(smallPoolConfig()); err != nil {
+ t.Fatalf("Init: %v", err)
+ }
+ defer Close()
+
+ p := globalPool.Load()
+ if p == nil {
+ t.Fatal("globalPool is nil after Init")
+ }
+
+ // Hold the only slot so the next acquire is forced onto the blocking path.
+ held, err := p.acquire()
+ if err != nil {
+ t.Fatalf("first acquire: %v", err)
+ }
+
+ done := make(chan error, 1)
+ started := make(chan struct{})
+ go func() {
+ close(started)
+ _, err := p.acquire()
+ done <- err
+ }()
+
+ <-started
+ // The pool is empty and at MaxSize, so the worker exhausts the fast path and
+ // the dynamic-expansion branch and parks in the timeout select. Give it a
+ // moment to get there before Close.
+ time.Sleep(50 * time.Millisecond)
+
+ closeDone := make(chan time.Duration, 1)
+ go func() {
+ begin := time.Now()
+ p.Close()
+ closeDone <- time.Since(begin)
+ }()
+
+ select {
+ case err := <-done:
+ if err == nil {
+ t.Fatal("parked acquire returned a nil error after Close")
+ }
+ case <-time.After(2 * time.Second):
+ t.Fatal("parked acquire did not wake within 2s of Close")
+ }
+
+ // Close is waiting for this instance; releasing it must not send on the
+ // closed channel and must let Close finish.
+ p.release(held)
+
+ select {
+ case elapsed := <-closeDone:
+ if elapsed > 2*time.Second {
+ t.Fatalf("Close took %v after the last borrower was released", elapsed)
+ }
+ case <-time.After(2 * time.Second):
+ t.Fatal("Close did not return after the last borrower was released")
+ }
+}
+
+// Releasing a checked-out instance after Close has flipped the pool must close
+// that instance rather than send on the closed channel.
+func TestReleaseAfterCloseSelfClosesInstance(t *testing.T) {
+ prepareCloseRaceTest(t)
+
+ if err := Init(smallPoolConfig()); err != nil {
+ t.Fatalf("Init: %v", err)
+ }
+
+ p := globalPool.Load()
+ held, err := p.acquire()
+ if err != nil {
+ t.Fatalf("acquire: %v", err)
+ }
+
+ closeDone := make(chan struct{})
+ go func() {
+ p.Close()
+ close(closeDone)
+ }()
+
+ waitPoolClosed(t, p)
+
+ p.release(held) // must not panic
+ <-closeDone
+}
+
+// Concurrent Close calls must close the underlying channel exactly once and all
+// return once the last borrower is released.
+func TestConcurrentCloseIsIdempotent(t *testing.T) {
+ prepareCloseRaceTest(t)
+
+ if err := Init(&PoolConfig{
+ DictPath: "",
+ MinSize: 1,
+ MaxSize: 4,
+ IdleTimeout: 5 * time.Second,
+ AcquireTimeout: 5 * time.Second,
+ }); err != nil {
+ t.Fatalf("Init: %v", err)
+ }
+
+ p := globalPool.Load()
+ held, err := p.acquire()
+ if err != nil {
+ t.Fatalf("acquire: %v", err)
+ }
+
+ const closers = 8
+ var wg sync.WaitGroup
+ wg.Add(closers)
+ for i := 0; i < closers; i++ {
+ go func() {
+ defer wg.Done()
+ defer func() {
+ if r := recover(); r != nil {
+ t.Errorf("concurrent Close panicked: %v", r)
+ }
+ }()
+ p.Close()
+ }()
+ }
+
+ waitPoolClosed(t, p)
+ p.release(held)
+ wg.Wait()
+
+ if IsInitialized() {
+ t.Error("pool still reports initialized after concurrent Close")
+ }
+}
+
+// Init must be able to rebuild a pool that a previous Close shut down.
+func TestInitAfterCloseRebuildsPool(t *testing.T) {
+ prepareCloseRaceTest(t)
+
+ if err := Init(smallPoolConfig()); err != nil {
+ t.Fatalf("Init: %v", err)
+ }
+ if !IsInitialized() {
+ t.Fatal("pool not initialized after Init")
+ }
+
+ Close()
+ if IsInitialized() {
+ t.Fatal("pool still reports initialized after Close")
+ }
+
+ if err := Init(smallPoolConfig()); err != nil {
+ t.Fatalf("re-Init after Close: %v", err)
+ }
+ defer Close()
+
+ if !IsInitialized() {
+ t.Fatal("pool not initialized after re-Init")
+ }
+ if _, err := New("").Tokenize("hello world"); err != nil {
+ t.Fatalf("Tokenize after re-Init: %v", err)
+ }
+}
+
+// Hammer tokenization from many goroutines while Close runs. Every error is
+// tolerated; the assertions are that nothing panics and, under -race, that the
+// lifecycle state is not read and written concurrently. Close must also wait for
+// the in-flight tokenizations before it destroys the shared dictionaries.
+func TestTokenizeCloseRaceStress(t *testing.T) {
+ prepareCloseRaceTest(t)
+
+ if err := Init(&PoolConfig{
+ DictPath: "",
+ MinSize: 2,
+ MaxSize: 8,
+ IdleTimeout: 50 * time.Millisecond,
+ AcquireTimeout: 100 * time.Millisecond,
+ }); err != nil {
+ t.Fatalf("Init: %v", err)
+ }
+
+ stop := make(chan struct{})
+ var wg sync.WaitGroup
+ const workers = 16
+ wg.Add(workers)
+ for i := 0; i < workers; i++ {
+ go func() {
+ defer wg.Done()
+ defer func() {
+ if r := recover(); r != nil {
+ t.Errorf("Tokenize panicked during Close: %v", r)
+ }
+ }()
+ for {
+ select {
+ case <-stop:
+ return
+ default:
+ }
+ _, _ = New("").Tokenize("hello world 测试")
+ }
+ }()
+ }
+
+ time.Sleep(20 * time.Millisecond)
+ // Close while the workers are still hammering the pool. Close must wait for
+ // the in-flight tokenizations to return their instances before it destroys
+ // the base analyzer whose dictionaries they share.
+ Close()
+ close(stop)
+ wg.Wait()
+
+ // The pool must be usable again after the close storm.
+ if err := Init(smallPoolConfig()); err != nil {
+ t.Fatalf("re-Init after close storm: %v", err)
+ }
+ Close()
+}