RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.
Go agent-harnessagent-native-databaseagentic-aiagentic-retrievalagentic-searchaiai-agentscontext-enginecontext-engineeringcontext-managementharness-engineeringknowledge-compilationragretrieval-augmented-generationretrieval-harnesssearch-harness
WorkerPool resize overspawns after shrinking an idle pool (#20490)
Taranum Wasu1826402ada63
diff --git a/internal/utility/workerpool.go b/internal/utility/workerpool.goindex 1ad429abef..fa75ea83f2 100644--- a/internal/utility/workerpool.go+++ b/internal/utility/workerpool.go@@ -125,14 +125,17 @@ func NewWorkerPool[T any, R any](workers, queueSize int, handler WorkerPoolHandlfunc (p *WorkerPool[T, R]) start(workers int) {for range workers {p.workerWg.Add(1)+ atomic.AddInt64(&p.liveWorkers, 1)go p.worker()}}func (p *WorkerPool[T, R]) worker() {- atomic.AddInt64(&p.liveWorkers, 1)+ retired := falsedefer func() {- atomic.AddInt64(&p.liveWorkers, -1)+ if !retired {+ atomic.AddInt64(&p.liveWorkers, -1)+ }p.workerWg.Done()}()@@ -164,9 +167,14 @@ func (p *WorkerPool[T, R]) worker() {}p.markDone()+ p.mu.Lock()if atomic.LoadInt64(&p.liveWorkers) > atomic.LoadInt64(&p.desiredWorkers) {+ atomic.AddInt64(&p.liveWorkers, -1)+ retired = true+ p.mu.Unlock()return}+ p.mu.Unlock()}}@@ -191,7 +199,7 @@ func (p *WorkerPool[T, R]) Resize(workers int) {return}- current := int(atomic.LoadInt64(&p.desiredWorkers))+ current := int(atomic.LoadInt64(&p.liveWorkers))atomic.StoreInt64(&p.desiredWorkers, int64(workers))if workers > current {p.start(workers - current)diff --git a/internal/utility/workerpool_resize_test.go b/internal/utility/workerpool_resize_test.gonew file mode 100644index 0000000000..7f9b3f4505--- /dev/null+++ b/internal/utility/workerpool_resize_test.go@@ -0,0 +1,66 @@+package utility++import (+ "context"+ "testing"+ "time"+)++func TestWorkerPoolIdleShrinkGrowDoesNotOverspawn(t *testing.T) {+ p := NewWorkerPool[int, int](4, 10, func(context.Context, int) (int, error) { return 1, nil })+ defer p.StopWait()+ deadline := time.Now().Add(time.Second)+ for p.Stats().LiveWorkers < 4 && time.Now().Before(deadline) {+ time.Sleep(time.Millisecond)+ }+ p.Resize(1)+ p.Resize(3)+ time.Sleep(20 * time.Millisecond)+ if n := p.Stats().LiveWorkers; n > 4 {+ t.Fatalf("idle shrink/grow overspawned: %d workers, want at most initial 4", n)+ }+}++func TestWorkerPoolShrinksThenRunsQueuedWork(t *testing.T) {+ entered := make(chan struct{}, 4)+ release := make(chan struct{})+ p := NewWorkerPool[int, int](4, 10, func(_ context.Context, n int) (int, error) {+ if n < 4 {+ entered <- struct{}{}+ <-release+ }+ return n, nil+ })+ var futures []WorkerPoolFuture[int, int]+ for i := 0; i < 4; i++ {+ f, _ := p.Submit(context.Background(), i)+ futures = append(futures, f)+ }+ for i := 0; i < 4; i++ {+ <-entered+ }+ ctx, cancel := context.WithTimeout(context.Background(), time.Second)+ defer cancel()+ for i := 4; i < 12; i++ {+ f, err := p.Submit(ctx, i)+ if err != nil {+ t.Fatal(err)+ }+ futures = append(futures, f)+ }+ p.Resize(1)+ close(release)+ for _, f := range futures {+ if _, err := f.Wait(ctx); err != nil {+ t.Fatal(err)+ }+ }+ deadline := time.Now().Add(time.Second)+ for p.Stats().LiveWorkers != 1 && time.Now().Before(deadline) {+ time.Sleep(time.Millisecond)+ }+ if n := p.Stats().LiveWorkers; n != 1 {+ t.Fatalf("shrink left %d workers, want 1", n)+ }+ p.StopWait()+}