Skip to content

RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.

WorkerPool resize overspawns after shrinking an idle pool (#20490)

Taranum Wasu1826402ada63
diff --git a/internal/utility/workerpool.go b/internal/utility/workerpool.go
index 1ad429abef..fa75ea83f2 100644
--- a/internal/utility/workerpool.go
+++ b/internal/utility/workerpool.go
@@ -125,14 +125,17 @@ func NewWorkerPool[T any, R any](workers, queueSize int, handler WorkerPoolHandl
func (p *WorkerPool[T, R]) start(workers int) {
for range workers {
p.workerWg.Add(1)
+ atomic.AddInt64(&p.liveWorkers, 1)
go p.worker()
}
}
func (p *WorkerPool[T, R]) worker() {
- atomic.AddInt64(&p.liveWorkers, 1)
+ retired := false
defer func() {
- atomic.AddInt64(&p.liveWorkers, -1)
+ if !retired {
+ atomic.AddInt64(&p.liveWorkers, -1)
+ }
p.workerWg.Done()
}()
@@ -164,9 +167,14 @@ func (p *WorkerPool[T, R]) worker() {
}
p.markDone()
+ p.mu.Lock()
if atomic.LoadInt64(&p.liveWorkers) > atomic.LoadInt64(&p.desiredWorkers) {
+ atomic.AddInt64(&p.liveWorkers, -1)
+ retired = true
+ p.mu.Unlock()
return
}
+ p.mu.Unlock()
}
}
@@ -191,7 +199,7 @@ func (p *WorkerPool[T, R]) Resize(workers int) {
return
}
- current := int(atomic.LoadInt64(&p.desiredWorkers))
+ current := int(atomic.LoadInt64(&p.liveWorkers))
atomic.StoreInt64(&p.desiredWorkers, int64(workers))
if workers > current {
p.start(workers - current)
diff --git a/internal/utility/workerpool_resize_test.go b/internal/utility/workerpool_resize_test.go
new file mode 100644
index 0000000000..7f9b3f4505
--- /dev/null
+++ b/internal/utility/workerpool_resize_test.go
@@ -0,0 +1,66 @@
+package utility
+
+import (
+ "context"
+ "testing"
+ "time"
+)
+
+func TestWorkerPoolIdleShrinkGrowDoesNotOverspawn(t *testing.T) {
+ p := NewWorkerPool[int, int](4, 10, func(context.Context, int) (int, error) { return 1, nil })
+ defer p.StopWait()
+ deadline := time.Now().Add(time.Second)
+ for p.Stats().LiveWorkers < 4 && time.Now().Before(deadline) {
+ time.Sleep(time.Millisecond)
+ }
+ p.Resize(1)
+ p.Resize(3)
+ time.Sleep(20 * time.Millisecond)
+ if n := p.Stats().LiveWorkers; n > 4 {
+ t.Fatalf("idle shrink/grow overspawned: %d workers, want at most initial 4", n)
+ }
+}
+
+func TestWorkerPoolShrinksThenRunsQueuedWork(t *testing.T) {
+ entered := make(chan struct{}, 4)
+ release := make(chan struct{})
+ p := NewWorkerPool[int, int](4, 10, func(_ context.Context, n int) (int, error) {
+ if n < 4 {
+ entered <- struct{}{}
+ <-release
+ }
+ return n, nil
+ })
+ var futures []WorkerPoolFuture[int, int]
+ for i := 0; i < 4; i++ {
+ f, _ := p.Submit(context.Background(), i)
+ futures = append(futures, f)
+ }
+ for i := 0; i < 4; i++ {
+ <-entered
+ }
+ ctx, cancel := context.WithTimeout(context.Background(), time.Second)
+ defer cancel()
+ for i := 4; i < 12; i++ {
+ f, err := p.Submit(ctx, i)
+ if err != nil {
+ t.Fatal(err)
+ }
+ futures = append(futures, f)
+ }
+ p.Resize(1)
+ close(release)
+ for _, f := range futures {
+ if _, err := f.Wait(ctx); err != nil {
+ t.Fatal(err)
+ }
+ }
+ deadline := time.Now().Add(time.Second)
+ for p.Stats().LiveWorkers != 1 && time.Now().Before(deadline) {
+ time.Sleep(time.Millisecond)
+ }
+ if n := p.Stats().LiveWorkers; n != 1 {
+ t.Fatalf("shrink left %d workers, want 1", n)
+ }
+ p.StopWait()
+}