Skip to content

RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.

fix(workerpool): close the SubmitTo/StopWait race without losing in-flight tasks (#18646)

changshenhan4afa19e7e478
diff --git a/internal/utility/workerpool.go b/internal/utility/workerpool.go
index af2ed077c9..c4a98675f2 100644
--- a/internal/utility/workerpool.go
+++ b/internal/utility/workerpool.go
@@ -68,6 +68,10 @@ type WorkerPool[T any, R any] struct {
handler WorkerPoolHandler[T, R]
workChan chan workerPoolJob[T, R]
+ // state is guarded by mu: the only transition is Running→Stopped (in
+ // StopWait); SubmitTo and Resize read it to reject new work once the pool
+ // is stopped. It is never accessed atomically — every read/write happens
+ // under mu so the check and the channel send/close stay mutually exclusive.
state uint32
desiredWorkers int64
@@ -82,7 +86,17 @@ type WorkerPool[T any, R any] struct {
submitN uint64
doneN uint64
- workerWg sync.WaitGroup
+ // activeSend counts submits that have passed the stopped-state check and
+ // are about to (or currently) send on workChan. It is guarded by mu.
+ // StopWait waits for it to reach zero before closing workChan, so a send
+ // can never race the close, while SubmitTo is free to release mu before a
+ // potentially blocking send (holding mu across a full-queue send would
+ // deadlock: a worker blocked in markDone on the same mu can never receive
+ // the next job to drain the queue).
+ activeSend uint64
+
+ senderDone sync.Cond
+ workerWg sync.WaitGroup
}
// NewWorkerPool creates a worker pool with fixed queue capacity and starts workers immediately.
@@ -103,6 +117,7 @@ func NewWorkerPool[T any, R any](workers, queueSize int, handler WorkerPoolHandl
desiredWorkers: int64(workers),
}
p.taskDone.L = &p.mu
+ p.senderDone.L = &p.mu
p.start(workers)
return p
}
@@ -157,11 +172,25 @@ func (p *WorkerPool[T, R]) worker() {
// Resize adjusts the target worker count. When shrinking, extra workers retire
// after completing their current task.
+//
+// Resize is a no-op on a stopped pool: it must never spawn workers after
+// StopWait has marked the pool stopped, because workerWg.Add racing
+// StopWait's workerWg.Wait is a WaitGroup misuse.
func (p *WorkerPool[T, R]) Resize(workers int) {
if workers <= 0 {
panic("workerpool: workers must be greater than zero")
}
+ // Guard the read-modify-write of desiredWorkers and the workerWg.Add in
+ // start() with mu, the same lock StopWait holds while marking the pool
+ // stopped. The lock serializes Resize against StopWait: once the pool is
+ // stopped no Resize can add workers, so workerWg.Add never races Wait.
+ p.mu.Lock()
+ defer p.mu.Unlock()
+ if p.state == workerPoolStateStopped {
+ return
+ }
+
current := int(atomic.LoadInt64(&p.desiredWorkers))
atomic.StoreInt64(&p.desiredWorkers, int64(workers))
if workers > current {
@@ -179,20 +208,48 @@ func (p *WorkerPool[T, R]) Submit(ctx context.Context, input T) (WorkerPoolFutur
}
// SubmitTo enqueues one task and routes its result into out.
+//
+// The stopped-state check and the activeSend increment are atomic with
+// respect to StopWait's close(workChan): both run while holding mu, and
+// StopWait marks the pool stopped and waits for activeSend to reach zero
+// before closing the channel. A submit that has passed the check therefore
+// either sends before the close, or (if its context is already done) returns
+// without sending — never a "send on closed channel" panic. submitN is
+// incremented before the send, so StopWait's drain counts a submit whose send
+// is still blocked on a full queue and never returns while that task is in
+// flight. mu is released before the send itself: a send that blocks on a full
+// queue must not hold mu, or a worker finishing its current job would deadlock
+// in markDone before it can receive the next job and drain the queue.
func (p *WorkerPool[T, R]) SubmitTo(ctx context.Context, input T, out chan<- WorkerPoolResult[T, R]) error {
- if atomic.LoadUint32(&p.state) == workerPoolStateStopped {
+ j := workerPoolJob[T, R]{ctx: ctx, input: input, out: out}
+
+ p.mu.Lock()
+ if p.state == workerPoolStateStopped {
+ p.mu.Unlock()
return ErrWorkerPoolStopped
}
+ p.submitN++
+ p.activeSend++
+ p.mu.Unlock()
- j := workerPoolJob[T, R]{ctx: ctx, input: input, out: out}
select {
case <-ctx.Done():
+ p.mu.Lock()
+ p.submitN--
+ p.activeSend--
+ if p.activeSend == 0 {
+ p.senderDone.Broadcast()
+ }
+ p.mu.Unlock()
return ctx.Err()
case p.workChan <- j:
- atomic.AddUint64(&p.submittedTotal, 1)
p.mu.Lock()
- p.submitN++
+ p.activeSend--
+ if p.activeSend == 0 {
+ p.senderDone.Broadcast()
+ }
p.mu.Unlock()
+ atomic.AddUint64(&p.submittedTotal, 1)
return nil
}
}
@@ -208,18 +265,30 @@ func (p *WorkerPool[T, R]) markDone() {
// StopWait stops accepting new tasks, waits for queued/running tasks to finish,
// then shuts down the worker pool.
+//
+// Safe to call concurrently with Submit, SubmitTo and Resize. The pool is
+// marked stopped and all counters are drained under mu. StopWait first waits
+// for activeSend to reach zero — no submit that has already passed the
+// stopped-state check is still sending — before closing workChan, so the close
+// can never race a send. Because SubmitTo releases mu before a potentially
+// blocking send, workers are free to drain the queue and unblock those senders
+// instead of deadlocking on markDone. No Resize can spawn workers after the
+// pool stops.
func (p *WorkerPool[T, R]) StopWait() {
- if !atomic.CompareAndSwapUint32(&p.state, workerPoolStateRunning, workerPoolStateStopped) {
+ p.mu.Lock()
+ if p.state == workerPoolStateStopped {
+ p.mu.Unlock()
return
}
-
- p.mu.Lock()
+ p.state = workerPoolStateStopped
+ for p.activeSend != 0 {
+ p.senderDone.Wait()
+ }
for p.submitN != p.doneN {
p.taskDone.Wait()
}
- p.mu.Unlock()
-
close(p.workChan)
+ p.mu.Unlock()
p.workerWg.Wait()
}
diff --git a/internal/utility/workerpool_test.go b/internal/utility/workerpool_test.go
index edfb101eca..feeb9e533d 100644
--- a/internal/utility/workerpool_test.go
+++ b/internal/utility/workerpool_test.go
@@ -2,6 +2,7 @@ package utility
import (
"context"
+ "sync"
"sync/atomic"
"testing"
"time"
@@ -107,3 +108,200 @@ func TestWorkerPoolResize(t *testing.T) {
t.Fatalf("DesiredWorkers = %d, want 3", stats.DesiredWorkers)
}
}
+
+// TestStopWaitConcurrentSubmitDoesNotPanic hammers Submit from several
+// goroutines while StopWait runs, repeated many times. Pre-fix, a submit that
+// passed the stopped-state check before StopWait closed workChan panicked with
+// "send on closed channel", and the drain was blind to submits whose send was
+// still blocked on a full queue. Post-fix every submit either lands on a live
+// channel (and is fully processed) or returns ErrWorkerPoolStopped, and
+// StopWait never returns with a submitted-but-unfinished task.
+func TestStopWaitConcurrentSubmitDoesNotPanic(t *testing.T) {
+ for iter := 0; iter < 200; iter++ {
+ pool := NewWorkerPool[int, int](4, 8, func(_ context.Context, in int) (int, error) {
+ return in + 1, nil
+ })
+ var wg sync.WaitGroup
+ stop := make(chan struct{})
+ for g := 0; g < 8; g++ {
+ wg.Add(1)
+ go func(seed int) {
+ defer wg.Done()
+ for i := 0; i < 200; i++ {
+ select {
+ case <-stop:
+ return
+ default:
+ }
+ if _, err := pool.Submit(context.Background(), seed+i); err != nil {
+ if err != ErrWorkerPoolStopped {
+ t.Errorf("unexpected submit error: %v", err)
+ }
+ return
+ }
+ }
+ }(g * 1000)
+ }
+ pool.StopWait()
+ close(stop)
+ wg.Wait()
+
+ st := pool.Stats()
+ if st.SubmittedTotal != st.CompletedTotal {
+ t.Fatalf("iter %d: StopWait returned with %d submitted but %d completed",
+ iter, st.SubmittedTotal, st.CompletedTotal)
+ }
+ }
+}
+
+// TestStopWaitWaitsForInFlightTask verifies the StopWait drain blocks until a
+// task currently running in a worker has completed, so its result is delivered
+// before StopWait returns.
+func TestStopWaitWaitsForInFlightTask(t *testing.T) {
+ started := make(chan struct{})
+ release := make(chan struct{})
+ done := make(chan struct{})
+
+ pool := NewWorkerPool[int, int](1, 2, func(_ context.Context, in int) (int, error) {
+ close(started)
+ <-release
+ return in, nil
+ })
+ // release is closed inline once the worker is guaranteed blocked in the
+ // handler; the deferred close covers the early-return (failing) path so a
+ // worker is never left blocked forever. sync.Once keeps the two paths from
+ // double-closing the channel.
+ var closeReleaseOnce sync.Once
+ closeRelease := func() { closeReleaseOnce.Do(func() { close(release) }) }
+ defer closeRelease()
+
+ f, err := pool.Submit(context.Background(), 42)
+ if err != nil {
+ t.Fatalf("Submit: %v", err)
+ }
+ <-started
+
+ go func() {
+ pool.StopWait()
+ close(done)
+ }()
+
+ // StopWait must not return while the in-flight task is still running.
+ select {
+ case <-done:
+ t.Fatal("StopWait returned before the in-flight task completed")
+ case <-time.After(100 * time.Millisecond):
+ }
+
+ closeRelease()
+ res, err := f.Wait(context.Background())
+ if err != nil {
+ t.Fatalf("Wait: %v", err)
+ }
+ if res.Value != 42 {
+ t.Fatalf("result = %d, want 42", res.Value)
+ }
+ <-done
+}
+
+// TestSubmitAfterStopWaitReturnsStopped verifies the post-stop contract:
+// submits on a stopped pool fail fast with ErrWorkerPoolStopped.
+func TestSubmitAfterStopWaitReturnsStopped(t *testing.T) {
+ pool := NewWorkerPool[int, int](1, 2, func(_ context.Context, in int) (int, error) {
+ return in, nil
+ })
+ pool.StopWait()
+ if _, err := pool.Submit(context.Background(), 1); err != ErrWorkerPoolStopped {
+ t.Fatalf("Submit after StopWait = %v, want ErrWorkerPoolStopped", err)
+ }
+}
+
+// TestStopWaitIdempotent verifies a second StopWait is a no-op: the state is
+// already stopped and the channel already closed, so it must not double-close
+// (panicking) or double-wait.
+func TestStopWaitIdempotent(t *testing.T) {
+ pool := NewWorkerPool[int, int](1, 2, func(_ context.Context, in int) (int, error) {
+ return in, nil
+ })
+ pool.StopWait()
+ pool.StopWait()
+}
+
+// TestResizeAfterStopWaitIsNoop verifies Resize on a stopped pool does not
+// panic (workerWg.Add racing workerWg.Wait is a WaitGroup misuse) and does not
+// revive workers.
+func TestResizeAfterStopWaitIsNoop(t *testing.T) {
+ pool := NewWorkerPool[int, int](2, 4, func(_ context.Context, in int) (int, error) {
+ return in, nil
+ })
+ pool.StopWait()
+ pool.Resize(8)
+ if got := pool.Stats().LiveWorkers; got != 0 {
+ t.Fatalf("Resize after StopWait revived workers: live=%d, want 0", got)
+ }
+}
+
+// TestStopWaitWithBlockedSubmitNoDeadlock verifies that a submit blocked on a
+// full workChan (all workers busy and the queue full) does not deadlock
+// StopWait. Previously SubmitTo held mu across the blocking channel send, so a
+// worker finishing its current job blocked in markDone on the same mu and could
+// never receive the next queued job: the queue never drained, the blocked
+// sender never made progress, and StopWait hung forever.
+func TestStopWaitWithBlockedSubmitNoDeadlock(t *testing.T) {
+ started := make(chan struct{})
+ released := make(chan struct{})
+ pool := NewWorkerPool[int, int](1, 1, func(_ context.Context, in int) (int, error) {
+ if in == 1 {
+ close(started)
+ }
+ <-released
+ return in, nil
+ })
+
+ ctx := context.Background()
+ if _, err := pool.Submit(ctx, 1); err != nil {
+ t.Fatalf("first Submit: %v", err)
+ }
+ <-started // the worker is now inside the handler, blocked on released
+
+ if _, err := pool.Submit(ctx, 2); err != nil {
+ t.Fatalf("second Submit: %v", err)
+ }
+ // Queue capacity is 1 and now holds task 2, so the next send must block.
+
+ blocked := make(chan error, 1)
+ go func() {
+ _, err := pool.Submit(ctx, 3)
+ blocked <- err
+ }()
+ time.Sleep(50 * time.Millisecond) // let the third submit reach the channel send
+
+ stopDone := make(chan struct{})
+ go func() {
+ pool.StopWait()
+ close(stopDone)
+ }()
+ close(released) // let the worker drain the queue so the blocked send can proceed
+
+ select {
+ case <-stopDone:
+ // No deadlock.
+ case <-time.After(5 * time.Second):
+ t.Fatal("StopWait deadlocked with a submit blocked on a full queue")
+ }
+
+ select {
+ case err := <-blocked:
+ if err != nil {
+ t.Fatalf("blocked submit returned: %v", err)
+ }
+ case <-time.After(5 * time.Second):
+ t.Fatal("blocked submit never completed after StopWait")
+ }
+
+ st := pool.Stats()
+ if st.SubmittedTotal != 3 || st.CompletedTotal != 3 {
+ t.Fatalf("expected 3/3 tasks completed, got submitted=%d completed=%d",
+ st.SubmittedTotal, st.CompletedTotal)
+ }
+}