RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.
Go agent-harnessagent-native-databaseagentic-aiagentic-retrievalagentic-searchaiai-agentscontext-enginecontext-engineeringcontext-managementharness-engineeringknowledge-compilationragretrieval-augmented-generationretrieval-harnesssearch-harness
fix(workerpool): close the SubmitTo/StopWait race without losing in-flight tasks (#18646)
changshenhan4afa19e7e478
diff --git a/internal/utility/workerpool.go b/internal/utility/workerpool.goindex af2ed077c9..c4a98675f2 100644--- a/internal/utility/workerpool.go+++ b/internal/utility/workerpool.go@@ -68,6 +68,10 @@ type WorkerPool[T any, R any] struct {handler WorkerPoolHandler[T, R]workChan chan workerPoolJob[T, R]+ // state is guarded by mu: the only transition is Running→Stopped (in+ // StopWait); SubmitTo and Resize read it to reject new work once the pool+ // is stopped. It is never accessed atomically — every read/write happens+ // under mu so the check and the channel send/close stay mutually exclusive.state uint32desiredWorkers int64@@ -82,7 +86,17 @@ type WorkerPool[T any, R any] struct {submitN uint64doneN uint64- workerWg sync.WaitGroup+ // activeSend counts submits that have passed the stopped-state check and+ // are about to (or currently) send on workChan. It is guarded by mu.+ // StopWait waits for it to reach zero before closing workChan, so a send+ // can never race the close, while SubmitTo is free to release mu before a+ // potentially blocking send (holding mu across a full-queue send would+ // deadlock: a worker blocked in markDone on the same mu can never receive+ // the next job to drain the queue).+ activeSend uint64++ senderDone sync.Cond+ workerWg sync.WaitGroup}// NewWorkerPool creates a worker pool with fixed queue capacity and starts workers immediately.@@ -103,6 +117,7 @@ func NewWorkerPool[T any, R any](workers, queueSize int, handler WorkerPoolHandldesiredWorkers: int64(workers),}p.taskDone.L = &p.mu+ p.senderDone.L = &p.mup.start(workers)return p}@@ -157,11 +172,25 @@ func (p *WorkerPool[T, R]) worker() {// Resize adjusts the target worker count. When shrinking, extra workers retire// after completing their current task.+//+// Resize is a no-op on a stopped pool: it must never spawn workers after+// StopWait has marked the pool stopped, because workerWg.Add racing+// StopWait's workerWg.Wait is a WaitGroup misuse.func (p *WorkerPool[T, R]) Resize(workers int) {if workers <= 0 {panic("workerpool: workers must be greater than zero")}+ // Guard the read-modify-write of desiredWorkers and the workerWg.Add in+ // start() with mu, the same lock StopWait holds while marking the pool+ // stopped. The lock serializes Resize against StopWait: once the pool is+ // stopped no Resize can add workers, so workerWg.Add never races Wait.+ p.mu.Lock()+ defer p.mu.Unlock()+ if p.state == workerPoolStateStopped {+ return+ }+current := int(atomic.LoadInt64(&p.desiredWorkers))atomic.StoreInt64(&p.desiredWorkers, int64(workers))if workers > current {@@ -179,20 +208,48 @@ func (p *WorkerPool[T, R]) Submit(ctx context.Context, input T) (WorkerPoolFutur}// SubmitTo enqueues one task and routes its result into out.+//+// The stopped-state check and the activeSend increment are atomic with+// respect to StopWait's close(workChan): both run while holding mu, and+// StopWait marks the pool stopped and waits for activeSend to reach zero+// before closing the channel. A submit that has passed the check therefore+// either sends before the close, or (if its context is already done) returns+// without sending — never a "send on closed channel" panic. submitN is+// incremented before the send, so StopWait's drain counts a submit whose send+// is still blocked on a full queue and never returns while that task is in+// flight. mu is released before the send itself: a send that blocks on a full+// queue must not hold mu, or a worker finishing its current job would deadlock+// in markDone before it can receive the next job and drain the queue.func (p *WorkerPool[T, R]) SubmitTo(ctx context.Context, input T, out chan<- WorkerPoolResult[T, R]) error {- if atomic.LoadUint32(&p.state) == workerPoolStateStopped {+ j := workerPoolJob[T, R]{ctx: ctx, input: input, out: out}++ p.mu.Lock()+ if p.state == workerPoolStateStopped {+ p.mu.Unlock()return ErrWorkerPoolStopped}+ p.submitN+++ p.activeSend+++ p.mu.Unlock()- j := workerPoolJob[T, R]{ctx: ctx, input: input, out: out}select {case <-ctx.Done():+ p.mu.Lock()+ p.submitN--+ p.activeSend--+ if p.activeSend == 0 {+ p.senderDone.Broadcast()+ }+ p.mu.Unlock()return ctx.Err()case p.workChan <- j:- atomic.AddUint64(&p.submittedTotal, 1)p.mu.Lock()- p.submitN+++ p.activeSend--+ if p.activeSend == 0 {+ p.senderDone.Broadcast()+ }p.mu.Unlock()+ atomic.AddUint64(&p.submittedTotal, 1)return nil}}@@ -208,18 +265,30 @@ func (p *WorkerPool[T, R]) markDone() {// StopWait stops accepting new tasks, waits for queued/running tasks to finish,// then shuts down the worker pool.+//+// Safe to call concurrently with Submit, SubmitTo and Resize. The pool is+// marked stopped and all counters are drained under mu. StopWait first waits+// for activeSend to reach zero — no submit that has already passed the+// stopped-state check is still sending — before closing workChan, so the close+// can never race a send. Because SubmitTo releases mu before a potentially+// blocking send, workers are free to drain the queue and unblock those senders+// instead of deadlocking on markDone. No Resize can spawn workers after the+// pool stops.func (p *WorkerPool[T, R]) StopWait() {- if !atomic.CompareAndSwapUint32(&p.state, workerPoolStateRunning, workerPoolStateStopped) {+ p.mu.Lock()+ if p.state == workerPoolStateStopped {+ p.mu.Unlock()return}-- p.mu.Lock()+ p.state = workerPoolStateStopped+ for p.activeSend != 0 {+ p.senderDone.Wait()+ }for p.submitN != p.doneN {p.taskDone.Wait()}- p.mu.Unlock()-close(p.workChan)+ p.mu.Unlock()p.workerWg.Wait()}diff --git a/internal/utility/workerpool_test.go b/internal/utility/workerpool_test.goindex edfb101eca..feeb9e533d 100644--- a/internal/utility/workerpool_test.go+++ b/internal/utility/workerpool_test.go@@ -2,6 +2,7 @@ package utilityimport ("context"+ "sync""sync/atomic""testing""time"@@ -107,3 +108,200 @@ func TestWorkerPoolResize(t *testing.T) {t.Fatalf("DesiredWorkers = %d, want 3", stats.DesiredWorkers)}}++// TestStopWaitConcurrentSubmitDoesNotPanic hammers Submit from several+// goroutines while StopWait runs, repeated many times. Pre-fix, a submit that+// passed the stopped-state check before StopWait closed workChan panicked with+// "send on closed channel", and the drain was blind to submits whose send was+// still blocked on a full queue. Post-fix every submit either lands on a live+// channel (and is fully processed) or returns ErrWorkerPoolStopped, and+// StopWait never returns with a submitted-but-unfinished task.+func TestStopWaitConcurrentSubmitDoesNotPanic(t *testing.T) {+ for iter := 0; iter < 200; iter++ {+ pool := NewWorkerPool[int, int](4, 8, func(_ context.Context, in int) (int, error) {+ return in + 1, nil+ })+ var wg sync.WaitGroup+ stop := make(chan struct{})+ for g := 0; g < 8; g++ {+ wg.Add(1)+ go func(seed int) {+ defer wg.Done()+ for i := 0; i < 200; i++ {+ select {+ case <-stop:+ return+ default:+ }+ if _, err := pool.Submit(context.Background(), seed+i); err != nil {+ if err != ErrWorkerPoolStopped {+ t.Errorf("unexpected submit error: %v", err)+ }+ return+ }+ }+ }(g * 1000)+ }+ pool.StopWait()+ close(stop)+ wg.Wait()++ st := pool.Stats()+ if st.SubmittedTotal != st.CompletedTotal {+ t.Fatalf("iter %d: StopWait returned with %d submitted but %d completed",+ iter, st.SubmittedTotal, st.CompletedTotal)+ }+ }+}++// TestStopWaitWaitsForInFlightTask verifies the StopWait drain blocks until a+// task currently running in a worker has completed, so its result is delivered+// before StopWait returns.+func TestStopWaitWaitsForInFlightTask(t *testing.T) {+ started := make(chan struct{})+ release := make(chan struct{})+ done := make(chan struct{})++ pool := NewWorkerPool[int, int](1, 2, func(_ context.Context, in int) (int, error) {+ close(started)+ <-release+ return in, nil+ })+ // release is closed inline once the worker is guaranteed blocked in the+ // handler; the deferred close covers the early-return (failing) path so a+ // worker is never left blocked forever. sync.Once keeps the two paths from+ // double-closing the channel.+ var closeReleaseOnce sync.Once+ closeRelease := func() { closeReleaseOnce.Do(func() { close(release) }) }+ defer closeRelease()++ f, err := pool.Submit(context.Background(), 42)+ if err != nil {+ t.Fatalf("Submit: %v", err)+ }+ <-started++ go func() {+ pool.StopWait()+ close(done)+ }()++ // StopWait must not return while the in-flight task is still running.+ select {+ case <-done:+ t.Fatal("StopWait returned before the in-flight task completed")+ case <-time.After(100 * time.Millisecond):+ }++ closeRelease()+ res, err := f.Wait(context.Background())+ if err != nil {+ t.Fatalf("Wait: %v", err)+ }+ if res.Value != 42 {+ t.Fatalf("result = %d, want 42", res.Value)+ }+ <-done+}++// TestSubmitAfterStopWaitReturnsStopped verifies the post-stop contract:+// submits on a stopped pool fail fast with ErrWorkerPoolStopped.+func TestSubmitAfterStopWaitReturnsStopped(t *testing.T) {+ pool := NewWorkerPool[int, int](1, 2, func(_ context.Context, in int) (int, error) {+ return in, nil+ })+ pool.StopWait()+ if _, err := pool.Submit(context.Background(), 1); err != ErrWorkerPoolStopped {+ t.Fatalf("Submit after StopWait = %v, want ErrWorkerPoolStopped", err)+ }+}++// TestStopWaitIdempotent verifies a second StopWait is a no-op: the state is+// already stopped and the channel already closed, so it must not double-close+// (panicking) or double-wait.+func TestStopWaitIdempotent(t *testing.T) {+ pool := NewWorkerPool[int, int](1, 2, func(_ context.Context, in int) (int, error) {+ return in, nil+ })+ pool.StopWait()+ pool.StopWait()+}++// TestResizeAfterStopWaitIsNoop verifies Resize on a stopped pool does not+// panic (workerWg.Add racing workerWg.Wait is a WaitGroup misuse) and does not+// revive workers.+func TestResizeAfterStopWaitIsNoop(t *testing.T) {+ pool := NewWorkerPool[int, int](2, 4, func(_ context.Context, in int) (int, error) {+ return in, nil+ })+ pool.StopWait()+ pool.Resize(8)+ if got := pool.Stats().LiveWorkers; got != 0 {+ t.Fatalf("Resize after StopWait revived workers: live=%d, want 0", got)+ }+}++// TestStopWaitWithBlockedSubmitNoDeadlock verifies that a submit blocked on a+// full workChan (all workers busy and the queue full) does not deadlock+// StopWait. Previously SubmitTo held mu across the blocking channel send, so a+// worker finishing its current job blocked in markDone on the same mu and could+// never receive the next queued job: the queue never drained, the blocked+// sender never made progress, and StopWait hung forever.+func TestStopWaitWithBlockedSubmitNoDeadlock(t *testing.T) {+ started := make(chan struct{})+ released := make(chan struct{})+ pool := NewWorkerPool[int, int](1, 1, func(_ context.Context, in int) (int, error) {+ if in == 1 {+ close(started)+ }+ <-released+ return in, nil+ })++ ctx := context.Background()+ if _, err := pool.Submit(ctx, 1); err != nil {+ t.Fatalf("first Submit: %v", err)+ }+ <-started // the worker is now inside the handler, blocked on released++ if _, err := pool.Submit(ctx, 2); err != nil {+ t.Fatalf("second Submit: %v", err)+ }+ // Queue capacity is 1 and now holds task 2, so the next send must block.++ blocked := make(chan error, 1)+ go func() {+ _, err := pool.Submit(ctx, 3)+ blocked <- err+ }()+ time.Sleep(50 * time.Millisecond) // let the third submit reach the channel send++ stopDone := make(chan struct{})+ go func() {+ pool.StopWait()+ close(stopDone)+ }()+ close(released) // let the worker drain the queue so the blocked send can proceed++ select {+ case <-stopDone:+ // No deadlock.+ case <-time.After(5 * time.Second):+ t.Fatal("StopWait deadlocked with a submit blocked on a full queue")+ }++ select {+ case err := <-blocked:+ if err != nil {+ t.Fatalf("blocked submit returned: %v", err)+ }+ case <-time.After(5 * time.Second):+ t.Fatal("blocked submit never completed after StopWait")+ }++ st := pool.Stats()+ if st.SubmittedTotal != 3 || st.CompletedTotal != 3 {+ t.Fatalf("expected 3/3 tasks completed, got submitted=%d completed=%d",+ st.SubmittedTotal, st.CompletedTotal)+ }+}