RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.
Go agent-harnessagent-native-databaseagentic-aiagentic-retrievalagentic-searchaiai-agentscontext-enginecontext-engineeringcontext-managementharness-engineeringknowledge-compilationragretrieval-augmented-generationretrieval-harnesssearch-harness
fix(parser): stop MinerU polling when the run is cancelled (#21571)
CCLUCKY350df0863253
diff --git a/internal/parser/parser/pdf_parser_mineru.go b/internal/parser/parser/pdf_parser_mineru.goindex c80164bb4a..fc1e68cc97 100644--- a/internal/parser/parser/pdf_parser_mineru.go+++ b/internal/parser/parser/pdf_parser_mineru.go@@ -115,9 +115,31 @@ func pollMinerUTask(ctx context.Context, driver *models.MinerULocalModel, taskIDtimeout = minerUPollTimeout}deadline := time.Now().Add(timeout)+ // Each status call can wait up to its own request timeout, which is longer+ // than a short poll budget can be. Bound the call by the poll deadline too,+ // otherwise a slow status response outlives the budget and its content is+ // returned as if the poll had succeeded in time.+ pollCtx, cancel := context.WithDeadline(ctx, deadline)+ defer cancel()var lastErr errorfor {- task, err := driver.ShowTask(ctx, taskID, apiConfig)+ if err := ctx.Err(); err != nil {+ return "", err+ }+ task, err := driver.ShowTask(pollCtx, taskID, apiConfig)+ if pollCtx.Err() != nil {+ if ctxErr := ctx.Err(); ctxErr != nil {+ return "", ctxErr+ }+ cause := lastErr+ if cause == nil {+ cause = err+ }+ if cause == nil {+ cause = context.DeadlineExceeded+ }+ return "", fmt.Errorf("timed out after %s waiting for MinerU task %s: %w", timeout, taskID, cause)+ }if err == nil {for _, segment := range task.Segments {if strings.TrimSpace(segment.Content) != "" {@@ -128,12 +150,22 @@ func pollMinerUTask(ctx context.Context, driver *models.MinerULocalModel, taskID} else {lastErr = err}- if time.Now().After(deadline) {- if lastErr == nil {- lastErr = fmt.Errorf("timed out waiting for MinerU task %s", taskID)- }- return "", lastErr+ // A cancelled run has to leave the poll at once. Sleeping the scan+ // interval, let alone the remaining timeout, keeps an ingestion worker+ // polling a server it will never accept a result from.+ wait := time.Until(deadline)+ if wait <= 0 {+ return "", fmt.Errorf("timed out after %s waiting for MinerU task %s: %w", timeout, taskID, lastErr)+ }+ if wait > minerUPollInterval {+ wait = minerUPollInterval+ }+ timer := time.NewTimer(wait)+ select {+ case <-ctx.Done():+ timer.Stop()+ return "", ctx.Err()+ case <-timer.C:}- time.Sleep(minerUPollInterval)}}diff --git a/internal/parser/parser/pdf_parser_mineru_poll_test.go b/internal/parser/parser/pdf_parser_mineru_poll_test.gonew file mode 100644index 0000000000..76e3185dc8--- /dev/null+++ b/internal/parser/parser/pdf_parser_mineru_poll_test.go@@ -0,0 +1,192 @@+package parser++import (+ "context"+ "errors"+ "net/http"+ "net/http/httptest"+ "strings"+ "sync"+ "sync/atomic"+ "testing"+ "time"++ models "ragflow/internal/entity/models"+)++// minerURunningServer answers every status call the way a MinerU server does+// while a task is still being parsed: HTTP 200 with no "results" block, which+// ShowTask reports as an error and the poll therefore treats as "not done".+func minerURunningServer(t *testing.T) *models.MinerULocalModel {+ t.Helper()+ server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {+ w.Header().Set("Content-Type", "application/json")+ _, _ = w.Write([]byte(`{"status":"running"}`))+ }))+ t.Cleanup(server.Close)+ return models.NewMinerLocalUModel(+ map[string]string{"default": server.URL},+ models.URLSuffix{Task: "tasks"},+ )+}++func TestPollMinerUTask_StopsOnContextCancel(t *testing.T) {+ withSSRFBypass(t)+ driver := minerURunningServer(t)++ ctx, cancel := context.WithCancel(t.Context())+ t.Cleanup(cancel)+ go func() {+ time.Sleep(300 * time.Millisecond)+ cancel()+ }()++ start := time.Now()+ _, err := pollMinerUTask(ctx, driver, "task-1", &models.APIConfig{}, 30*time.Second)+ elapsed := time.Since(start)++ if !errors.Is(err, context.Canceled) {+ t.Fatalf("error = %v, want context.Canceled", err)+ }+ // The poll must leave on cancellation instead of sleeping out the 30s+ // budget it was given.+ if elapsed > 5*time.Second {+ t.Fatalf("poll returned after %s, want it to stop at cancellation", elapsed)+ }+}++func TestPollMinerUTask_ReportsTimeoutWithLastError(t *testing.T) {+ withSSRFBypass(t)+ driver := minerURunningServer(t)++ _, err := pollMinerUTask(t.Context(), driver, "task-2", &models.APIConfig{}, 300*time.Millisecond)+ if err == nil {+ t.Fatal("poll: want timeout error, got nil")+ }+ if errors.Is(err, context.Canceled) {+ t.Fatalf("error = %v, want a timeout rather than cancellation", err)+ }+ if !strings.Contains(err.Error(), "timed out after") {+ t.Fatalf("error = %q, want it to name the timeout", err)+ }+ if !strings.Contains(err.Error(), "missing results field") {+ t.Fatalf("error = %q, want the last status error as the cause", err)+ }+ cause := errors.Unwrap(err)+ if cause == nil || !strings.Contains(cause.Error(), "missing results field") {+ t.Fatalf("cause = %v, want the last status error wrapped with %%w", cause)+ }+}++func TestPollMinerUTask_StopsBeforeStatusCallWhenAlreadyCancelled(t *testing.T) {+ withSSRFBypass(t)+ var hits atomic.Int64+ server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {+ hits.Add(1)+ w.Header().Set("Content-Type", "application/json")+ _, _ = w.Write([]byte(`{"results":{"doc":{"md_content":"# Done"}}}`))+ }))+ defer server.Close()+ driver := models.NewMinerLocalUModel(+ map[string]string{"default": server.URL},+ models.URLSuffix{Task: "tasks"},+ )++ ctx, cancel := context.WithCancel(t.Context())+ cancel()++ _, err := pollMinerUTask(ctx, driver, "task-3", &models.APIConfig{}, 30*time.Second)+ if !errors.Is(err, context.Canceled) {+ t.Fatalf("error = %v, want context.Canceled", err)+ }+ if hits.Load() != 0 {+ t.Fatalf("status requests = %d, want 0 for an already cancelled run", hits.Load())+ }+}++// A status call can outlive the poll budget on its own request timeout. Its+// late content must not be returned as an in-time success.+func TestPollMinerUTask_ReportsTimeoutWhenStatusReturnsLate(t *testing.T) {+ withSSRFBypass(t)+ server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {+ select {+ case <-time.After(5 * time.Second):+ case <-r.Context().Done():+ return+ }+ w.Header().Set("Content-Type", "application/json")+ _, _ = w.Write([]byte(`{"results":{"doc":{"md_content":"# Late"}}}`))+ }))+ defer server.Close()+ driver := models.NewMinerLocalUModel(+ map[string]string{"default": server.URL},+ models.URLSuffix{Task: "tasks"},+ )++ start := time.Now()+ _, err := pollMinerUTask(t.Context(), driver, "task-late", &models.APIConfig{}, 100*time.Millisecond)+ elapsed := time.Since(start)++ if err == nil {+ t.Fatal("poll returned a late result after the budget elapsed")+ }+ if !strings.Contains(err.Error(), "timed out after") {+ t.Fatalf("error = %v, want it to name the timeout", err)+ }+ if elapsed > time.Second {+ t.Fatalf("poll returned after %s, want it bounded by the 100ms budget", elapsed)+ }+}++// A cancel that lands while a status call is still waiting has to abort that+// call, not wait for its own request timeout.+func TestPollMinerUTask_StopsWhenCancelledDuringStatusCall(t *testing.T) {+ withSSRFBypass(t)+ var calls atomic.Int64+ inFlight := make(chan struct{})+ var signalOnce sync.Once+ server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {+ calls.Add(1)+ signalOnce.Do(func() { close(inFlight) })+ select {+ case <-r.Context().Done():+ case <-time.After(30 * time.Second):+ }+ }))+ defer server.Close()+ driver := models.NewMinerLocalUModel(+ map[string]string{"default": server.URL},+ models.URLSuffix{Task: "tasks"},+ )++ ctx, cancel := context.WithCancel(t.Context())+ t.Cleanup(cancel)+ done := make(chan error, 1)+ go func() {+ _, err := pollMinerUTask(ctx, driver, "task-inflight", &models.APIConfig{}, 30*time.Second)+ done <- err+ }()++ select {+ case <-inFlight:+ case <-time.After(5 * time.Second):+ t.Fatal("status request never reached the server")+ }+ start := time.Now()+ cancel()++ select {+ case err := <-done:+ if !errors.Is(err, context.Canceled) {+ t.Fatalf("error = %v, want context.Canceled", err)+ }+ if elapsed := time.Since(start); elapsed > 5*time.Second {+ t.Fatalf("poll returned after %s, want it to stop when the run was cancelled", elapsed)+ }+ case <-time.After(5 * time.Second):+ t.Fatal("poll kept waiting on the in-flight status call after the run was cancelled")+ }+ if calls.Load() != 1 {+ t.Fatalf("status requests = %d, want 1: the poll must cancel the call in flight instead of starting another", calls.Load())+ }+}