Skip to content

RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.

fix(ingestion): stop the MinerU /file_parse upload when the run is cancelled (#21573)

CCLUCKYe47e515fdd5c
diff --git a/internal/ingestion/component/pdf_vision_dispatch.go b/internal/ingestion/component/pdf_vision_dispatch.go
index b1af9f8749..a9b610c325 100644
--- a/internal/ingestion/component/pdf_vision_dispatch.go
+++ b/internal/ingestion/component/pdf_vision_dispatch.go
@@ -610,7 +610,7 @@ func dispatchMinerUPDF(
}
apiURL := strings.TrimRight(baseURL, "/") + "/file_parse"
- zipBytes, err := mineruStreamParse(apiURL, apiKeyRaw, binary, parseMethod, mineruLang, backend, serverURL, extraction)
+ zipBytes, err := mineruStreamParse(ctx, apiURL, apiKeyRaw, binary, parseMethod, mineruLang, backend, serverURL, extraction)
if err != nil {
return parser.ParseResult{}, fmt.Errorf("parser: MinerU stream: %w", err)
}
@@ -768,10 +768,14 @@ func mineruAPIParseMethod(raw string) string {
return "auto"
}
+// mineruStreamTimeout bounds a single /file_parse upload. It matches the
+// MinerU 4 V1 parse budget so both API generations give up at the same point.
+const mineruStreamTimeout = 30 * time.Minute
+
// mineruStreamParse POSTs the PDF binary to the MinerU /file_parse
// endpoint with streaming and returns the zip response body.
// Mirrors Python's mineru_parser.py._run_mineru_api with stream=True.
-func mineruStreamParse(apiURL string, apiKey string, binary []byte, parseMethod, lang, backend, serverURL string, extraction mineruExtractionOptions) ([]byte, error) {
+func mineruStreamParse(ctx context.Context, apiURL string, apiKey string, binary []byte, parseMethod, lang, backend, serverURL string, extraction mineruExtractionOptions) ([]byte, error) {
var body bytes.Buffer
writer := multipart.NewWriter(&body)
@@ -804,7 +808,7 @@ func mineruStreamParse(apiURL string, apiKey string, binary []byte, parseMethod,
return nil, fmt.Errorf("finalize form: %w", err)
}
- ctx, cancel := context.WithTimeout(context.Background(), 30*time.Minute)
+ ctx, cancel := context.WithTimeout(ctx, mineruStreamTimeout)
defer cancel()
req, err := http.NewRequestWithContext(ctx, "POST", apiURL, &body)
@@ -823,7 +827,10 @@ func mineruStreamParse(apiURL string, apiKey string, binary []byte, parseMethod,
defer resp.Body.Close()
if resp.StatusCode != http.StatusOK {
- raw, _ := io.ReadAll(resp.Body)
+ raw, readErr := io.ReadAll(resp.Body)
+ if readErr != nil {
+ return nil, fmt.Errorf("HTTP %d: read error response: %w", resp.StatusCode, readErr)
+ }
return nil, fmt.Errorf("HTTP %d: %s", resp.StatusCode, string(raw))
}
diff --git a/internal/ingestion/component/pdf_vision_dispatch_test.go b/internal/ingestion/component/pdf_vision_dispatch_test.go
index 9876b69cb1..09b3850c3e 100644
--- a/internal/ingestion/component/pdf_vision_dispatch_test.go
+++ b/internal/ingestion/component/pdf_vision_dispatch_test.go
@@ -21,12 +21,14 @@ import (
"context"
"encoding/json"
"errors"
+ "io"
"math"
"net/http"
"net/http/httptest"
"os"
"path/filepath"
"strings"
+ "sync"
"testing"
"time"
@@ -606,3 +608,139 @@ func TestDispatchPaddleOCRPDFAcceptsLocatedBlocksWithoutText(t *testing.T) {
t.Fatalf("items=%v", result.JSON)
}
}
+
+// A cancelled ingestion run must abort the MinerU 3.x /file_parse upload
+// instead of holding the worker and the remote server for the whole stream
+// budget.
+func TestDispatchPDFMinerUStreamHonoursCancellation(t *testing.T) {
+ withSSRFBypass(t)
+ started := make(chan struct{})
+ release := make(chan struct{})
+ server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ if r.URL.Path != "/file_parse" {
+ http.NotFound(w, r)
+ return
+ }
+ close(started)
+ select {
+ case <-r.Context().Done():
+ case <-release:
+ }
+ }))
+ defer func() {
+ close(release)
+ server.Close()
+ }()
+
+ apiKey := "mineru-secret"
+ baseURL := server.URL
+ original := resolveMinerUModelForDispatch
+ t.Cleanup(func() { resolveMinerUModelForDispatch = original })
+ resolveMinerUModelForDispatch = func(context.Context, *gorm.DB, string, string) (modelModule.ModelDriver, string, *modelModule.APIConfig, error) {
+ return &mineruTestDriver{}, "vlm", &modelModule.APIConfig{ApiKey: &apiKey, BaseURL: &baseURL}, nil
+ }
+
+ ctx, cancel := context.WithCancel(t.Context())
+ done := make(chan error, 1)
+ go func() {
+ _, err := dispatchMinerUPDF(ctx, nil, "scan.pdf", []byte("%PDF-1.4"), "tenant", nil, "")
+ done <- err
+ }()
+
+ select {
+ case <-started:
+ case <-time.After(5 * time.Second):
+ t.Fatal("MinerU /file_parse was never reached")
+ }
+ cancel()
+ select {
+ case err := <-done:
+ if !errors.Is(err, context.Canceled) {
+ t.Fatalf("dispatch error = %v, want context.Canceled", err)
+ }
+ case <-time.After(5 * time.Second):
+ t.Fatal("dispatch kept the /file_parse request open after cancellation")
+ }
+}
+
+// readSignalBody closes signal once the client has read body bytes, which
+// tells the test the response header was consumed and ReadAll is in progress.
+type readSignalBody struct {
+ io.ReadCloser
+ once sync.Once
+ signal chan struct{}
+}
+
+func (b *readSignalBody) Read(p []byte) (int, error) {
+ n, err := b.ReadCloser.Read(p)
+ if n > 0 {
+ b.once.Do(func() { close(b.signal) })
+ }
+ return n, err
+}
+
+type readSignalTransport struct {
+ base http.RoundTripper
+ signal chan struct{}
+}
+
+func (t readSignalTransport) RoundTrip(req *http.Request) (*http.Response, error) {
+ resp, err := t.base.RoundTrip(req)
+ if err != nil {
+ return nil, err
+ }
+ resp.Body = &readSignalBody{ReadCloser: resp.Body, signal: t.signal}
+ return resp, nil
+}
+
+// A run cancelled while a non-200 error body is still streaming must still
+// report context.Canceled; reporting the bare HTTP status hides the reason the
+// request ended.
+func TestMineruStreamKeepsCancellationWhileReadingErrorResponse(t *testing.T) {
+ release := make(chan struct{})
+ server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ w.WriteHeader(http.StatusBadGateway)
+ _, _ = w.Write([]byte(`{"detail":"upstream`))
+ if flusher, ok := w.(http.Flusher); ok {
+ flusher.Flush()
+ }
+ select {
+ case <-r.Context().Done():
+ case <-release:
+ }
+ }))
+ defer func() {
+ close(release)
+ server.Close()
+ }()
+
+ bodyRead := make(chan struct{})
+ originalClient := http.DefaultClient
+ http.DefaultClient = &http.Client{Transport: readSignalTransport{base: http.DefaultTransport, signal: bodyRead}}
+ t.Cleanup(func() { http.DefaultClient = originalClient })
+
+ ctx, cancel := context.WithCancel(t.Context())
+ done := make(chan error, 1)
+ go func() {
+ _, err := mineruStreamParse(ctx, server.URL+"/file_parse", "mineru-secret", []byte("%PDF-1.4"), "auto", "en", "", "", mineruExtractionOptions{})
+ done <- err
+ }()
+
+ select {
+ case <-bodyRead:
+ case <-time.After(5 * time.Second):
+ t.Fatal("MinerU never started reading the error response body")
+ }
+ cancel()
+ select {
+ case err := <-done:
+ if !strings.Contains(err.Error(), "HTTP 502") {
+ t.Fatalf("stream error = %v, want the HTTP 502 error response path", err)
+ }
+ if !errors.Is(err, context.Canceled) {
+ t.Fatalf("stream error = %v, want context.Canceled", err)
+ }
+ case <-time.After(5 * time.Second):
+ t.Fatal("stream kept reading the error response after cancellation")
+ }
+}