RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.
Go agent-harnessagent-native-databaseagentic-aiagentic-retrievalagentic-searchaiai-agentscontext-enginecontext-engineeringcontext-managementharness-engineeringknowledge-compilationragretrieval-augmented-generationretrieval-harnesssearch-harness
fix(ingestion): stop the MinerU /file_parse upload when the run is cancelled (#21573)
CCLUCKYe47e515fdd5c
diff --git a/internal/ingestion/component/pdf_vision_dispatch.go b/internal/ingestion/component/pdf_vision_dispatch.goindex b1af9f8749..a9b610c325 100644--- a/internal/ingestion/component/pdf_vision_dispatch.go+++ b/internal/ingestion/component/pdf_vision_dispatch.go@@ -610,7 +610,7 @@ func dispatchMinerUPDF(}apiURL := strings.TrimRight(baseURL, "/") + "/file_parse"- zipBytes, err := mineruStreamParse(apiURL, apiKeyRaw, binary, parseMethod, mineruLang, backend, serverURL, extraction)+ zipBytes, err := mineruStreamParse(ctx, apiURL, apiKeyRaw, binary, parseMethod, mineruLang, backend, serverURL, extraction)if err != nil {return parser.ParseResult{}, fmt.Errorf("parser: MinerU stream: %w", err)}@@ -768,10 +768,14 @@ func mineruAPIParseMethod(raw string) string {return "auto"}+// mineruStreamTimeout bounds a single /file_parse upload. It matches the+// MinerU 4 V1 parse budget so both API generations give up at the same point.+const mineruStreamTimeout = 30 * time.Minute+// mineruStreamParse POSTs the PDF binary to the MinerU /file_parse// endpoint with streaming and returns the zip response body.// Mirrors Python's mineru_parser.py._run_mineru_api with stream=True.-func mineruStreamParse(apiURL string, apiKey string, binary []byte, parseMethod, lang, backend, serverURL string, extraction mineruExtractionOptions) ([]byte, error) {+func mineruStreamParse(ctx context.Context, apiURL string, apiKey string, binary []byte, parseMethod, lang, backend, serverURL string, extraction mineruExtractionOptions) ([]byte, error) {var body bytes.Bufferwriter := multipart.NewWriter(&body)@@ -804,7 +808,7 @@ func mineruStreamParse(apiURL string, apiKey string, binary []byte, parseMethod,return nil, fmt.Errorf("finalize form: %w", err)}- ctx, cancel := context.WithTimeout(context.Background(), 30*time.Minute)+ ctx, cancel := context.WithTimeout(ctx, mineruStreamTimeout)defer cancel()req, err := http.NewRequestWithContext(ctx, "POST", apiURL, &body)@@ -823,7 +827,10 @@ func mineruStreamParse(apiURL string, apiKey string, binary []byte, parseMethod,defer resp.Body.Close()if resp.StatusCode != http.StatusOK {- raw, _ := io.ReadAll(resp.Body)+ raw, readErr := io.ReadAll(resp.Body)+ if readErr != nil {+ return nil, fmt.Errorf("HTTP %d: read error response: %w", resp.StatusCode, readErr)+ }return nil, fmt.Errorf("HTTP %d: %s", resp.StatusCode, string(raw))}diff --git a/internal/ingestion/component/pdf_vision_dispatch_test.go b/internal/ingestion/component/pdf_vision_dispatch_test.goindex 9876b69cb1..09b3850c3e 100644--- a/internal/ingestion/component/pdf_vision_dispatch_test.go+++ b/internal/ingestion/component/pdf_vision_dispatch_test.go@@ -21,12 +21,14 @@ import ("context""encoding/json""errors"+ "io""math""net/http""net/http/httptest""os""path/filepath""strings"+ "sync""testing""time"@@ -606,3 +608,139 @@ func TestDispatchPaddleOCRPDFAcceptsLocatedBlocksWithoutText(t *testing.T) {t.Fatalf("items=%v", result.JSON)}}++// A cancelled ingestion run must abort the MinerU 3.x /file_parse upload+// instead of holding the worker and the remote server for the whole stream+// budget.+func TestDispatchPDFMinerUStreamHonoursCancellation(t *testing.T) {+ withSSRFBypass(t)+ started := make(chan struct{})+ release := make(chan struct{})+ server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {+ if r.URL.Path != "/file_parse" {+ http.NotFound(w, r)+ return+ }+ close(started)+ select {+ case <-r.Context().Done():+ case <-release:+ }+ }))+ defer func() {+ close(release)+ server.Close()+ }()++ apiKey := "mineru-secret"+ baseURL := server.URL+ original := resolveMinerUModelForDispatch+ t.Cleanup(func() { resolveMinerUModelForDispatch = original })+ resolveMinerUModelForDispatch = func(context.Context, *gorm.DB, string, string) (modelModule.ModelDriver, string, *modelModule.APIConfig, error) {+ return &mineruTestDriver{}, "vlm", &modelModule.APIConfig{ApiKey: &apiKey, BaseURL: &baseURL}, nil+ }++ ctx, cancel := context.WithCancel(t.Context())+ done := make(chan error, 1)+ go func() {+ _, err := dispatchMinerUPDF(ctx, nil, "scan.pdf", []byte("%PDF-1.4"), "tenant", nil, "")+ done <- err+ }()++ select {+ case <-started:+ case <-time.After(5 * time.Second):+ t.Fatal("MinerU /file_parse was never reached")+ }+ cancel()+ select {+ case err := <-done:+ if !errors.Is(err, context.Canceled) {+ t.Fatalf("dispatch error = %v, want context.Canceled", err)+ }+ case <-time.After(5 * time.Second):+ t.Fatal("dispatch kept the /file_parse request open after cancellation")+ }+}++// readSignalBody closes signal once the client has read body bytes, which+// tells the test the response header was consumed and ReadAll is in progress.+type readSignalBody struct {+ io.ReadCloser+ once sync.Once+ signal chan struct{}+}++func (b *readSignalBody) Read(p []byte) (int, error) {+ n, err := b.ReadCloser.Read(p)+ if n > 0 {+ b.once.Do(func() { close(b.signal) })+ }+ return n, err+}++type readSignalTransport struct {+ base http.RoundTripper+ signal chan struct{}+}++func (t readSignalTransport) RoundTrip(req *http.Request) (*http.Response, error) {+ resp, err := t.base.RoundTrip(req)+ if err != nil {+ return nil, err+ }+ resp.Body = &readSignalBody{ReadCloser: resp.Body, signal: t.signal}+ return resp, nil+}++// A run cancelled while a non-200 error body is still streaming must still+// report context.Canceled; reporting the bare HTTP status hides the reason the+// request ended.+func TestMineruStreamKeepsCancellationWhileReadingErrorResponse(t *testing.T) {+ release := make(chan struct{})+ server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {+ w.WriteHeader(http.StatusBadGateway)+ _, _ = w.Write([]byte(`{"detail":"upstream`))+ if flusher, ok := w.(http.Flusher); ok {+ flusher.Flush()+ }+ select {+ case <-r.Context().Done():+ case <-release:+ }+ }))+ defer func() {+ close(release)+ server.Close()+ }()++ bodyRead := make(chan struct{})+ originalClient := http.DefaultClient+ http.DefaultClient = &http.Client{Transport: readSignalTransport{base: http.DefaultTransport, signal: bodyRead}}+ t.Cleanup(func() { http.DefaultClient = originalClient })++ ctx, cancel := context.WithCancel(t.Context())+ done := make(chan error, 1)+ go func() {+ _, err := mineruStreamParse(ctx, server.URL+"/file_parse", "mineru-secret", []byte("%PDF-1.4"), "auto", "en", "", "", mineruExtractionOptions{})+ done <- err+ }()++ select {+ case <-bodyRead:+ case <-time.After(5 * time.Second):+ t.Fatal("MinerU never started reading the error response body")+ }+ cancel()+ select {+ case err := <-done:+ if !strings.Contains(err.Error(), "HTTP 502") {+ t.Fatalf("stream error = %v, want the HTTP 502 error response path", err)+ }+ if !errors.Is(err, context.Canceled) {+ t.Fatalf("stream error = %v, want context.Canceled", err)+ }+ case <-time.After(5 * time.Second):+ t.Fatal("stream kept reading the error response after cancellation")+ }+}