RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.
Go agent-harnessagent-native-databaseagentic-aiagentic-retrievalagentic-searchaiai-agentscontext-enginecontext-engineeringcontext-managementharness-engineeringknowledge-compilationragretrieval-augmented-generationretrieval-harnesssearch-harness
fix(parser): honour the pipeline context in SoMark and Docling PDF calls (#21536)
CCLUCKYde59ebf378fa
diff --git a/internal/parser/parser/pdf_parser_docling.go b/internal/parser/parser/pdf_parser_docling.goindex cfe2ce5d24..5bc02acd5a 100644--- a/internal/parser/parser/pdf_parser_docling.go+++ b/internal/parser/parser/pdf_parser_docling.go@@ -101,8 +101,14 @@ func parsePDFWithDocling(ctx context.Context, filename string, data []byte, parsvar lastErr errorfor _, candidate := range payloads {+ // Each candidate marshals the whole base64 PDF, so stop before building+ // another request once the run is cancelled.+ if err := ctx.Err(); err != nil {+ lastErr = err+ break+ }url := baseURL + candidate.endpoint- resp, err := models.PostJSONRequest(context.Background(), common.GetSSRFHTTPClient(), url, auth, candidate.body())+ resp, err := models.PostJSONRequest(ctx, common.GetSSRFHTTPClient(), url, auth, candidate.body())if err != nil {lastErr = fmt.Errorf("%s: %w", candidate.endpoint, err)continuediff --git a/internal/parser/parser/pdf_parser_remote_ctx_test.go b/internal/parser/parser/pdf_parser_remote_ctx_test.gonew file mode 100644index 0000000000..71bbe101cf--- /dev/null+++ b/internal/parser/parser/pdf_parser_remote_ctx_test.go@@ -0,0 +1,266 @@+package parser++import (+ "context"+ "errors"+ "net/http"+ "net/http/httptest"+ "strings"+ "sync"+ "sync/atomic"+ "testing"+ "time"+)++// The remote PDF backends must honour the pipeline context: a cancelled run+// stops the request instead of waiting on the transport's header timeout.++// holdRequest starts a server that records the request and reports entry, then+// keeps the response open until release closes. Cancelling before the release+// is what proves the call aborts mid-flight: a call that ignored the context+// would read the released response and return without context.Canceled.+func holdRequest(t *testing.T) (*httptest.Server, *atomic.Int64, chan struct{}, chan struct{}) {+ t.Helper()+ var hits atomic.Int64+ entered := make(chan struct{}, 1)+ release := make(chan struct{})+ server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {+ hits.Add(1)+ select {+ case entered <- struct{}{}:+ default:+ }+ <-release+ w.WriteHeader(http.StatusServiceUnavailable)+ }))+ t.Cleanup(server.Close)+ return server, &hits, entered, release+}++func TestParsePDFWithSoMark_CancelledContextStopsBeforeSubmit(t *testing.T) {+ withSSRFBypass(t)+ var hits int+ server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {+ hits+++ _, _ = w.Write([]byte(`{"code":0,"data":{"task_id":"task-1"}}`))+ }))+ defer server.Close()++ ctx, cancel := context.WithCancel(t.Context())+ cancel()++ p := NewPDFParser()+ p.SoMarkBaseURL = server.URL+ p.SoMarkAPIKey = "key"+ res := parsePDFWithSoMark(ctx, "sample.pdf", []byte("%PDF-1.4"), p)+ if res.Err == nil {+ t.Fatalf("error = nil, want context cancellation from submit")+ }+ if !errors.Is(res.Err, context.Canceled) {+ t.Fatalf("error = %v, want context.Canceled in chain", res.Err)+ }+ if hits != 0 {+ t.Fatalf("server requests = %d, want 0 after cancellation", hits)+ }+}++func TestSoMarkSubmit_CancelDuringRequestAbortsIt(t *testing.T) {+ withSSRFBypass(t)+ server, hits, entered, release := holdRequest(t)+ var releaseOnce sync.Once+ releaseHandler := func() { releaseOnce.Do(func() { close(release) }) }+ defer releaseHandler()++ ctx, cancel := context.WithCancel(t.Context())+ defer cancel()++ // The call must return on cancellation before the server writes its+ // response: releasing first would let a context-ignoring client read the+ // 503 instead of failing with context.Canceled, hiding the regression.+ done := make(chan error, 1)+ go func() {+ _, err := soMarkSubmit(ctx, server.URL, "sample.pdf", []byte("%PDF-1.4"), NewPDFParser(), "key")+ done <- err+ }()++ select {+ case <-entered:+ case <-time.After(2 * time.Second):+ t.Fatal("the SoMark submit request never reached the server")+ }+ cancel()++ var err error+ select {+ case err = <-done:+ case <-time.After(2 * time.Second):+ releaseHandler()+ t.Fatal("soMarkSubmit returned nothing after cancellation: the in-flight request kept running")+ }+ releaseHandler()++ if !errors.Is(err, context.Canceled) {+ t.Fatalf("error = %v, want context.Canceled in chain", err)+ }+ if got := hits.Load(); got != 1 {+ t.Fatalf("server requests = %d, want 1 aborted request", got)+ }+}++// Cancelling while the poll request is in flight must abort that request. This+// is the case that proves parsePDFWithSoMark hands its own context to soMarkPoll:+// a variant that derived a fresh background context there would read the+// released response instead of failing with context.Canceled.+func TestParsePDFWithSoMark_CancelDuringPollAbortsPollRequest(t *testing.T) {+ withSSRFBypass(t)+ var pollHits atomic.Int64+ entered := make(chan struct{}, 1)+ release := make(chan struct{})+ var once sync.Once+ server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {+ if r.URL.Path != "/parse/async_check" {+ _, _ = w.Write([]byte(`{"code":0,"data":{"task_id":"task-1"}}`))+ return+ }+ pollHits.Add(1)+ once.Do(func() { close(entered) })+ <-release+ w.WriteHeader(http.StatusServiceUnavailable)+ }))+ defer server.Close()++ ctx, cancel := context.WithCancel(t.Context())+ defer cancel()++ var releaseOnce sync.Once+ releaseHandler := func() { releaseOnce.Do(func() { close(release) }) }+ defer releaseHandler()++ p := NewPDFParser()+ p.SoMarkBaseURL = server.URL+ p.SoMarkAPIKey = "key"+ done := make(chan ParseResult, 1)+ go func() { done <- parsePDFWithSoMark(ctx, "sample.pdf", []byte("%PDF-1.4"), p) }()++ select {+ case <-entered:+ case <-time.After(2 * time.Second):+ t.Fatal("the SoMark poll request never reached the server")+ }+ cancel()++ var res ParseResult+ select {+ case res = <-done:+ case <-time.After(2 * time.Second):+ releaseHandler()+ t.Fatal("parsePDFWithSoMark returned nothing after cancellation: the in-flight poll kept running")+ }+ releaseHandler()++ if !errors.Is(res.Err, context.Canceled) {+ t.Fatalf("error = %v, want context.Canceled from the in-flight poll", res.Err)+ }+ if got := pollHits.Load(); got != 1 {+ t.Fatalf("poll requests = %d, want the single in-flight request aborted", got)+ }+}++func TestSoMarkPoll_CancelledContextStopsRequest(t *testing.T) {+ withSSRFBypass(t)+ var hits int+ server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {+ hits+++ _, _ = w.Write([]byte(`{"code":0,"data":{"status":"RUNNING"}}`))+ }))+ defer server.Close()++ ctx, cancel := context.WithCancel(t.Context())+ cancel()++ _, err := soMarkPoll(ctx, server.URL, "task-1", "key")+ if !errors.Is(err, context.Canceled) {+ t.Fatalf("error = %v, want context.Canceled in chain", err)+ }+ if hits != 0 {+ t.Fatalf("server requests = %d, want 0 after cancellation", hits)+ }+}++func TestParsePDFWithDocling_CancelledContextStopsBeforeFirstEndpoint(t *testing.T) {+ withSSRFBypass(t)+ var hits int+ server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {+ hits+++ w.WriteHeader(http.StatusUnprocessableEntity)+ }))+ defer server.Close()++ ctx, cancel := context.WithCancel(t.Context())+ cancel()++ p := NewPDFParser()+ p.DoclingServerURL = server.URL+ res := parsePDFWithDocling(ctx, "sample.pdf", []byte("%PDF-1.4"), p)+ if res.Err == nil {+ t.Fatalf("error = nil, want the four endpoint attempts to fail on cancellation")+ }+ // The loop tries four endpoints; every one must fail with the cancelled+ // context rather than reaching the server, proving ctx is threaded.+ if hits != 0 {+ t.Fatalf("server requests = %d, want 0 after cancellation", hits)+ }+ if !strings.Contains(res.Err.Error(), "context canceled") {+ t.Fatalf("error = %v, want context canceled in chain", res.Err)+ }+}++// Cancelling during the first Docling attempt must abort that request and keep+// the remaining endpoint attempts from reaching the server. The parser has to+// return before the held request is released: releasing on cancel would let a+// call that ignored ctx read the released response, and its later endpoint+// attempts would then supply the context.Canceled error the assertion looks+// for.+func TestParsePDFWithDocling_CancelDuringRequestAbortsItAndSkipsRetries(t *testing.T) {+ withSSRFBypass(t)+ server, hits, entered, release := holdRequest(t)+ var releaseOnce sync.Once+ releaseHandler := func() { releaseOnce.Do(func() { close(release) }) }+ defer releaseHandler()++ ctx, cancel := context.WithCancel(t.Context())+ defer cancel()++ done := make(chan ParseResult, 1)+ go func() {+ p := NewPDFParser()+ p.DoclingServerURL = server.URL+ done <- parsePDFWithDocling(ctx, "sample.pdf", []byte("%PDF-1.4"), p)+ }()++ select {+ case <-entered:+ case <-time.After(2 * time.Second):+ t.Fatal("the first Docling request never reached the server")+ }+ cancel()++ var res ParseResult+ select {+ case res = <-done:+ case <-time.After(2 * time.Second):+ releaseHandler()+ t.Fatal("parsePDFWithDocling returned nothing after cancellation: the in-flight request kept running")+ }+ releaseHandler()++ if res.Err == nil {+ t.Fatal("error = nil, want the cancelled request to fail the conversion")+ }+ if !errors.Is(res.Err, context.Canceled) {+ t.Fatalf("error = %v, want context.Canceled in chain", res.Err)+ }+ if got := hits.Load(); got != 1 {+ t.Fatalf("server requests = %d, want only the aborted in-flight attempt", got)+ }+}diff --git a/internal/parser/parser/pdf_parser_somark.go b/internal/parser/parser/pdf_parser_somark.goindex be6b28f439..1008d35e82 100644--- a/internal/parser/parser/pdf_parser_somark.go+++ b/internal/parser/parser/pdf_parser_somark.go@@ -30,11 +30,11 @@ func parsePDFWithSoMark(ctx context.Context, filename string, data []byte, parseif strings.TrimSpace(apiKey) == "" && !parser.UseTenantModelConfig {apiKey = strings.TrimSpace(common.GetEnv(common.EnvSOMarkAPIKey))}- taskID, err := soMarkSubmit(strings.TrimRight(baseURL, "/"), filename, data, parser, apiKey)+ taskID, err := soMarkSubmit(ctx, strings.TrimRight(baseURL, "/"), filename, data, parser, apiKey)if err != nil {return ParseResult{Err: err}}- result, err := soMarkPoll(strings.TrimRight(baseURL, "/"), taskID, apiKey)+ result, err := soMarkPoll(ctx, strings.TrimRight(baseURL, "/"), taskID, apiKey)if err != nil {return ParseResult{Err: err}}@@ -52,7 +52,7 @@ func parsePDFWithSoMark(ctx context.Context, filename string, data []byte, parse// soMarkSubmit uploads a PDF with the supplied credentials and parser options,// applying environment defaults only when no tenant instance was selected.-func soMarkSubmit(baseURL, filename string, data []byte, parser *PDFParser, apiKey string) (string, error) {+func soMarkSubmit(ctx context.Context, baseURL, filename string, data []byte, parser *PDFParser, apiKey string) (string, error) {var body bytes.Bufferwriter := multipart.NewWriter(&body)part, err := writer.CreateFormFile("file", filename)@@ -86,7 +86,7 @@ func soMarkSubmit(baseURL, filename string, data []byte, parser *PDFParser, apiKif err = writer.Close(); err != nil {return "", fmt.Errorf("parser: SoMark finalize form: %w", err)}- req, err := http.NewRequestWithContext(context.Background(), http.MethodPost, baseURL+"/parse/async", &body)+ req, err := http.NewRequestWithContext(ctx, http.MethodPost, baseURL+"/parse/async", &body)if err != nil {return "", fmt.Errorf("parser: SoMark request: %w", err)}@@ -119,12 +119,12 @@ func soMarkSubmit(baseURL, filename string, data []byte, parser *PDFParser, apiKreturn payload.Data.TaskID, nil}-func soMarkPoll(baseURL, taskID, apiKey string) (map[string]any, error) {+func soMarkPoll(ctx context.Context, baseURL, taskID, apiKey string) (map[string]any, error) {form := url.Values{"task_id": {taskID}}if apiKey != "" {form.Set("api_key", apiKey)}- req, err := http.NewRequestWithContext(context.Background(), http.MethodPost, baseURL+"/parse/async_check", strings.NewReader(form.Encode()))+ req, err := http.NewRequestWithContext(ctx, http.MethodPost, baseURL+"/parse/async_check", strings.NewReader(form.Encode()))if err != nil {return nil, fmt.Errorf("parser: SoMark poll request: %w", err)}diff --git a/internal/parser/parser/pdf_parser_somark_test.go b/internal/parser/parser/pdf_parser_somark_test.goindex f2dd57d1c7..9362055f33 100644--- a/internal/parser/parser/pdf_parser_somark_test.go+++ b/internal/parser/parser/pdf_parser_somark_test.go@@ -1,6 +1,7 @@package parserimport (+ "context""io""mime/multipart""net/http"@@ -103,7 +104,7 @@ func TestSoMarkSubmitMultipartShape(t *testing.T) {_, _ = w.Write([]byte(`{"code":0,"data":{"task_id":"task-3"}}`))}))defer server.Close()- taskID, err := soMarkSubmit(server.URL, "sample.pdf", []byte("%PDF"), NewPDFParser(), "key")+ taskID, err := soMarkSubmit(context.Background(), server.URL, "sample.pdf", []byte("%PDF"), NewPDFParser(), "key")if err != nil {t.Fatalf("soMarkSubmit: %v", err)}