Skip to content

RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.

fix: clarify compiler timeout errors during ingestion (#21011)

buua4364854a64285c0
diff --git a/internal/ingestion/pipeline/pipeline.go b/internal/ingestion/pipeline/pipeline.go
index 92bd9eb154..9b1ff658ee 100644
--- a/internal/ingestion/pipeline/pipeline.go
+++ b/internal/ingestion/pipeline/pipeline.go
@@ -398,8 +398,13 @@ func formatComponentFailure(componentName string, err error) string {
if err == nil {
return componentName + " Error"
}
- if componentName == "Extractor" && (errors.Is(err, context.DeadlineExceeded) || strings.Contains(strings.ToLower(err.Error()), context.DeadlineExceeded.Error())) {
- return "Extractor: Document extraction timed out while waiting for the configured LLM. Check that the model service is reachable, then retry."
+ if errors.Is(err, context.DeadlineExceeded) || strings.Contains(strings.ToLower(err.Error()), context.DeadlineExceeded.Error()) {
+ switch componentName {
+ case "Extractor":
+ return "Extractor: Document extraction timed out while waiting for the configured LLM. Check that the model service is reachable, then retry."
+ case "Compiler":
+ return "Compiler: Document compilation timed out. Check that the configured model service is reachable and responsive, then retry."
+ }
}
message := err.Error()
diff --git a/internal/ingestion/pipeline/pipeline_test.go b/internal/ingestion/pipeline/pipeline_test.go
index a09cccdf4f..e5970a21bc 100644
--- a/internal/ingestion/pipeline/pipeline_test.go
+++ b/internal/ingestion/pipeline/pipeline_test.go
@@ -18,6 +18,7 @@ package pipeline
import (
"context"
+ "errors"
"fmt"
"strings"
"sync"
@@ -297,6 +298,44 @@ func (r *recordingSink) OnComponentProgress(ctx context.Context, ev ProgressEven
r.events = append(r.events, ev)
}
+func TestPipelineTimeoutFailureProgress(t *testing.T) {
+ const compilerTimeout = "Compiler: Document compilation timed out. Check that the configured model service is reachable and responsive, then retry."
+ for _, tc := range []struct {
+ name string
+ component string
+ err error
+ want string
+ }{
+ {name: "compiler deadline", component: "Compiler", err: context.DeadlineExceeded, want: compilerTimeout},
+ {name: "compiler wrapped response deadline", component: "Compiler", err: fmt.Errorf("failed to read response: %w", context.DeadlineExceeded), want: compilerTimeout},
+ {name: "compiler serialized deadline", component: "Compiler", err: errors.New("failed to read response:context deadline exceeded"), want: compilerTimeout},
+ {name: "compiler ordinary failure", component: "Compiler", err: errors.New("failed to read response: unexpected EOF"), want: "Compiler: failed to read response: unexpected EOF"},
+ {name: "compiler canceled", component: "Compiler", err: context.Canceled, want: "Compiler: context canceled"},
+ {name: "extractor deadline", component: "Extractor", err: context.DeadlineExceeded, want: "Extractor: Document extraction timed out while waiting for the configured LLM. Check that the model service is reachable, then retry."},
+ } {
+ t.Run(tc.name, func(t *testing.T) {
+ sink := &recordingSink{}
+ const nodeID = "stage:one"
+ p := &Pipeline{
+ taskID: "task-timeout", documentID: "doc-timeout", sink: sink,
+ canvas: &canvas.Canvas{Components: map[string]canvas.CanvasComponent{
+ nodeID: {Obj: canvas.CanvasComponentObj{ComponentName: tc.component}},
+ }},
+ }
+ p.componentProgressCallback(t.Context())(runtime.ProgressEvent{
+ Component: nodeID, Phase: runtime.PhaseError, Err: tc.err,
+ })
+ if len(sink.events) != 1 {
+ t.Fatalf("progress events = %d, want 1", len(sink.events))
+ }
+ got := sink.events[0]
+ if got.Message != tc.want || got.Component != nodeID || got.Phase != int(runtime.PhaseError) {
+ t.Fatalf("failure progress = %+v, want component %q and message %q", got, nodeID, tc.want)
+ }
+ })
+ }
+}
+
// TestPipelineRunForwardsProgressToSink verifies the pipeline reports the
// component-total denominator once via OnComponentTotal and each component
// lifecycle event to the injected ProgressSink.
diff --git a/internal/ingestion/service/worker_dispatcher_integration_test.go b/internal/ingestion/service/worker_dispatcher_integration_test.go
index 7eb44a4af2..7ef954e143 100644
--- a/internal/ingestion/service/worker_dispatcher_integration_test.go
+++ b/internal/ingestion/service/worker_dispatcher_integration_test.go
@@ -283,8 +283,13 @@ func TestIntegration_SlowTaskHeartbeatPreventsPrematureRedelivery(t *testing.T)
// and are redelivered by the broker to a successor ingestor and completed.
func TestIntegration_ShutdownRedeliveryRecovery(t *testing.T) {
db := testutil.SetupTestDB(t)
+ sqlDB, err := db.DB()
+ if err != nil {
+ t.Fatalf("get test database: %v", err)
+ }
+ t.Cleanup(func() { _ = sqlDB.Close() })
cleanup := testutil.ReplaceDBForTest(t, db)
- defer cleanup()
+ t.Cleanup(cleanup)
host, port := setupRealNatsCluster(t)
mq := natsengine.NewNatsEngine(host, port)
@@ -315,6 +320,8 @@ func TestIntegration_ShutdownRedeliveryRecovery(t *testing.T) {
ingestor1.heartbeatInterval = 500 * time.Millisecond
taskBStarted := make(chan struct{})
+ releaseTaskB := make(chan struct{})
+ var releaseTaskBOnce sync.Once
var (
taskAExecutions atomic.Int32
taskBExecutions atomic.Int32
@@ -328,13 +335,10 @@ func TestIntegration_ShutdownRedeliveryRecovery(t *testing.T) {
if task.ID == taskIDB {
taskBExecutions.Add(1)
close(taskBStarted)
- // Simulate long-running task interrupted by shutdown
- select {
- case <-time.After(30 * time.Second):
- return nil
- case <-ctx.Done():
- return ctx.Err()
- }
+ // Keep the worker in flight until Stop has abandoned its lease,
+ // as with a native parser that cannot honor cancellation immediately.
+ <-releaseTaskB
+ return ctx.Err()
}
return nil
}
@@ -342,6 +346,15 @@ func TestIntegration_ShutdownRedeliveryRecovery(t *testing.T) {
if err := ingestor1.Start(); err != nil {
t.Fatalf("ingestor1.Start: %v", err)
}
+ t.Cleanup(func() {
+ releaseTaskBOnce.Do(func() { close(releaseTaskB) })
+ stopCtx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
+ defer cancel()
+ ingestor1.Stop(stopCtx)
+ if stopCtx.Err() != nil {
+ t.Error("ingestor1 workers did not exit during cleanup")
+ }
+ })
// Publish taskA and taskB
for _, id := range []string{taskIDA, taskIDB} {
@@ -358,20 +371,48 @@ func TestIntegration_ShutdownRedeliveryRecovery(t *testing.T) {
}
// Wait for taskB to start execution and taskA to complete
- <-taskBStarted
+ select {
+ case <-taskBStarted:
+ case <-time.After(5 * time.Second):
+ t.Fatal("taskB execution did not start")
+ }
deadlineA := time.Now().Add(5 * time.Second)
+ taskACompleted := false
for time.Now().Before(deadlineA) {
var taskA entity.IngestionTask
if err := db.Where("id = ?", taskIDA).First(&taskA).Error; err == nil && taskA.Status == common.COMPLETED {
+ taskACompleted = true
break
}
time.Sleep(50 * time.Millisecond)
}
+ if !taskACompleted {
+ t.Fatal("taskA did not complete before shutdown")
+ }
// Simulate SIGTERM with 200ms graceful shutdown timeout (which will time out for taskB)
stopCtx, stopCancel := context.WithTimeout(context.Background(), 200*time.Millisecond)
defer stopCancel()
ingestor1.Stop(stopCtx)
+ if !ingestor1.stopLeases.Load() {
+ t.Fatal("shutdown did not abandon the in-flight broker lease")
+ }
+ // Join the abandoned worker before starting its successor. It must leave
+ // taskB RUNNING and unacknowledged so that the broker can redeliver it.
+ releaseTaskBOnce.Do(func() { close(releaseTaskB) })
+ joinCtx, joinCancel := context.WithTimeout(context.Background(), 5*time.Second)
+ ingestor1.Stop(joinCtx)
+ joinCancel()
+ if joinCtx.Err() == context.DeadlineExceeded {
+ t.Fatal("abandoned ingestor1 worker did not exit")
+ }
+ var interruptedTask entity.IngestionTask
+ if err := db.Where("id = ?", taskIDB).First(&interruptedTask).Error; err != nil {
+ t.Fatalf("read interrupted taskB: %v", err)
+ }
+ if interruptedTask.Status != common.RUNNING {
+ t.Fatalf("interrupted taskB status = %s, want %s", interruptedTask.Status, common.RUNNING)
+ }
// Ingestor1 timed out and cancelled taskB, calling stopActiveLeases().
// Now start ingestor2 to recover un-acked tasks from the shared consumer.
@@ -412,10 +453,16 @@ func TestIntegration_ShutdownRedeliveryRecovery(t *testing.T) {
}
if !taskBCompleted {
- t.Fatalf("taskB was not redelivered and completed after ingestor1 shutdown")
+ var taskB entity.IngestionTask
+ _ = db.Where("id = ?", taskIDB).First(&taskB).Error
+ t.Fatalf("taskB was not redelivered and completed after ingestor1 shutdown: status=%s, original executions=%d, recovery executions=%d",
+ taskB.Status, taskBExecutions.Load(), taskBExec2.Load())
}
// Assertions:
+ if taskAExecutions.Load() != 1 || taskBExecutions.Load() != 1 {
+ t.Fatalf("original executions: taskA=%d taskB=%d, want one each", taskAExecutions.Load(), taskBExecutions.Load())
+ }
// 1. taskA was settled by ingestor1 and NEVER executed by ingestor2 (no duplicate delivery)
if taskAExec2.Load() != 0 {
t.Fatalf("taskA was redelivered to ingestor2 (%d times), but it was already completed and Acked", taskAExec2.Load())