RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.
Go agent-harnessagent-native-databaseagentic-aiagentic-retrievalagentic-searchaiai-agentscontext-enginecontext-engineeringcontext-managementharness-engineeringknowledge-compilationragretrieval-augmented-generationretrieval-harnesssearch-harness
fix: clarify compiler timeout errors during ingestion (#21011)
buua4364854a64285c0
diff --git a/internal/ingestion/pipeline/pipeline.go b/internal/ingestion/pipeline/pipeline.goindex 92bd9eb154..9b1ff658ee 100644--- a/internal/ingestion/pipeline/pipeline.go+++ b/internal/ingestion/pipeline/pipeline.go@@ -398,8 +398,13 @@ func formatComponentFailure(componentName string, err error) string {if err == nil {return componentName + " Error"}- if componentName == "Extractor" && (errors.Is(err, context.DeadlineExceeded) || strings.Contains(strings.ToLower(err.Error()), context.DeadlineExceeded.Error())) {- return "Extractor: Document extraction timed out while waiting for the configured LLM. Check that the model service is reachable, then retry."+ if errors.Is(err, context.DeadlineExceeded) || strings.Contains(strings.ToLower(err.Error()), context.DeadlineExceeded.Error()) {+ switch componentName {+ case "Extractor":+ return "Extractor: Document extraction timed out while waiting for the configured LLM. Check that the model service is reachable, then retry."+ case "Compiler":+ return "Compiler: Document compilation timed out. Check that the configured model service is reachable and responsive, then retry."+ }}message := err.Error()diff --git a/internal/ingestion/pipeline/pipeline_test.go b/internal/ingestion/pipeline/pipeline_test.goindex a09cccdf4f..e5970a21bc 100644--- a/internal/ingestion/pipeline/pipeline_test.go+++ b/internal/ingestion/pipeline/pipeline_test.go@@ -18,6 +18,7 @@ package pipelineimport ("context"+ "errors""fmt""strings""sync"@@ -297,6 +298,44 @@ func (r *recordingSink) OnComponentProgress(ctx context.Context, ev ProgressEvenr.events = append(r.events, ev)}+func TestPipelineTimeoutFailureProgress(t *testing.T) {+ const compilerTimeout = "Compiler: Document compilation timed out. Check that the configured model service is reachable and responsive, then retry."+ for _, tc := range []struct {+ name string+ component string+ err error+ want string+ }{+ {name: "compiler deadline", component: "Compiler", err: context.DeadlineExceeded, want: compilerTimeout},+ {name: "compiler wrapped response deadline", component: "Compiler", err: fmt.Errorf("failed to read response: %w", context.DeadlineExceeded), want: compilerTimeout},+ {name: "compiler serialized deadline", component: "Compiler", err: errors.New("failed to read response:context deadline exceeded"), want: compilerTimeout},+ {name: "compiler ordinary failure", component: "Compiler", err: errors.New("failed to read response: unexpected EOF"), want: "Compiler: failed to read response: unexpected EOF"},+ {name: "compiler canceled", component: "Compiler", err: context.Canceled, want: "Compiler: context canceled"},+ {name: "extractor deadline", component: "Extractor", err: context.DeadlineExceeded, want: "Extractor: Document extraction timed out while waiting for the configured LLM. Check that the model service is reachable, then retry."},+ } {+ t.Run(tc.name, func(t *testing.T) {+ sink := &recordingSink{}+ const nodeID = "stage:one"+ p := &Pipeline{+ taskID: "task-timeout", documentID: "doc-timeout", sink: sink,+ canvas: &canvas.Canvas{Components: map[string]canvas.CanvasComponent{+ nodeID: {Obj: canvas.CanvasComponentObj{ComponentName: tc.component}},+ }},+ }+ p.componentProgressCallback(t.Context())(runtime.ProgressEvent{+ Component: nodeID, Phase: runtime.PhaseError, Err: tc.err,+ })+ if len(sink.events) != 1 {+ t.Fatalf("progress events = %d, want 1", len(sink.events))+ }+ got := sink.events[0]+ if got.Message != tc.want || got.Component != nodeID || got.Phase != int(runtime.PhaseError) {+ t.Fatalf("failure progress = %+v, want component %q and message %q", got, nodeID, tc.want)+ }+ })+ }+}+// TestPipelineRunForwardsProgressToSink verifies the pipeline reports the// component-total denominator once via OnComponentTotal and each component// lifecycle event to the injected ProgressSink.diff --git a/internal/ingestion/service/worker_dispatcher_integration_test.go b/internal/ingestion/service/worker_dispatcher_integration_test.goindex 7eb44a4af2..7ef954e143 100644--- a/internal/ingestion/service/worker_dispatcher_integration_test.go+++ b/internal/ingestion/service/worker_dispatcher_integration_test.go@@ -283,8 +283,13 @@ func TestIntegration_SlowTaskHeartbeatPreventsPrematureRedelivery(t *testing.T)// and are redelivered by the broker to a successor ingestor and completed.func TestIntegration_ShutdownRedeliveryRecovery(t *testing.T) {db := testutil.SetupTestDB(t)+ sqlDB, err := db.DB()+ if err != nil {+ t.Fatalf("get test database: %v", err)+ }+ t.Cleanup(func() { _ = sqlDB.Close() })cleanup := testutil.ReplaceDBForTest(t, db)- defer cleanup()+ t.Cleanup(cleanup)host, port := setupRealNatsCluster(t)mq := natsengine.NewNatsEngine(host, port)@@ -315,6 +320,8 @@ func TestIntegration_ShutdownRedeliveryRecovery(t *testing.T) {ingestor1.heartbeatInterval = 500 * time.MillisecondtaskBStarted := make(chan struct{})+ releaseTaskB := make(chan struct{})+ var releaseTaskBOnce sync.Oncevar (taskAExecutions atomic.Int32taskBExecutions atomic.Int32@@ -328,13 +335,10 @@ func TestIntegration_ShutdownRedeliveryRecovery(t *testing.T) {if task.ID == taskIDB {taskBExecutions.Add(1)close(taskBStarted)- // Simulate long-running task interrupted by shutdown- select {- case <-time.After(30 * time.Second):- return nil- case <-ctx.Done():- return ctx.Err()- }+ // Keep the worker in flight until Stop has abandoned its lease,+ // as with a native parser that cannot honor cancellation immediately.+ <-releaseTaskB+ return ctx.Err()}return nil}@@ -342,6 +346,15 @@ func TestIntegration_ShutdownRedeliveryRecovery(t *testing.T) {if err := ingestor1.Start(); err != nil {t.Fatalf("ingestor1.Start: %v", err)}+ t.Cleanup(func() {+ releaseTaskBOnce.Do(func() { close(releaseTaskB) })+ stopCtx, cancel := context.WithTimeout(context.Background(), 5*time.Second)+ defer cancel()+ ingestor1.Stop(stopCtx)+ if stopCtx.Err() != nil {+ t.Error("ingestor1 workers did not exit during cleanup")+ }+ })// Publish taskA and taskBfor _, id := range []string{taskIDA, taskIDB} {@@ -358,20 +371,48 @@ func TestIntegration_ShutdownRedeliveryRecovery(t *testing.T) {}// Wait for taskB to start execution and taskA to complete- <-taskBStarted+ select {+ case <-taskBStarted:+ case <-time.After(5 * time.Second):+ t.Fatal("taskB execution did not start")+ }deadlineA := time.Now().Add(5 * time.Second)+ taskACompleted := falsefor time.Now().Before(deadlineA) {var taskA entity.IngestionTaskif err := db.Where("id = ?", taskIDA).First(&taskA).Error; err == nil && taskA.Status == common.COMPLETED {+ taskACompleted = truebreak}time.Sleep(50 * time.Millisecond)}+ if !taskACompleted {+ t.Fatal("taskA did not complete before shutdown")+ }// Simulate SIGTERM with 200ms graceful shutdown timeout (which will time out for taskB)stopCtx, stopCancel := context.WithTimeout(context.Background(), 200*time.Millisecond)defer stopCancel()ingestor1.Stop(stopCtx)+ if !ingestor1.stopLeases.Load() {+ t.Fatal("shutdown did not abandon the in-flight broker lease")+ }+ // Join the abandoned worker before starting its successor. It must leave+ // taskB RUNNING and unacknowledged so that the broker can redeliver it.+ releaseTaskBOnce.Do(func() { close(releaseTaskB) })+ joinCtx, joinCancel := context.WithTimeout(context.Background(), 5*time.Second)+ ingestor1.Stop(joinCtx)+ joinCancel()+ if joinCtx.Err() == context.DeadlineExceeded {+ t.Fatal("abandoned ingestor1 worker did not exit")+ }+ var interruptedTask entity.IngestionTask+ if err := db.Where("id = ?", taskIDB).First(&interruptedTask).Error; err != nil {+ t.Fatalf("read interrupted taskB: %v", err)+ }+ if interruptedTask.Status != common.RUNNING {+ t.Fatalf("interrupted taskB status = %s, want %s", interruptedTask.Status, common.RUNNING)+ }// Ingestor1 timed out and cancelled taskB, calling stopActiveLeases().// Now start ingestor2 to recover un-acked tasks from the shared consumer.@@ -412,10 +453,16 @@ func TestIntegration_ShutdownRedeliveryRecovery(t *testing.T) {}if !taskBCompleted {- t.Fatalf("taskB was not redelivered and completed after ingestor1 shutdown")+ var taskB entity.IngestionTask+ _ = db.Where("id = ?", taskIDB).First(&taskB).Error+ t.Fatalf("taskB was not redelivered and completed after ingestor1 shutdown: status=%s, original executions=%d, recovery executions=%d",+ taskB.Status, taskBExecutions.Load(), taskBExec2.Load())}// Assertions:+ if taskAExecutions.Load() != 1 || taskBExecutions.Load() != 1 {+ t.Fatalf("original executions: taskA=%d taskB=%d, want one each", taskAExecutions.Load(), taskBExecutions.Load())+ }// 1. taskA was settled by ingestor1 and NEVER executed by ingestor2 (no duplicate delivery)if taskAExec2.Load() != 0 {t.Fatalf("taskA was redelivered to ingestor2 (%d times), but it was already completed and Acked", taskAExec2.Load())