Open-source LLM knowledge platform: turn raw documents into a queryable RAG, an autonomous reasoning agent, and a self-maintaining Wiki.
Go agentagenticaichatbotdsh-pluginembeddingsevaluationgenerative-aigolangknowledge-basellmmulti-tenantollamaopenaiquestion-answeringragrerankingsemantic-searchvector-searchwiki
fix(knowledge): leave the root span open while a retry is left (#4357)
The one -Niuf5cfa0ff5fa8
diff --git a/internal/application/service/knowledge.go b/internal/application/service/knowledge.goindex c7126a272..7a3376733 100644--- a/internal/application/service/knowledge.go+++ b/internal/application/service/knowledge.go@@ -331,6 +331,18 @@ func (s *knowledgeService) endStage(ctx context.Context, kid, name string, outpu}func (s *knowledgeService) failStage(ctx context.Context, kid, name, code, msg string, err error) {+ s.failStageWithRetry(ctx, kid, name, code, msg, err, false)+}++// failStageRetryable is for failures returned to the task runner. Remaining+// retry budget alone is insufficient: terminal failures acknowledge the task.+func (s *knowledgeService) failStageRetryable(ctx context.Context, kid, name, code, msg string, err error) {+ s.failStageWithRetry(ctx, kid, name, code, msg, err, true)+}++func (s *knowledgeService) failStageWithRetry(+ ctx context.Context, kid, name, code, msg string, err error, retryable bool,+) {a := attemptFromCtx(ctx)if a <= 0 {return@@ -339,7 +351,11 @@ func (s *knowledgeService) failStage(ctx context.Context, kid, name, code, msg sif span == nil {return}- s.tracker().FailSpan(ctx, span, code, msg, err)+ if retryable {+ s.tracker().FailSpanRetryable(ctx, span, code, msg, err)+ } else {+ s.tracker().FailSpan(ctx, span, code, msg, err)+ }}func (s *knowledgeService) skipStage(ctx context.Context, kid, name, reason string) {diff --git a/internal/application/service/knowledge_post_process.go b/internal/application/service/knowledge_post_process.goindex 67b9f90b4..1f889e504 100644--- a/internal/application/service/knowledge_post_process.go+++ b/internal/application/service/knowledge_post_process.go@@ -274,7 +274,7 @@ func (s *KnowledgePostProcessService) Handle(ctx context.Context, task *asynq.Ta// enqueue its KB-scoped trigger. Retry only the trigger: appending a// second pending op would duplicate durable work and its finalizer.if err := enqueueWikiIngestTrigger(ctx, s.taskEnqueuer, payload.TenantID, payload.KnowledgeBaseID); err != nil {- s.tracker().FailSpan(ctx, postSpan, "WIKI_TRIGGER_ENQUEUE_FAILED", err.Error(), err)+ s.tracker().FailSpanRetryable(ctx, postSpan, "WIKI_TRIGGER_ENQUEUE_FAILED", err.Error(), err)return fmt.Errorf("retry wiki ingest trigger: %w", err)}logger.Infof(ctx, "[KnowledgePostProcess] Re-enqueued wiki ingest trigger for %s", payload.KnowledgeID)@@ -303,7 +303,7 @@ func (s *KnowledgePostProcessService) Handle(ctx context.Context, task *asynq.Tacase expectedSubtasks == 0:completed, err := s.knowledgeRepo.CompleteProcessingWithoutSubtasks(ctx, payload.KnowledgeID)if err != nil {- s.tracker().FailSpan(ctx, postSpan, "COMPLETION_FAILED", err.Error(), err)+ s.tracker().FailSpanRetryable(ctx, postSpan, "COMPLETION_FAILED", err.Error(), err)return fmt.Errorf("complete knowledge without enrichment: %w", err)}if !completed {@@ -340,7 +340,7 @@ func (s *KnowledgePostProcessService) Handle(ctx context.Context, task *asynq.Ta// strand the row; retry so the handoff (or dead-letter) happens.logger.Warnf(ctx, "[KnowledgePostProcess] SetFinalizing failed for %s: %v",payload.KnowledgeID, err)- s.tracker().FailSpan(ctx, postSpan, "FINALIZING_HANDOFF_FAILED", err.Error(), err)+ s.tracker().FailSpanRetryable(ctx, postSpan, "FINALIZING_HANDOFF_FAILED", err.Error(), err)return fmt.Errorf("enter finalizing: %w", err)}if promoted {diff --git a/internal/application/service/knowledge_post_process_summary_test.go b/internal/application/service/knowledge_post_process_summary_test.goindex 1c1e65dfc..5e4e9f76b 100644--- a/internal/application/service/knowledge_post_process_summary_test.go+++ b/internal/application/service/knowledge_post_process_summary_test.go@@ -145,9 +145,14 @@ func TestKnowledgePostProcessWithoutSummaryCompletionFailure(t *testing.T) {wikiEnqueueFailureKnowledgeRepo: wikiEnqueueFailureKnowledgeRepo{knowledge: knowledge},completionErr: tc.err, completionSkipped: tc.skipped,}+ tracker, db := setupSpanTrackerTest(t)+ ctx := types.WithTaskRetryMetadata(context.Background(), 0, 3)+ _, attempt, err := tracker.OpenAttempt(ctx, knowledge.ID, "")+ require.NoError(t, err)queue := &wikiEnqueueFailureTaskQueue{}svc := &KnowledgePostProcessService{knowledgeRepo: repo,+ spanTracker: tracker,kbService: &wikiEnqueueFailureKBService{kb: &types.KnowledgeBase{ID: "kb", TenantID: 7, Type: types.KnowledgeBaseTypeDocument,AutoTagConfig: &types.AutoTagConfig{Enabled: true},@@ -162,15 +167,24 @@ func TestKnowledgePostProcessWithoutSummaryCompletionFailure(t *testing.T) {TenantID: 7, KnowledgeID: "doc", KnowledgeBaseID: "kb",})require.NoError(t, err)- err = svc.Handle(context.Background(), asynq.NewTask(types.TypeKnowledgePostProcess, payload))+ err = svc.Handle(ctx, asynq.NewTask(types.TypeKnowledgePostProcess, payload))require.ErrorIs(t, err, tc.err)require.Empty(t, queue.taskTypes)require.Equal(t, types.ParseStatusProcessing, knowledge.ParseStatus)if tc.err != nil {+ status, code, message := rootSpan(t, db, knowledge.ID, attempt)+ require.Equal(t, types.SpanStatusRunning, status)+ require.Empty(t, code)+ require.Empty(t, message)repo.completionErr = nil+ ctx = types.WithTaskRetryMetadata(ctx, 1, 3)task := asynq.NewTask(types.TypeKnowledgePostProcess, payload)- require.NoError(t, svc.Handle(context.Background(), task))+ require.NoError(t, svc.Handle(ctx, task))require.Equal(t, types.ParseStatusCompleted, knowledge.ParseStatus)+ status, code, message = rootSpan(t, db, knowledge.ID, attempt)+ require.Equal(t, types.SpanStatusDone, status)+ require.Empty(t, code)+ require.Empty(t, message)require.Equal(t, []string{types.TypeKnowledgeAutoTag}, queue.taskTypes)}})diff --git a/internal/application/service/knowledge_process.go b/internal/application/service/knowledge_process.goindex 96e51b57e..7ffd22f00 100644--- a/internal/application/service/knowledge_process.go+++ b/internal/application/service/knowledge_process.go@@ -769,11 +769,13 @@ func (s *knowledgeService) processChunks(ctx context.Context,if isLikelyRateLimitError(err) {code = werrors.ErrCodeEmbeddingRateLimit}- s.failStage(ctx, knowledge.ID, types.StageEmbedding,- code, "batch index failed", err)if willRetry {+ s.failStageRetryable(ctx, knowledge.ID, types.StageEmbedding,+ code, "batch index failed", err)return fmt.Errorf("batch index: %w", err)}+ s.failStage(ctx, knowledge.ID, types.StageEmbedding,+ code, "batch index failed", err)return nil}logger.GetLogger(ctx).Infof("processChunks batch index successfully, with %d index", len(indexInfoList))@@ -4241,16 +4243,18 @@ func (s *knowledgeService) convert(if !isURL {fileReader, err := s.resolveFileServiceForPath(ctx, kb, payload.FilePath).GetFile(ctx, payload.FilePath)if err != nil {- s.failStage(ctx, knowledge.ID, types.StageDocReader,- werrors.ErrCodeDocReaderParseFailed, "failed to get file", err)- return s.failKnowledge(ctx, knowledge, isLastRetry, "failed to get file: %v", err)+ result, taskErr := s.failKnowledge(ctx, knowledge, isLastRetry, "failed to get file: %v", err)+ s.failStageWithRetry(ctx, knowledge.ID, types.StageDocReader,+ werrors.ErrCodeDocReaderParseFailed, "failed to get file", err, taskErr != nil)+ return result, taskErr}defer fileReader.Close()contentBytes, err := io.ReadAll(fileReader)if err != nil {- s.failStage(ctx, knowledge.ID, types.StageDocReader,- werrors.ErrCodeDocReaderParseFailed, "failed to read file", err)- return s.failKnowledge(ctx, knowledge, isLastRetry, "failed to read file: %v", err)+ result, taskErr := s.failKnowledge(ctx, knowledge, isLastRetry, "failed to read file: %v", err)+ s.failStageWithRetry(ctx, knowledge.ID, types.StageDocReader,+ werrors.ErrCodeDocReaderParseFailed, "failed to read file", err, taskErr != nil)+ return result, taskErr}req.FileContent = contentBytesreq.FileName = payload.FileName@@ -4261,9 +4265,10 @@ func (s *knowledgeService) convert(if err != nil {code, message := docReaderFailure(err)logger.Errorf(ctx, "[convert] DocReader failed knowledge=%s code=%s: %v", knowledge.ID, code, err)- s.failStage(ctx, knowledge.ID, types.StageDocReader,- code, message, err)- return s.failKnowledge(ctx, knowledge, isLastRetry, "%s", message)+ result, taskErr := s.failKnowledge(ctx, knowledge, isLastRetry, "%s", message)+ s.failStageWithRetry(ctx, knowledge.ID, types.StageDocReader,+ code, message, err, taskErr != nil)+ return result, taskErr}sanitizeReadResult(result)if result.Error != "" {diff --git a/internal/application/service/knowledge_process_embedding_failure_test.go b/internal/application/service/knowledge_process_embedding_failure_test.goindex 238ed02fa..18842bafd 100644--- a/internal/application/service/knowledge_process_embedding_failure_test.go+++ b/internal/application/service/knowledge_process_embedding_failure_test.go@@ -7,6 +7,7 @@ import ("strings""testing"+ werrors "github.com/Tencent/WeKnora/internal/errors""github.com/Tencent/WeKnora/internal/models/embedding""github.com/Tencent/WeKnora/internal/types""github.com/Tencent/WeKnora/internal/types/interfaces"@@ -64,6 +65,8 @@ func (s embedFailureModelService) GetEmbeddingModel(// attempt to keep processing past the missing embedder would panic and fail// this test, which is exactly the regression worth catching.func TestProcessChunksFailsKnowledgeWhenEmbeddingModelUnavailable(t *testing.T) {+ tracker, db := setupSpanTrackerTest(t)+ ctx := types.WithTaskRetryMetadata(context.Background(), 0, 3)modelErr := errors.New("embedding model not found")knowledge := &types.Knowledge{ID: "knowledge-1",@@ -75,6 +78,7 @@ func TestProcessChunksFailsKnowledgeWhenEmbeddingModelUnavailable(t *testing.T)svc := &knowledgeService{repo: repo,modelService: embedFailureModelService{err: modelErr},+ spanTracker: tracker,}kb := &types.KnowledgeBase{ID: "kb-1",@@ -83,8 +87,12 @@ func TestProcessChunksFailsKnowledgeWhenEmbeddingModelUnavailable(t *testing.T)IndexingStrategy: types.IndexingStrategy{VectorEnabled: true},}- svc.processChunks(context.Background(), kb, knowledge,+ _, attempt, err := tracker.OpenAttempt(ctx, knowledge.ID, "")+ require.NoError(t, err)+ ctx = withAttempt(ctx, attempt)+ err = svc.processChunks(ctx, kb, knowledge,[]types.ParsedChunk{{Content: "body", Seq: 0, Start: 0, End: 4}})+ require.NoError(t, err, "the task is acknowledged even though retry budget remains")require.Len(t, repo.updates, 1, "the failure must be persisted exactly once")require.Equal(t, types.ParseStatusFailed, repo.updates[0].ParseStatus)@@ -93,6 +101,21 @@ func TestProcessChunksFailsKnowledgeWhenEmbeddingModelUnavailable(t *testing.T)"error_message should name the underlying cause, got %q", repo.updates[0].ErrorMessage,)require.Equal(t, types.ParseStatusFailed, knowledge.ParseStatus)+ status, code, message := rootSpan(t, db, knowledge.ID, attempt)+ require.Equal(t, types.SpanStatusFailed, status)+ require.Equal(t, werrors.ErrCodeEmbeddingProviderFail, code)+ require.Equal(t, knowledge.ErrorMessage, message)++ // Housekeeping must not relabel this settled failure as TASK_STALLED.+ require.NoError(t, db.Exec(knowledgeTestDDL).Error)+ require.NoError(t, db.Exec(+ "INSERT INTO knowledges (id, parse_status, error_message) VALUES (?, ?, ?)",+ knowledge.ID, knowledge.ParseStatus, knowledge.ErrorMessage).Error)+ (&HousekeepingService{db: db}).reapTerminalSpans(ctx)+ status, code, message = rootSpan(t, db, knowledge.ID, attempt)+ require.Equal(t, types.SpanStatusFailed, status)+ require.Equal(t, werrors.ErrCodeEmbeddingProviderFail, code)+ require.Equal(t, knowledge.ErrorMessage, message)}// A cancelled context means the run was interrupted, not that the model isdiff --git a/internal/application/service/knowledge_process_index_retry_test.go b/internal/application/service/knowledge_process_index_retry_test.goindex 89bcf7379..e6703a723 100644--- a/internal/application/service/knowledge_process_index_retry_test.go+++ b/internal/application/service/knowledge_process_index_retry_test.go@@ -5,6 +5,7 @@ import ("errors""testing"+ werrors "github.com/Tencent/WeKnora/internal/errors""github.com/Tencent/WeKnora/internal/models/embedding""github.com/Tencent/WeKnora/internal/types""github.com/stretchr/testify/require"@@ -48,7 +49,7 @@ func (e *failingIndexEngine) DeleteByKnowledgeIDList(context.Context, []string,}func runProcessChunksWithFailingIndex(- ctx context.Context, t *testing.T,+ ctx context.Context, t *testing.T, tracker SpanTracker,) (*indexRetryKnowledgeRepo, *indexRetryChunkRepo, *failingIndexEngine, error) {t.Helper()knowledge := &types.Knowledge{@@ -70,6 +71,7 @@ func runProcessChunksWithFailingIndex(ctx = context.WithValue(ctx, types.TenantInfoContextKey, tenant)svc := &knowledgeService{repo: repo,+ spanTracker: tracker,chunkRepo: chunks,modelService: parentChildModelService{embedder: parentChildEmbedder{}},retrieveEngine: parentChildRetrieveRegistry{engine: engine},@@ -95,9 +97,17 @@ func runProcessChunksWithFailingIndex(func TestProcessChunksRetriesBatchIndexFailureBeforeLastAttempt(t *testing.T) {ctx := types.WithTaskRetryMetadata(context.Background(), 0, 3)- repo, chunks, engine, err := runProcessChunksWithFailingIndex(ctx, t)+ tracker, db := setupSpanTrackerTest(t)+ _, attempt, err := tracker.OpenAttempt(ctx, "knowledge-1", "")+ require.NoError(t, err)+ ctx = withAttempt(ctx, attempt)+ repo, chunks, engine, err := runProcessChunksWithFailingIndex(ctx, t, tracker)require.ErrorContains(t, err, "embedding endpoint unavailable")+ status, code, message := rootSpan(t, db, "knowledge-1", attempt)+ require.Equal(t, types.SpanStatusRunning, status)+ require.Empty(t, code)+ require.Empty(t, message)for _, update := range repo.updates {require.NotEqual(t, types.ParseStatusFailed, update.ParseStatus)}@@ -109,9 +119,17 @@ func TestProcessChunksRetriesBatchIndexFailureBeforeLastAttempt(t *testing.T) {func TestProcessChunksFailsKnowledgeOnLastBatchIndexAttempt(t *testing.T) {ctx := types.WithTaskRetryMetadata(context.Background(), 3, 3)- repo, chunks, engine, err := runProcessChunksWithFailingIndex(ctx, t)+ tracker, db := setupSpanTrackerTest(t)+ _, attempt, err := tracker.OpenAttempt(ctx, "knowledge-1", "")+ require.NoError(t, err)+ ctx = withAttempt(ctx, attempt)+ repo, chunks, engine, err := runProcessChunksWithFailingIndex(ctx, t, tracker)require.NoError(t, err)+ status, code, message := rootSpan(t, db, "knowledge-1", attempt)+ require.Equal(t, types.SpanStatusFailed, status)+ require.Equal(t, werrors.ErrCodeVectorStoreWriteFailed, code)+ require.Equal(t, "batch index failed", message)require.NotEmpty(t, repo.updates)last := repo.updates[len(repo.updates)-1]require.Equal(t, types.ParseStatusFailed, last.ParseStatus)diff --git a/internal/application/service/knowledge_process_span_retry_test.go b/internal/application/service/knowledge_process_span_retry_test.gonew file mode 100644index 000000000..2eca9adc4--- /dev/null+++ b/internal/application/service/knowledge_process_span_retry_test.go@@ -0,0 +1,113 @@+package service++import (+ "context"+ "errors"+ "io"+ "strings"+ "testing"+ "testing/iotest"++ werrors "github.com/Tencent/WeKnora/internal/errors"+ "github.com/Tencent/WeKnora/internal/types"+ "github.com/Tencent/WeKnora/internal/types/interfaces"+ "github.com/stretchr/testify/require"+)++type spanRetryDocReader struct {+ interfaces.DocumentReader+ result *types.ReadResult+ err error+}++func (r *spanRetryDocReader) Read(context.Context, *types.ReadRequest) (*types.ReadResult, error) {+ return r.result, r.err+}++type spanRetryFileService struct {+ interfaces.FileService+ getErr, readErr error+}++func (s spanRetryFileService) GetFile(context.Context, string) (io.ReadCloser, error) {+ if s.getErr != nil {+ return nil, s.getErr+ }+ if s.readErr != nil {+ return io.NopCloser(iotest.ErrReader(s.readErr)), nil+ }+ return io.NopCloser(strings.NewReader("document")), nil+}++// Exercise convert's actual return value: a parser response with Error is+// acknowledged, whereas transport/file errors are returned for queue retry.+func TestConvertFailureSpanMatchesTaskOutcome(t *testing.T) {+ transient := errors.New("temporary read failure")+ for _, tc := range []struct {+ name string+ missingReader bool+ parserError string+ getErr, readErr, rpcErr error+ retryable bool+ code string+ }{+ {name: "reader unavailable", missingReader: true, code: werrors.ErrCodeDocReaderUnavailable},+ {+ name: "parser rejects document", parserError: "unsupported document",+ code: werrors.ErrCodeDocReaderParseFailed,+ },+ {name: "file fetch fails", getErr: transient, retryable: true, code: werrors.ErrCodeDocReaderParseFailed},+ {name: "file read fails", readErr: transient, retryable: true, code: werrors.ErrCodeDocReaderParseFailed},+ {name: "reader call fails", rpcErr: transient, retryable: true, code: werrors.ErrCodeDocReaderParseFailed},+ } {+ for _, retried := range []int{0, 3} {+ name := tc.name + "/retry_available"+ if retried == 3 {+ name = tc.name + "/last_retry"+ }+ t.Run(name, func(t *testing.T) {+ tracker, db := setupSpanTrackerTest(t)+ ctx := types.WithTaskRetryMetadata(context.Background(), retried, 3)+ knowledge := &types.Knowledge{+ ID: "doc", TenantID: 1, KnowledgeBaseID: "kb", FilePath: "doc.pdf",+ ParseStatus: types.ParseStatusProcessing,+ }+ _, attempt, err := tracker.OpenAttempt(ctx, knowledge.ID, "")+ require.NoError(t, err)+ ctx = withAttempt(ctx, attempt)+ svc := &knowledgeService{+ repo: &embedFailureKnowledgeRepo{knowledge: knowledge},+ spanTracker: tracker,+ tenantService: &stubTenantServiceForModelDelete{},+ fileSvc: spanRetryFileService{getErr: tc.getErr, readErr: tc.readErr},+ }+ if !tc.missingReader {+ svc.documentReader = &spanRetryDocReader{+ result: &types.ReadResult{Error: tc.parserError}, err: tc.rpcErr,+ }+ }+ result, err := svc.convert(ctx, types.DocumentProcessPayload{+ FilePath: "doc.pdf", FileName: "doc.pdf", FileType: "pdf",+ }, &types.KnowledgeBase{ID: "kb"}, knowledge, types.EffectiveProcessConfig{}, retried == 3)+ require.Nil(t, result)+ if tc.retryable {+ require.Error(t, err)+ } else {+ require.NoError(t, err)+ }+ status, code, message := rootSpan(t, db, knowledge.ID, attempt)+ if tc.retryable && retried < 3 {+ require.Equal(t, types.ParseStatusProcessing, knowledge.ParseStatus)+ require.Equal(t, types.SpanStatusRunning, status)+ require.Empty(t, code)+ require.Empty(t, message)+ } else {+ require.Equal(t, types.ParseStatusFailed, knowledge.ParseStatus)+ require.Equal(t, types.SpanStatusFailed, status)+ require.Equal(t, tc.code, code)+ require.NotEmpty(t, message)+ }+ })+ }+ }+}diff --git a/internal/application/service/knowledge_span_tracker.go b/internal/application/service/knowledge_span_tracker.goindex 100c27d07..f5bac889f 100644--- a/internal/application/service/knowledge_span_tracker.go+++ b/internal/application/service/knowledge_span_tracker.go@@ -36,6 +36,7 @@ import ("github.com/Tencent/WeKnora/internal/logger""github.com/Tencent/WeKnora/internal/types""github.com/google/uuid"+ "github.com/hibiken/asynq""gorm.io/gorm")@@ -110,9 +111,14 @@ type SpanTracker interface {// FailSpan marks span as failed and cascade-cancels its// descendants. errorDetail (a Go error) is recorded verbatim in- // error_detail (truncated to 8 KB) for admin views.+ // error_detail (truncated to 8 KB) for admin views. Main pipeline stage+ // failures close the root even when unused retry budget remains.FailSpan(ctx context.Context, span *Span, errorCode, errorMessage string, errorDetail error)+ // FailSpanRetryable records a failure that the caller will return to the+ // task runner for retry. It leaves the root open only while retries remain.+ FailSpanRetryable(ctx context.Context, span *Span, errorCode, errorMessage string, errorDetail error)+// SkipSpan marks an intentionally not-run span (e.g. multimodal// on a text-only document). Distinct from cancelled — skipped is// "we chose not to" while cancelled is "an upstream broke".@@ -493,6 +499,18 @@ func (t *spanTracker) EndSpan(ctx context.Context, span *Span, output types.JSON}func (t *spanTracker) FailSpan(ctx context.Context, span *Span, errorCode, errorMessage string, errorDetail error) {+ t.failSpan(ctx, span, errorCode, errorMessage, errorDetail, false)+}++func (t *spanTracker) FailSpanRetryable(+ ctx context.Context, span *Span, errorCode, errorMessage string, errorDetail error,+) {+ t.failSpan(ctx, span, errorCode, errorMessage, errorDetail, attemptStillHasRetry(ctx))+}++func (t *spanTracker) failSpan(+ ctx context.Context, span *Span, errorCode, errorMessage string, errorDetail error, willRetry bool,+) {if span == nil {return}@@ -540,13 +558,16 @@ func (t *spanTracker) FailSpan(ctx context.Context, span *Span, errorCode, error// in StageDependencies (those are siblings, not descendants).if span.Kind == types.SpanKindStage {t.cascadeDependentStages(ctx, span, reason)- // Any failure in a MAIN pipeline stage means the attempt is- // done — the parse cannot succeed past this point. Close the- // root span as failed so the UI doesn't show "进行中" forever.- // Optional downstream stages (summary/question/wiki/graph) do- // not poison the attempt: they can fail without invalidating- // the parsed document.- if isMainPipelineStage(span.Name) {+ // Only an explicitly retryable failure with remaining budget leaves+ // the root open. Retries share the same attempt, so closing it early+ // prevents a successful retry from finalizing it as done (#4326).+ // Terminal failures must close it even when retry budget is unused:+ // callers can record a permanent failure and acknowledge the task.+ //+ // Optional downstream stages (summary/question/wiki/graph) do not+ // poison the attempt: they can fail without invalidating the parsed+ // document.+ if isMainPipelineStage(span.Name) && !willRetry {t.FinalizeAttempt(ctx, span.KnowledgeID, span.Attempt,types.SpanStatusFailed, nil, errorCode, errorMessage)}@@ -725,12 +746,28 @@ func contains(haystack []string, needle string) bool {return false}+// attemptStillHasRetry reports whether retry budget remains, not whether the+// caller will actually request a retry. It reads ProcessDocument's isLastRetry+// counters: the Lite executor's injected metadata wins, otherwise+// the asynq counters. Outside a task runner (a synchronous reparse, a test+// harness) neither is present and the failure is final.+func attemptStillHasRetry(ctx context.Context) bool {+ if retried, limit, ok := types.TaskRetryMetadataFromContext(ctx); ok {+ return retried < limit+ }+ retryCount, ok := asynq.GetRetryCount(ctx)+ if !ok {+ return false+ }+ maxRetry, ok := asynq.GetMaxRetry(ctx)+ return ok && retryCount < maxRetry+}+// isMainPipelineStage reports whether stage is one of the 5 mandatory// pipeline stages (docreader / chunking / embedding / multimodal /-// postprocess). A failure in any of these terminally invalidates the-// attempt and must close the root as failed. Optional downstream stages-// added later (summary, question, wiki, graph) do NOT match — those-// can fail individually without poisoning the parse result.+// postprocess). A terminal failure in any of these closes the root. Optional+// downstream stages added later (summary, question, wiki, graph) do NOT match+// — those can fail individually without poisoning the parse result.func isMainPipelineStage(name string) bool {for _, s := range types.AllStages {if s == name {@@ -884,10 +921,11 @@ func (noopSpanTracker) BeginStage(_ context.Context, _ string, _ int, _ string,func (noopSpanTracker) BeginSubSpan(_ context.Context, _ *Span, _, _ string, _ types.JSONMap) *Span {return nil}-func (noopSpanTracker) EndSpan(_ context.Context, _ *Span, _ types.JSONMap) {}-func (noopSpanTracker) FailSpan(_ context.Context, _ *Span, _, _ string, _ error) {}-func (noopSpanTracker) SkipSpan(_ context.Context, _ *Span, _ string) {}-func (noopSpanTracker) LookupStage(_ context.Context, _ string, _ int, _ string) *Span { return nil }+func (noopSpanTracker) EndSpan(_ context.Context, _ *Span, _ types.JSONMap) {}+func (noopSpanTracker) FailSpan(_ context.Context, _ *Span, _, _ string, _ error) {}+func (noopSpanTracker) FailSpanRetryable(_ context.Context, _ *Span, _, _ string, _ error) {}+func (noopSpanTracker) SkipSpan(_ context.Context, _ *Span, _ string) {}+func (noopSpanTracker) LookupStage(_ context.Context, _ string, _ int, _ string) *Span { return nil }func (noopSpanTracker) LookupSpanByName(_ context.Context, _ string, _ int, _ string) *Span {return nil}diff --git a/internal/application/service/knowledge_span_tracker_test.go b/internal/application/service/knowledge_span_tracker_test.goindex ba732fb39..9a4c6e153 100644--- a/internal/application/service/knowledge_span_tracker_test.go+++ b/internal/application/service/knowledge_span_tracker_test.go@@ -537,3 +537,100 @@ func TestSpanTracker_OpenAttemptCancelsSupersededOpenSpans(t *testing.T) {"kid-reparse").Row().Scan(&newStatus))assert.Equal(t, types.SpanStatusRunning, newStatus)}++// rootSpan reads the attempt's root span: the status the UI's error banner and+// its progress bar are both driven by.+func rootSpan(t *testing.T, db *gorm.DB, knowledgeID string, attempt int) (status, code, message string) {+ t.Helper()+ require.NoError(t, db.Raw(+ `SELECT status, COALESCE(error_code, ''), COALESCE(error_message, '')+ FROM knowledge_processing_spans+ WHERE knowledge_id = ? AND attempt = ? AND kind = 'root'`,+ knowledgeID, attempt).Row().Scan(&status, &code, &message))+ return status, code, message+}++func TestSpanTracker_FailSpanRetryable_KeepsRootOpenWhileRetriesRemain(t *testing.T) {+ ctx := context.Background()+ // Every main pipeline stage goes through the same deferral, not just the+ // docreader one a report happens to show.+ for _, stage := range types.AllStages {+ t.Run(stage, func(t *testing.T) {+ tracker, db := setupSpanTrackerTest(t)+ kid := "kid-retry-" + stage+ _, attempt, err := tracker.OpenAttempt(ctx, kid, "")+ require.NoError(t, err)++ // The first run fails in a main stage. asynq retries reuse the+ // payload, so the next run records onto this same attempt and may+ // still complete it.+ first := types.WithTaskRetryMetadata(ctx, 0, 3)+ span := tracker.BeginStage(first, kid, attempt, stage, nil)+ tracker.FailSpanRetryable(first, span, "STAGE_FAILED", "transient upstream", errors.New("boom"))++ status, code, _ := rootSpan(t, db, kid, attempt)+ assert.Equal(t, types.SpanStatusRunning, status, "root stays open while a retry is left")+ assert.Empty(t, code, "an attempt that can still succeed must not be attributed a failure")++ // The retry succeeds. Before this fix the root was already terminal+ // here, so FinalizeAttempt returned on its guard and the UI showed+ // a red error banner above an all-done trace.+ tracker.FinalizeAttempt(types.WithTaskRetryMetadata(ctx, 1, 3),+ kid, attempt, types.SpanStatusDone, nil, "", "")+ status, code, message := rootSpan(t, db, kid, attempt)+ assert.Equal(t, types.SpanStatusDone, status)+ assert.Empty(t, code, "no stale error code survives a successful retry")+ assert.Empty(t, message)++ // A failure arriving after the attempt closed must not reopen it.+ late := tracker.BeginStage(first, kid, attempt, stage, nil)+ tracker.FailSpanRetryable(types.WithTaskRetryMetadata(ctx, 3, 3),+ late, "STAGE_FAILED", "later failure", errors.New("boom"))+ status, _, _ = rootSpan(t, db, kid, attempt)+ assert.Equal(t, types.SpanStatusDone, status, "a terminal attempt stays terminal")+ })+ }+}++func TestSpanTracker_FailSpanRetryable_ClosesRootOnTheLastRetry(t *testing.T) {+ tracker, db := setupSpanTrackerTest(t)+ ctx := context.Background()++ _, attempt, err := tracker.OpenAttempt(ctx, "kid-final", "")+ require.NoError(t, err)+ last := types.WithTaskRetryMetadata(ctx, 3, 3)+ docreader := tracker.BeginStage(last, "kid-final", attempt, types.StageDocReader, nil)+ tracker.FailSpanRetryable(last, docreader,+ "DOCREADER_PARSE_FAILED", "engine core initialization failed", errors.New("boom"))+ status, code, message := rootSpan(t, db, "kid-final", attempt)+ assert.Equal(t, types.SpanStatusFailed, status, "exhausted retries must not hang at 进行中")+ assert.Equal(t, "DOCREADER_PARSE_FAILED", code)+ assert.Equal(t, "engine core initialization failed", message)++ // With no task runner in the context at all — a synchronous reparse, or a+ // caller outside asynq — the failure is final, as it was before.+ _, syncAttempt, err := tracker.OpenAttempt(ctx, "kid-sync", "")+ require.NoError(t, err)+ post := tracker.BeginStage(ctx, "kid-sync", syncAttempt, types.StagePostProcess, nil)+ tracker.FailSpanRetryable(ctx, post, "POSTPROCESS_FAILED", "no retry metadata", errors.New("boom"))+ status, _, _ = rootSpan(t, db, "kid-sync", syncAttempt)+ assert.Equal(t, types.SpanStatusFailed, status)+}++// Unused queue budget does not make an acknowledged failure retryable.+func TestSpanTracker_FailSpan_ClosesRootWithUnusedRetries(t *testing.T) {+ for _, stage := range types.AllStages {+ t.Run(stage, func(t *testing.T) {+ tracker, db := setupSpanTrackerTest(t)+ ctx := types.WithTaskRetryMetadata(context.Background(), 0, 3)+ _, attempt, err := tracker.OpenAttempt(ctx, "terminal", "")+ require.NoError(t, err)+ span := tracker.BeginStage(ctx, "terminal", attempt, stage, nil)+ tracker.FailSpan(ctx, span, "PERMANENT_FAILURE", "cannot continue", nil)+ status, code, message := rootSpan(t, db, "terminal", attempt)+ require.Equal(t, types.SpanStatusFailed, status)+ require.Equal(t, "PERMANENT_FAILURE", code)+ require.Equal(t, "cannot continue", message)+ })+ }+}