Skip to content

Open-source LLM knowledge platform: turn raw documents into a queryable RAG, an autonomous reasoning agent, and a self-maintaining Wiki.

fix(knowledge): leave the root span open while a retry is left (#4357)

The one -Niuf5cfa0ff5fa8
diff --git a/internal/application/service/knowledge.go b/internal/application/service/knowledge.go
index c7126a272..7a3376733 100644
--- a/internal/application/service/knowledge.go
+++ b/internal/application/service/knowledge.go
@@ -331,6 +331,18 @@ func (s *knowledgeService) endStage(ctx context.Context, kid, name string, outpu
}
func (s *knowledgeService) failStage(ctx context.Context, kid, name, code, msg string, err error) {
+ s.failStageWithRetry(ctx, kid, name, code, msg, err, false)
+}
+
+// failStageRetryable is for failures returned to the task runner. Remaining
+// retry budget alone is insufficient: terminal failures acknowledge the task.
+func (s *knowledgeService) failStageRetryable(ctx context.Context, kid, name, code, msg string, err error) {
+ s.failStageWithRetry(ctx, kid, name, code, msg, err, true)
+}
+
+func (s *knowledgeService) failStageWithRetry(
+ ctx context.Context, kid, name, code, msg string, err error, retryable bool,
+) {
a := attemptFromCtx(ctx)
if a <= 0 {
return
@@ -339,7 +351,11 @@ func (s *knowledgeService) failStage(ctx context.Context, kid, name, code, msg s
if span == nil {
return
}
- s.tracker().FailSpan(ctx, span, code, msg, err)
+ if retryable {
+ s.tracker().FailSpanRetryable(ctx, span, code, msg, err)
+ } else {
+ s.tracker().FailSpan(ctx, span, code, msg, err)
+ }
}
func (s *knowledgeService) skipStage(ctx context.Context, kid, name, reason string) {
diff --git a/internal/application/service/knowledge_post_process.go b/internal/application/service/knowledge_post_process.go
index 67b9f90b4..1f889e504 100644
--- a/internal/application/service/knowledge_post_process.go
+++ b/internal/application/service/knowledge_post_process.go
@@ -274,7 +274,7 @@ func (s *KnowledgePostProcessService) Handle(ctx context.Context, task *asynq.Ta
// enqueue its KB-scoped trigger. Retry only the trigger: appending a
// second pending op would duplicate durable work and its finalizer.
if err := enqueueWikiIngestTrigger(ctx, s.taskEnqueuer, payload.TenantID, payload.KnowledgeBaseID); err != nil {
- s.tracker().FailSpan(ctx, postSpan, "WIKI_TRIGGER_ENQUEUE_FAILED", err.Error(), err)
+ s.tracker().FailSpanRetryable(ctx, postSpan, "WIKI_TRIGGER_ENQUEUE_FAILED", err.Error(), err)
return fmt.Errorf("retry wiki ingest trigger: %w", err)
}
logger.Infof(ctx, "[KnowledgePostProcess] Re-enqueued wiki ingest trigger for %s", payload.KnowledgeID)
@@ -303,7 +303,7 @@ func (s *KnowledgePostProcessService) Handle(ctx context.Context, task *asynq.Ta
case expectedSubtasks == 0:
completed, err := s.knowledgeRepo.CompleteProcessingWithoutSubtasks(ctx, payload.KnowledgeID)
if err != nil {
- s.tracker().FailSpan(ctx, postSpan, "COMPLETION_FAILED", err.Error(), err)
+ s.tracker().FailSpanRetryable(ctx, postSpan, "COMPLETION_FAILED", err.Error(), err)
return fmt.Errorf("complete knowledge without enrichment: %w", err)
}
if !completed {
@@ -340,7 +340,7 @@ func (s *KnowledgePostProcessService) Handle(ctx context.Context, task *asynq.Ta
// strand the row; retry so the handoff (or dead-letter) happens.
logger.Warnf(ctx, "[KnowledgePostProcess] SetFinalizing failed for %s: %v",
payload.KnowledgeID, err)
- s.tracker().FailSpan(ctx, postSpan, "FINALIZING_HANDOFF_FAILED", err.Error(), err)
+ s.tracker().FailSpanRetryable(ctx, postSpan, "FINALIZING_HANDOFF_FAILED", err.Error(), err)
return fmt.Errorf("enter finalizing: %w", err)
}
if promoted {
diff --git a/internal/application/service/knowledge_post_process_summary_test.go b/internal/application/service/knowledge_post_process_summary_test.go
index 1c1e65dfc..5e4e9f76b 100644
--- a/internal/application/service/knowledge_post_process_summary_test.go
+++ b/internal/application/service/knowledge_post_process_summary_test.go
@@ -145,9 +145,14 @@ func TestKnowledgePostProcessWithoutSummaryCompletionFailure(t *testing.T) {
wikiEnqueueFailureKnowledgeRepo: wikiEnqueueFailureKnowledgeRepo{knowledge: knowledge},
completionErr: tc.err, completionSkipped: tc.skipped,
}
+ tracker, db := setupSpanTrackerTest(t)
+ ctx := types.WithTaskRetryMetadata(context.Background(), 0, 3)
+ _, attempt, err := tracker.OpenAttempt(ctx, knowledge.ID, "")
+ require.NoError(t, err)
queue := &wikiEnqueueFailureTaskQueue{}
svc := &KnowledgePostProcessService{
knowledgeRepo: repo,
+ spanTracker: tracker,
kbService: &wikiEnqueueFailureKBService{kb: &types.KnowledgeBase{
ID: "kb", TenantID: 7, Type: types.KnowledgeBaseTypeDocument,
AutoTagConfig: &types.AutoTagConfig{Enabled: true},
@@ -162,15 +167,24 @@ func TestKnowledgePostProcessWithoutSummaryCompletionFailure(t *testing.T) {
TenantID: 7, KnowledgeID: "doc", KnowledgeBaseID: "kb",
})
require.NoError(t, err)
- err = svc.Handle(context.Background(), asynq.NewTask(types.TypeKnowledgePostProcess, payload))
+ err = svc.Handle(ctx, asynq.NewTask(types.TypeKnowledgePostProcess, payload))
require.ErrorIs(t, err, tc.err)
require.Empty(t, queue.taskTypes)
require.Equal(t, types.ParseStatusProcessing, knowledge.ParseStatus)
if tc.err != nil {
+ status, code, message := rootSpan(t, db, knowledge.ID, attempt)
+ require.Equal(t, types.SpanStatusRunning, status)
+ require.Empty(t, code)
+ require.Empty(t, message)
repo.completionErr = nil
+ ctx = types.WithTaskRetryMetadata(ctx, 1, 3)
task := asynq.NewTask(types.TypeKnowledgePostProcess, payload)
- require.NoError(t, svc.Handle(context.Background(), task))
+ require.NoError(t, svc.Handle(ctx, task))
require.Equal(t, types.ParseStatusCompleted, knowledge.ParseStatus)
+ status, code, message = rootSpan(t, db, knowledge.ID, attempt)
+ require.Equal(t, types.SpanStatusDone, status)
+ require.Empty(t, code)
+ require.Empty(t, message)
require.Equal(t, []string{types.TypeKnowledgeAutoTag}, queue.taskTypes)
}
})
diff --git a/internal/application/service/knowledge_process.go b/internal/application/service/knowledge_process.go
index 96e51b57e..7ffd22f00 100644
--- a/internal/application/service/knowledge_process.go
+++ b/internal/application/service/knowledge_process.go
@@ -769,11 +769,13 @@ func (s *knowledgeService) processChunks(ctx context.Context,
if isLikelyRateLimitError(err) {
code = werrors.ErrCodeEmbeddingRateLimit
}
- s.failStage(ctx, knowledge.ID, types.StageEmbedding,
- code, "batch index failed", err)
if willRetry {
+ s.failStageRetryable(ctx, knowledge.ID, types.StageEmbedding,
+ code, "batch index failed", err)
return fmt.Errorf("batch index: %w", err)
}
+ s.failStage(ctx, knowledge.ID, types.StageEmbedding,
+ code, "batch index failed", err)
return nil
}
logger.GetLogger(ctx).Infof("processChunks batch index successfully, with %d index", len(indexInfoList))
@@ -4241,16 +4243,18 @@ func (s *knowledgeService) convert(
if !isURL {
fileReader, err := s.resolveFileServiceForPath(ctx, kb, payload.FilePath).GetFile(ctx, payload.FilePath)
if err != nil {
- s.failStage(ctx, knowledge.ID, types.StageDocReader,
- werrors.ErrCodeDocReaderParseFailed, "failed to get file", err)
- return s.failKnowledge(ctx, knowledge, isLastRetry, "failed to get file: %v", err)
+ result, taskErr := s.failKnowledge(ctx, knowledge, isLastRetry, "failed to get file: %v", err)
+ s.failStageWithRetry(ctx, knowledge.ID, types.StageDocReader,
+ werrors.ErrCodeDocReaderParseFailed, "failed to get file", err, taskErr != nil)
+ return result, taskErr
}
defer fileReader.Close()
contentBytes, err := io.ReadAll(fileReader)
if err != nil {
- s.failStage(ctx, knowledge.ID, types.StageDocReader,
- werrors.ErrCodeDocReaderParseFailed, "failed to read file", err)
- return s.failKnowledge(ctx, knowledge, isLastRetry, "failed to read file: %v", err)
+ result, taskErr := s.failKnowledge(ctx, knowledge, isLastRetry, "failed to read file: %v", err)
+ s.failStageWithRetry(ctx, knowledge.ID, types.StageDocReader,
+ werrors.ErrCodeDocReaderParseFailed, "failed to read file", err, taskErr != nil)
+ return result, taskErr
}
req.FileContent = contentBytes
req.FileName = payload.FileName
@@ -4261,9 +4265,10 @@ func (s *knowledgeService) convert(
if err != nil {
code, message := docReaderFailure(err)
logger.Errorf(ctx, "[convert] DocReader failed knowledge=%s code=%s: %v", knowledge.ID, code, err)
- s.failStage(ctx, knowledge.ID, types.StageDocReader,
- code, message, err)
- return s.failKnowledge(ctx, knowledge, isLastRetry, "%s", message)
+ result, taskErr := s.failKnowledge(ctx, knowledge, isLastRetry, "%s", message)
+ s.failStageWithRetry(ctx, knowledge.ID, types.StageDocReader,
+ code, message, err, taskErr != nil)
+ return result, taskErr
}
sanitizeReadResult(result)
if result.Error != "" {
diff --git a/internal/application/service/knowledge_process_embedding_failure_test.go b/internal/application/service/knowledge_process_embedding_failure_test.go
index 238ed02fa..18842bafd 100644
--- a/internal/application/service/knowledge_process_embedding_failure_test.go
+++ b/internal/application/service/knowledge_process_embedding_failure_test.go
@@ -7,6 +7,7 @@ import (
"strings"
"testing"
+ werrors "github.com/Tencent/WeKnora/internal/errors"
"github.com/Tencent/WeKnora/internal/models/embedding"
"github.com/Tencent/WeKnora/internal/types"
"github.com/Tencent/WeKnora/internal/types/interfaces"
@@ -64,6 +65,8 @@ func (s embedFailureModelService) GetEmbeddingModel(
// attempt to keep processing past the missing embedder would panic and fail
// this test, which is exactly the regression worth catching.
func TestProcessChunksFailsKnowledgeWhenEmbeddingModelUnavailable(t *testing.T) {
+ tracker, db := setupSpanTrackerTest(t)
+ ctx := types.WithTaskRetryMetadata(context.Background(), 0, 3)
modelErr := errors.New("embedding model not found")
knowledge := &types.Knowledge{
ID: "knowledge-1",
@@ -75,6 +78,7 @@ func TestProcessChunksFailsKnowledgeWhenEmbeddingModelUnavailable(t *testing.T)
svc := &knowledgeService{
repo: repo,
modelService: embedFailureModelService{err: modelErr},
+ spanTracker: tracker,
}
kb := &types.KnowledgeBase{
ID: "kb-1",
@@ -83,8 +87,12 @@ func TestProcessChunksFailsKnowledgeWhenEmbeddingModelUnavailable(t *testing.T)
IndexingStrategy: types.IndexingStrategy{VectorEnabled: true},
}
- svc.processChunks(context.Background(), kb, knowledge,
+ _, attempt, err := tracker.OpenAttempt(ctx, knowledge.ID, "")
+ require.NoError(t, err)
+ ctx = withAttempt(ctx, attempt)
+ err = svc.processChunks(ctx, kb, knowledge,
[]types.ParsedChunk{{Content: "body", Seq: 0, Start: 0, End: 4}})
+ require.NoError(t, err, "the task is acknowledged even though retry budget remains")
require.Len(t, repo.updates, 1, "the failure must be persisted exactly once")
require.Equal(t, types.ParseStatusFailed, repo.updates[0].ParseStatus)
@@ -93,6 +101,21 @@ func TestProcessChunksFailsKnowledgeWhenEmbeddingModelUnavailable(t *testing.T)
"error_message should name the underlying cause, got %q", repo.updates[0].ErrorMessage,
)
require.Equal(t, types.ParseStatusFailed, knowledge.ParseStatus)
+ status, code, message := rootSpan(t, db, knowledge.ID, attempt)
+ require.Equal(t, types.SpanStatusFailed, status)
+ require.Equal(t, werrors.ErrCodeEmbeddingProviderFail, code)
+ require.Equal(t, knowledge.ErrorMessage, message)
+
+ // Housekeeping must not relabel this settled failure as TASK_STALLED.
+ require.NoError(t, db.Exec(knowledgeTestDDL).Error)
+ require.NoError(t, db.Exec(
+ "INSERT INTO knowledges (id, parse_status, error_message) VALUES (?, ?, ?)",
+ knowledge.ID, knowledge.ParseStatus, knowledge.ErrorMessage).Error)
+ (&HousekeepingService{db: db}).reapTerminalSpans(ctx)
+ status, code, message = rootSpan(t, db, knowledge.ID, attempt)
+ require.Equal(t, types.SpanStatusFailed, status)
+ require.Equal(t, werrors.ErrCodeEmbeddingProviderFail, code)
+ require.Equal(t, knowledge.ErrorMessage, message)
}
// A cancelled context means the run was interrupted, not that the model is
diff --git a/internal/application/service/knowledge_process_index_retry_test.go b/internal/application/service/knowledge_process_index_retry_test.go
index 89bcf7379..e6703a723 100644
--- a/internal/application/service/knowledge_process_index_retry_test.go
+++ b/internal/application/service/knowledge_process_index_retry_test.go
@@ -5,6 +5,7 @@ import (
"errors"
"testing"
+ werrors "github.com/Tencent/WeKnora/internal/errors"
"github.com/Tencent/WeKnora/internal/models/embedding"
"github.com/Tencent/WeKnora/internal/types"
"github.com/stretchr/testify/require"
@@ -48,7 +49,7 @@ func (e *failingIndexEngine) DeleteByKnowledgeIDList(context.Context, []string,
}
func runProcessChunksWithFailingIndex(
- ctx context.Context, t *testing.T,
+ ctx context.Context, t *testing.T, tracker SpanTracker,
) (*indexRetryKnowledgeRepo, *indexRetryChunkRepo, *failingIndexEngine, error) {
t.Helper()
knowledge := &types.Knowledge{
@@ -70,6 +71,7 @@ func runProcessChunksWithFailingIndex(
ctx = context.WithValue(ctx, types.TenantInfoContextKey, tenant)
svc := &knowledgeService{
repo: repo,
+ spanTracker: tracker,
chunkRepo: chunks,
modelService: parentChildModelService{embedder: parentChildEmbedder{}},
retrieveEngine: parentChildRetrieveRegistry{engine: engine},
@@ -95,9 +97,17 @@ func runProcessChunksWithFailingIndex(
func TestProcessChunksRetriesBatchIndexFailureBeforeLastAttempt(t *testing.T) {
ctx := types.WithTaskRetryMetadata(context.Background(), 0, 3)
- repo, chunks, engine, err := runProcessChunksWithFailingIndex(ctx, t)
+ tracker, db := setupSpanTrackerTest(t)
+ _, attempt, err := tracker.OpenAttempt(ctx, "knowledge-1", "")
+ require.NoError(t, err)
+ ctx = withAttempt(ctx, attempt)
+ repo, chunks, engine, err := runProcessChunksWithFailingIndex(ctx, t, tracker)
require.ErrorContains(t, err, "embedding endpoint unavailable")
+ status, code, message := rootSpan(t, db, "knowledge-1", attempt)
+ require.Equal(t, types.SpanStatusRunning, status)
+ require.Empty(t, code)
+ require.Empty(t, message)
for _, update := range repo.updates {
require.NotEqual(t, types.ParseStatusFailed, update.ParseStatus)
}
@@ -109,9 +119,17 @@ func TestProcessChunksRetriesBatchIndexFailureBeforeLastAttempt(t *testing.T) {
func TestProcessChunksFailsKnowledgeOnLastBatchIndexAttempt(t *testing.T) {
ctx := types.WithTaskRetryMetadata(context.Background(), 3, 3)
- repo, chunks, engine, err := runProcessChunksWithFailingIndex(ctx, t)
+ tracker, db := setupSpanTrackerTest(t)
+ _, attempt, err := tracker.OpenAttempt(ctx, "knowledge-1", "")
+ require.NoError(t, err)
+ ctx = withAttempt(ctx, attempt)
+ repo, chunks, engine, err := runProcessChunksWithFailingIndex(ctx, t, tracker)
require.NoError(t, err)
+ status, code, message := rootSpan(t, db, "knowledge-1", attempt)
+ require.Equal(t, types.SpanStatusFailed, status)
+ require.Equal(t, werrors.ErrCodeVectorStoreWriteFailed, code)
+ require.Equal(t, "batch index failed", message)
require.NotEmpty(t, repo.updates)
last := repo.updates[len(repo.updates)-1]
require.Equal(t, types.ParseStatusFailed, last.ParseStatus)
diff --git a/internal/application/service/knowledge_process_span_retry_test.go b/internal/application/service/knowledge_process_span_retry_test.go
new file mode 100644
index 000000000..2eca9adc4
--- /dev/null
+++ b/internal/application/service/knowledge_process_span_retry_test.go
@@ -0,0 +1,113 @@
+package service
+
+import (
+ "context"
+ "errors"
+ "io"
+ "strings"
+ "testing"
+ "testing/iotest"
+
+ werrors "github.com/Tencent/WeKnora/internal/errors"
+ "github.com/Tencent/WeKnora/internal/types"
+ "github.com/Tencent/WeKnora/internal/types/interfaces"
+ "github.com/stretchr/testify/require"
+)
+
+type spanRetryDocReader struct {
+ interfaces.DocumentReader
+ result *types.ReadResult
+ err error
+}
+
+func (r *spanRetryDocReader) Read(context.Context, *types.ReadRequest) (*types.ReadResult, error) {
+ return r.result, r.err
+}
+
+type spanRetryFileService struct {
+ interfaces.FileService
+ getErr, readErr error
+}
+
+func (s spanRetryFileService) GetFile(context.Context, string) (io.ReadCloser, error) {
+ if s.getErr != nil {
+ return nil, s.getErr
+ }
+ if s.readErr != nil {
+ return io.NopCloser(iotest.ErrReader(s.readErr)), nil
+ }
+ return io.NopCloser(strings.NewReader("document")), nil
+}
+
+// Exercise convert's actual return value: a parser response with Error is
+// acknowledged, whereas transport/file errors are returned for queue retry.
+func TestConvertFailureSpanMatchesTaskOutcome(t *testing.T) {
+ transient := errors.New("temporary read failure")
+ for _, tc := range []struct {
+ name string
+ missingReader bool
+ parserError string
+ getErr, readErr, rpcErr error
+ retryable bool
+ code string
+ }{
+ {name: "reader unavailable", missingReader: true, code: werrors.ErrCodeDocReaderUnavailable},
+ {
+ name: "parser rejects document", parserError: "unsupported document",
+ code: werrors.ErrCodeDocReaderParseFailed,
+ },
+ {name: "file fetch fails", getErr: transient, retryable: true, code: werrors.ErrCodeDocReaderParseFailed},
+ {name: "file read fails", readErr: transient, retryable: true, code: werrors.ErrCodeDocReaderParseFailed},
+ {name: "reader call fails", rpcErr: transient, retryable: true, code: werrors.ErrCodeDocReaderParseFailed},
+ } {
+ for _, retried := range []int{0, 3} {
+ name := tc.name + "/retry_available"
+ if retried == 3 {
+ name = tc.name + "/last_retry"
+ }
+ t.Run(name, func(t *testing.T) {
+ tracker, db := setupSpanTrackerTest(t)
+ ctx := types.WithTaskRetryMetadata(context.Background(), retried, 3)
+ knowledge := &types.Knowledge{
+ ID: "doc", TenantID: 1, KnowledgeBaseID: "kb", FilePath: "doc.pdf",
+ ParseStatus: types.ParseStatusProcessing,
+ }
+ _, attempt, err := tracker.OpenAttempt(ctx, knowledge.ID, "")
+ require.NoError(t, err)
+ ctx = withAttempt(ctx, attempt)
+ svc := &knowledgeService{
+ repo: &embedFailureKnowledgeRepo{knowledge: knowledge},
+ spanTracker: tracker,
+ tenantService: &stubTenantServiceForModelDelete{},
+ fileSvc: spanRetryFileService{getErr: tc.getErr, readErr: tc.readErr},
+ }
+ if !tc.missingReader {
+ svc.documentReader = &spanRetryDocReader{
+ result: &types.ReadResult{Error: tc.parserError}, err: tc.rpcErr,
+ }
+ }
+ result, err := svc.convert(ctx, types.DocumentProcessPayload{
+ FilePath: "doc.pdf", FileName: "doc.pdf", FileType: "pdf",
+ }, &types.KnowledgeBase{ID: "kb"}, knowledge, types.EffectiveProcessConfig{}, retried == 3)
+ require.Nil(t, result)
+ if tc.retryable {
+ require.Error(t, err)
+ } else {
+ require.NoError(t, err)
+ }
+ status, code, message := rootSpan(t, db, knowledge.ID, attempt)
+ if tc.retryable && retried < 3 {
+ require.Equal(t, types.ParseStatusProcessing, knowledge.ParseStatus)
+ require.Equal(t, types.SpanStatusRunning, status)
+ require.Empty(t, code)
+ require.Empty(t, message)
+ } else {
+ require.Equal(t, types.ParseStatusFailed, knowledge.ParseStatus)
+ require.Equal(t, types.SpanStatusFailed, status)
+ require.Equal(t, tc.code, code)
+ require.NotEmpty(t, message)
+ }
+ })
+ }
+ }
+}
diff --git a/internal/application/service/knowledge_span_tracker.go b/internal/application/service/knowledge_span_tracker.go
index 100c27d07..f5bac889f 100644
--- a/internal/application/service/knowledge_span_tracker.go
+++ b/internal/application/service/knowledge_span_tracker.go
@@ -36,6 +36,7 @@ import (
"github.com/Tencent/WeKnora/internal/logger"
"github.com/Tencent/WeKnora/internal/types"
"github.com/google/uuid"
+ "github.com/hibiken/asynq"
"gorm.io/gorm"
)
@@ -110,9 +111,14 @@ type SpanTracker interface {
// FailSpan marks span as failed and cascade-cancels its
// descendants. errorDetail (a Go error) is recorded verbatim in
- // error_detail (truncated to 8 KB) for admin views.
+ // error_detail (truncated to 8 KB) for admin views. Main pipeline stage
+ // failures close the root even when unused retry budget remains.
FailSpan(ctx context.Context, span *Span, errorCode, errorMessage string, errorDetail error)
+ // FailSpanRetryable records a failure that the caller will return to the
+ // task runner for retry. It leaves the root open only while retries remain.
+ FailSpanRetryable(ctx context.Context, span *Span, errorCode, errorMessage string, errorDetail error)
+
// SkipSpan marks an intentionally not-run span (e.g. multimodal
// on a text-only document). Distinct from cancelled — skipped is
// "we chose not to" while cancelled is "an upstream broke".
@@ -493,6 +499,18 @@ func (t *spanTracker) EndSpan(ctx context.Context, span *Span, output types.JSON
}
func (t *spanTracker) FailSpan(ctx context.Context, span *Span, errorCode, errorMessage string, errorDetail error) {
+ t.failSpan(ctx, span, errorCode, errorMessage, errorDetail, false)
+}
+
+func (t *spanTracker) FailSpanRetryable(
+ ctx context.Context, span *Span, errorCode, errorMessage string, errorDetail error,
+) {
+ t.failSpan(ctx, span, errorCode, errorMessage, errorDetail, attemptStillHasRetry(ctx))
+}
+
+func (t *spanTracker) failSpan(
+ ctx context.Context, span *Span, errorCode, errorMessage string, errorDetail error, willRetry bool,
+) {
if span == nil {
return
}
@@ -540,13 +558,16 @@ func (t *spanTracker) FailSpan(ctx context.Context, span *Span, errorCode, error
// in StageDependencies (those are siblings, not descendants).
if span.Kind == types.SpanKindStage {
t.cascadeDependentStages(ctx, span, reason)
- // Any failure in a MAIN pipeline stage means the attempt is
- // done — the parse cannot succeed past this point. Close the
- // root span as failed so the UI doesn't show "进行中" forever.
- // Optional downstream stages (summary/question/wiki/graph) do
- // not poison the attempt: they can fail without invalidating
- // the parsed document.
- if isMainPipelineStage(span.Name) {
+ // Only an explicitly retryable failure with remaining budget leaves
+ // the root open. Retries share the same attempt, so closing it early
+ // prevents a successful retry from finalizing it as done (#4326).
+ // Terminal failures must close it even when retry budget is unused:
+ // callers can record a permanent failure and acknowledge the task.
+ //
+ // Optional downstream stages (summary/question/wiki/graph) do not
+ // poison the attempt: they can fail without invalidating the parsed
+ // document.
+ if isMainPipelineStage(span.Name) && !willRetry {
t.FinalizeAttempt(ctx, span.KnowledgeID, span.Attempt,
types.SpanStatusFailed, nil, errorCode, errorMessage)
}
@@ -725,12 +746,28 @@ func contains(haystack []string, needle string) bool {
return false
}
+// attemptStillHasRetry reports whether retry budget remains, not whether the
+// caller will actually request a retry. It reads ProcessDocument's isLastRetry
+// counters: the Lite executor's injected metadata wins, otherwise
+// the asynq counters. Outside a task runner (a synchronous reparse, a test
+// harness) neither is present and the failure is final.
+func attemptStillHasRetry(ctx context.Context) bool {
+ if retried, limit, ok := types.TaskRetryMetadataFromContext(ctx); ok {
+ return retried < limit
+ }
+ retryCount, ok := asynq.GetRetryCount(ctx)
+ if !ok {
+ return false
+ }
+ maxRetry, ok := asynq.GetMaxRetry(ctx)
+ return ok && retryCount < maxRetry
+}
+
// isMainPipelineStage reports whether stage is one of the 5 mandatory
// pipeline stages (docreader / chunking / embedding / multimodal /
-// postprocess). A failure in any of these terminally invalidates the
-// attempt and must close the root as failed. Optional downstream stages
-// added later (summary, question, wiki, graph) do NOT match — those
-// can fail individually without poisoning the parse result.
+// postprocess). A terminal failure in any of these closes the root. Optional
+// downstream stages added later (summary, question, wiki, graph) do NOT match
+// — those can fail individually without poisoning the parse result.
func isMainPipelineStage(name string) bool {
for _, s := range types.AllStages {
if s == name {
@@ -884,10 +921,11 @@ func (noopSpanTracker) BeginStage(_ context.Context, _ string, _ int, _ string,
func (noopSpanTracker) BeginSubSpan(_ context.Context, _ *Span, _, _ string, _ types.JSONMap) *Span {
return nil
}
-func (noopSpanTracker) EndSpan(_ context.Context, _ *Span, _ types.JSONMap) {}
-func (noopSpanTracker) FailSpan(_ context.Context, _ *Span, _, _ string, _ error) {}
-func (noopSpanTracker) SkipSpan(_ context.Context, _ *Span, _ string) {}
-func (noopSpanTracker) LookupStage(_ context.Context, _ string, _ int, _ string) *Span { return nil }
+func (noopSpanTracker) EndSpan(_ context.Context, _ *Span, _ types.JSONMap) {}
+func (noopSpanTracker) FailSpan(_ context.Context, _ *Span, _, _ string, _ error) {}
+func (noopSpanTracker) FailSpanRetryable(_ context.Context, _ *Span, _, _ string, _ error) {}
+func (noopSpanTracker) SkipSpan(_ context.Context, _ *Span, _ string) {}
+func (noopSpanTracker) LookupStage(_ context.Context, _ string, _ int, _ string) *Span { return nil }
func (noopSpanTracker) LookupSpanByName(_ context.Context, _ string, _ int, _ string) *Span {
return nil
}
diff --git a/internal/application/service/knowledge_span_tracker_test.go b/internal/application/service/knowledge_span_tracker_test.go
index ba732fb39..9a4c6e153 100644
--- a/internal/application/service/knowledge_span_tracker_test.go
+++ b/internal/application/service/knowledge_span_tracker_test.go
@@ -537,3 +537,100 @@ func TestSpanTracker_OpenAttemptCancelsSupersededOpenSpans(t *testing.T) {
"kid-reparse").Row().Scan(&newStatus))
assert.Equal(t, types.SpanStatusRunning, newStatus)
}
+
+// rootSpan reads the attempt's root span: the status the UI's error banner and
+// its progress bar are both driven by.
+func rootSpan(t *testing.T, db *gorm.DB, knowledgeID string, attempt int) (status, code, message string) {
+ t.Helper()
+ require.NoError(t, db.Raw(
+ `SELECT status, COALESCE(error_code, ''), COALESCE(error_message, '')
+ FROM knowledge_processing_spans
+ WHERE knowledge_id = ? AND attempt = ? AND kind = 'root'`,
+ knowledgeID, attempt).Row().Scan(&status, &code, &message))
+ return status, code, message
+}
+
+func TestSpanTracker_FailSpanRetryable_KeepsRootOpenWhileRetriesRemain(t *testing.T) {
+ ctx := context.Background()
+ // Every main pipeline stage goes through the same deferral, not just the
+ // docreader one a report happens to show.
+ for _, stage := range types.AllStages {
+ t.Run(stage, func(t *testing.T) {
+ tracker, db := setupSpanTrackerTest(t)
+ kid := "kid-retry-" + stage
+ _, attempt, err := tracker.OpenAttempt(ctx, kid, "")
+ require.NoError(t, err)
+
+ // The first run fails in a main stage. asynq retries reuse the
+ // payload, so the next run records onto this same attempt and may
+ // still complete it.
+ first := types.WithTaskRetryMetadata(ctx, 0, 3)
+ span := tracker.BeginStage(first, kid, attempt, stage, nil)
+ tracker.FailSpanRetryable(first, span, "STAGE_FAILED", "transient upstream", errors.New("boom"))
+
+ status, code, _ := rootSpan(t, db, kid, attempt)
+ assert.Equal(t, types.SpanStatusRunning, status, "root stays open while a retry is left")
+ assert.Empty(t, code, "an attempt that can still succeed must not be attributed a failure")
+
+ // The retry succeeds. Before this fix the root was already terminal
+ // here, so FinalizeAttempt returned on its guard and the UI showed
+ // a red error banner above an all-done trace.
+ tracker.FinalizeAttempt(types.WithTaskRetryMetadata(ctx, 1, 3),
+ kid, attempt, types.SpanStatusDone, nil, "", "")
+ status, code, message := rootSpan(t, db, kid, attempt)
+ assert.Equal(t, types.SpanStatusDone, status)
+ assert.Empty(t, code, "no stale error code survives a successful retry")
+ assert.Empty(t, message)
+
+ // A failure arriving after the attempt closed must not reopen it.
+ late := tracker.BeginStage(first, kid, attempt, stage, nil)
+ tracker.FailSpanRetryable(types.WithTaskRetryMetadata(ctx, 3, 3),
+ late, "STAGE_FAILED", "later failure", errors.New("boom"))
+ status, _, _ = rootSpan(t, db, kid, attempt)
+ assert.Equal(t, types.SpanStatusDone, status, "a terminal attempt stays terminal")
+ })
+ }
+}
+
+func TestSpanTracker_FailSpanRetryable_ClosesRootOnTheLastRetry(t *testing.T) {
+ tracker, db := setupSpanTrackerTest(t)
+ ctx := context.Background()
+
+ _, attempt, err := tracker.OpenAttempt(ctx, "kid-final", "")
+ require.NoError(t, err)
+ last := types.WithTaskRetryMetadata(ctx, 3, 3)
+ docreader := tracker.BeginStage(last, "kid-final", attempt, types.StageDocReader, nil)
+ tracker.FailSpanRetryable(last, docreader,
+ "DOCREADER_PARSE_FAILED", "engine core initialization failed", errors.New("boom"))
+ status, code, message := rootSpan(t, db, "kid-final", attempt)
+ assert.Equal(t, types.SpanStatusFailed, status, "exhausted retries must not hang at 进行中")
+ assert.Equal(t, "DOCREADER_PARSE_FAILED", code)
+ assert.Equal(t, "engine core initialization failed", message)
+
+ // With no task runner in the context at all — a synchronous reparse, or a
+ // caller outside asynq — the failure is final, as it was before.
+ _, syncAttempt, err := tracker.OpenAttempt(ctx, "kid-sync", "")
+ require.NoError(t, err)
+ post := tracker.BeginStage(ctx, "kid-sync", syncAttempt, types.StagePostProcess, nil)
+ tracker.FailSpanRetryable(ctx, post, "POSTPROCESS_FAILED", "no retry metadata", errors.New("boom"))
+ status, _, _ = rootSpan(t, db, "kid-sync", syncAttempt)
+ assert.Equal(t, types.SpanStatusFailed, status)
+}
+
+// Unused queue budget does not make an acknowledged failure retryable.
+func TestSpanTracker_FailSpan_ClosesRootWithUnusedRetries(t *testing.T) {
+ for _, stage := range types.AllStages {
+ t.Run(stage, func(t *testing.T) {
+ tracker, db := setupSpanTrackerTest(t)
+ ctx := types.WithTaskRetryMetadata(context.Background(), 0, 3)
+ _, attempt, err := tracker.OpenAttempt(ctx, "terminal", "")
+ require.NoError(t, err)
+ span := tracker.BeginStage(ctx, "terminal", attempt, stage, nil)
+ tracker.FailSpan(ctx, span, "PERMANENT_FAILURE", "cannot continue", nil)
+ status, code, message := rootSpan(t, db, "terminal", attempt)
+ require.Equal(t, types.SpanStatusFailed, status)
+ require.Equal(t, "PERMANENT_FAILURE", code)
+ require.Equal(t, "cannot continue", message)
+ })
+ }
+}