Skip to content

Open-source LLM knowledge platform: turn raw documents into a queryable RAG, an autonomous reasoning agent, and a self-maintaining Wiki.

fix(knowledge): stop synthesizing never-run stages as failed

Ankit Jha0b4cd706c240
diff --git a/internal/handler/knowledge.go b/internal/handler/knowledge.go
index 672215b1c..a35fff357 100644
--- a/internal/handler/knowledge.go
+++ b/internal/handler/knowledge.go
@@ -731,6 +731,70 @@ func knowledgeSpansLastError(
}
}
+// missingStageStatusFunc decides what a canonical stage with no row should
+// render as. The parse_status-derived fallback cannot tell "this stage ran and
+// failed" from "this stage never ran", so once a stage actually recorded a
+// failure it paints every rows-less stage `failed` too: the UI picks the first
+// failed stage in canonical order and names the wrong one, and the blast radius
+// looks total rather than "one stage broke, the rest never started" (#3452).
+//
+// With a real stage failure on record the position of that failure is known, so:
+//
+// - downstream of it, per StageDependencies or canonical order → cancelled
+// ("not run because an upstream span failed")
+// - before it → skipped; the stage either did not apply (a text passage has no
+// docreader step) or its row was never written, and neither is a failure
+//
+// Without one, the parse_status fallback stands unchanged, which is the case it
+// was written for: legacy knowledge parsed before span tracking existed.
+func missingStageStatusFunc(
+ stageRowByName map[string]*types.KnowledgeProcessingSpan, fallback string,
+) func(string) string {
+ failedIdx := -1
+ for i, name := range types.AllStages {
+ if row, ok := stageRowByName[name]; ok && row.Status == types.SpanStatusFailed {
+ failedIdx = i
+ break
+ }
+ }
+ if failedIdx < 0 {
+ return func(string) string { return fallback }
+ }
+
+ // Transitive dependents of the failed stage. Today these are always later
+ // in AllStages too, so the canonical-order check below already covers them;
+ // consulting the DAG keeps the answer right if the stage list is ever
+ // reordered or a stage gains an upstream that is not its predecessor.
+ cancelled := map[string]bool{}
+ var markDependents func(string)
+ markDependents = func(stage string) {
+ for candidate, upstreams := range types.StageDependencies {
+ if cancelled[candidate] {
+ continue
+ }
+ for _, up := range upstreams {
+ if up == stage {
+ cancelled[candidate] = true
+ markDependents(candidate)
+ break
+ }
+ }
+ }
+ }
+ markDependents(types.AllStages[failedIdx])
+
+ stageIdx := make(map[string]int, len(types.AllStages))
+ for i, name := range types.AllStages {
+ stageIdx[name] = i
+ }
+ return func(name string) string {
+ if cancelled[name] || stageIdx[name] > failedIdx {
+ return types.SpanStatusCancelled
+ }
+ return types.SpanStatusSkipped
+ }
+}
+
// buildSpanTree assembles a flat list of span rows into a parent-child
// tree rooted at the (knowledge, attempt)'s root span. Missing canonical
// stages are filled in with pending placeholders so the UI always renders
@@ -785,6 +849,8 @@ func buildSpanTree(knowledgeID string, attempt int, rows []types.KnowledgeProces
syntheticStatus = types.SpanStatusDone
case types.ParseStatusFailed:
syntheticStatus = types.SpanStatusFailed
+ case types.ParseStatusCancelled:
+ syntheticStatus = types.SpanStatusCancelled
}
// Synthesize root if no rows came back so the API contract stays
@@ -838,6 +904,7 @@ func buildSpanTree(knowledgeID string, attempt int, rows []types.KnowledgeProces
// per-stage timing was never recorded. Appended in AllStages order
// so the canonical stage layout is deterministic regardless of
// which rows are missing.
+ missingStageStatus := missingStageStatusFunc(stageRowByName, syntheticStatus)
for _, name := range types.AllStages {
if _, ok := stageRowByName[name]; ok {
continue
@@ -847,7 +914,7 @@ func buildSpanTree(knowledgeID string, attempt int, rows []types.KnowledgeProces
Attempt: attempt,
Name: name,
Kind: types.SpanKindStage,
- Status: syntheticStatus,
+ Status: missingStageStatus(name),
CreatedAt: now,
UpdatedAt: now,
}
diff --git a/internal/handler/knowledge_spans_test.go b/internal/handler/knowledge_spans_test.go
index 0976794de..29afdddf5 100644
--- a/internal/handler/knowledge_spans_test.go
+++ b/internal/handler/knowledge_spans_test.go
@@ -171,3 +171,112 @@ func TestKnowledgeSpansLastError_SkipsRecoveryFallbackWithoutMessage(t *testing.
got := knowledgeSpansLastError(2, 2, types.ParseStatusFailed, "", now, nil)
assert.Nil(t, got)
}
+
+// stageStatuses maps each canonical stage to the status it renders as.
+func stageStatuses(t *testing.T, tree *types.SpanTreeNode) map[string]string {
+ t.Helper()
+ got := map[string]string{}
+ for _, child := range tree.Children {
+ if child.Kind == types.SpanKindStage {
+ got[child.Name] = child.Status
+ }
+ }
+ return got
+}
+
+// TestBuildSpanTree_MissingStagesAroundRealFailure records #3452: a text
+// passage is chunked without ever opening a docreader stage, so when
+// embedding fails the rows-less stages used to synthesize as "failed" too.
+// The frontend names the failed stage by taking the first failed one in
+// canonical order, so it reported docreader — a stage that never ran, for
+// content that has no document to parse — and four of five segments showed
+// as broken.
+func TestBuildSpanTree_MissingStagesAroundRealFailure(t *testing.T) {
+ now := time.Now()
+ finished := now.Add(2 * time.Second)
+ rows := []types.KnowledgeProcessingSpan{
+ {KnowledgeID: "kid", Attempt: 1, SpanID: "root", Name: "knowledge_processing", Kind: types.SpanKindRoot, Status: types.SpanStatusFailed, StartedAt: &now, FinishedAt: &finished},
+ {KnowledgeID: "kid", Attempt: 1, SpanID: "chunk", ParentSpanID: "root", Name: types.StageChunking, Kind: types.SpanKindStage, Status: types.SpanStatusDone, StartedAt: &now, FinishedAt: &finished},
+ {KnowledgeID: "kid", Attempt: 1, SpanID: "emb", ParentSpanID: "root", Name: types.StageEmbedding, Kind: types.SpanKindStage, Status: types.SpanStatusFailed, ErrorCode: "EMBED_UNREACHABLE", StartedAt: &now, FinishedAt: &finished},
+ }
+
+ tree, _, lastFail := buildSpanTree("kid", 1, rows, types.ParseStatusFailed)
+ a := assert.New(t)
+ got := stageStatuses(t, tree)
+
+ a.Equal(types.SpanStatusFailed, got[types.StageEmbedding],
+ "the stage that actually failed keeps its real row")
+ a.Equal(types.SpanStatusSkipped, got[types.StageDocReader],
+ "a stage before the failure with no row never ran; a text passage has no document to parse")
+ a.Equal(types.SpanStatusCancelled, got[types.StageMultimodal],
+ "a stage after the failure with no row was abandoned, not broken")
+ a.Equal(types.SpanStatusCancelled, got[types.StagePostProcess],
+ "postprocess depends on embedding, so it is cancelled rather than failed")
+
+ // The frontend picks the focused stage with the first `failed` in
+ // canonical order; embedding must be the only candidate.
+ var firstFailed string
+ for _, name := range types.AllStages {
+ if got[name] == types.SpanStatusFailed {
+ firstFailed = name
+ break
+ }
+ }
+ a.Equal(types.StageEmbedding, firstFailed,
+ "the timeline must name embedding, not an earlier stage that never ran")
+ a.NotNil(lastFail)
+ a.Equal(types.StageEmbedding, lastFail.Name)
+}
+
+// TestBuildSpanTree_MissingStagesWithoutFailureKeepFallback pins the case the
+// parse_status rule was written for: no stage recorded a failure, so there is
+// no position to reason from and the inferred terminal state still applies.
+func TestBuildSpanTree_MissingStagesWithoutFailureKeepFallback(t *testing.T) {
+ now := time.Now()
+ rows := []types.KnowledgeProcessingSpan{
+ {KnowledgeID: "kid", Attempt: 1, SpanID: "root", Name: "knowledge_processing", Kind: types.SpanKindRoot, Status: types.SpanStatusRunning, StartedAt: &now},
+ {KnowledgeID: "kid", Attempt: 1, SpanID: "doc", ParentSpanID: "root", Name: types.StageDocReader, Kind: types.SpanKindStage, Status: types.SpanStatusDone, StartedAt: &now},
+ }
+
+ tree, _, _ := buildSpanTree("kid", 1, rows, types.ParseStatusProcessing)
+ got := stageStatuses(t, tree)
+ a := assert.New(t)
+ a.Equal(types.SpanStatusPending, got[types.StageChunking],
+ "a run still in flight keeps pending placeholders")
+ a.Equal(types.SpanStatusPending, got[types.StagePostProcess])
+}
+
+// TestBuildSpanTree_CancelledParseRendersCancelled records that a cancelled
+// document no longer shows pending spinners for the stages it never reached:
+// parse_status "cancelled" used to fall through the switch to pending.
+func TestBuildSpanTree_CancelledParseRendersCancelled(t *testing.T) {
+ tree, _, _ := buildSpanTree("kid-cancelled", 0, nil, types.ParseStatusCancelled)
+ a := assert.New(t)
+ a.Equal(types.SpanStatusCancelled, tree.Status)
+ a.Len(tree.Children, len(types.AllStages))
+ for _, child := range tree.Children {
+ a.Equal(types.SpanStatusCancelled, child.Status,
+ "a cancelled parse must not render its unreached stages as pending")
+ }
+}
+
+// TestBuildSpanTree_DocReaderFailureCancelsEverythingAfter covers the failure
+// at the first stage: nothing precedes it, so no stage should be skipped and
+// every later rows-less stage is cancelled.
+func TestBuildSpanTree_DocReaderFailureCancelsEverythingAfter(t *testing.T) {
+ now := time.Now()
+ rows := []types.KnowledgeProcessingSpan{
+ {KnowledgeID: "kid", Attempt: 1, SpanID: "root", Name: "knowledge_processing", Kind: types.SpanKindRoot, Status: types.SpanStatusFailed, StartedAt: &now},
+ {KnowledgeID: "kid", Attempt: 1, SpanID: "doc", ParentSpanID: "root", Name: types.StageDocReader, Kind: types.SpanKindStage, Status: types.SpanStatusFailed, ErrorCode: "DOCREADER_TIMEOUT", StartedAt: &now},
+ }
+
+ tree, _, _ := buildSpanTree("kid", 1, rows, types.ParseStatusFailed)
+ got := stageStatuses(t, tree)
+ a := assert.New(t)
+ a.Equal(types.SpanStatusFailed, got[types.StageDocReader])
+ for _, name := range []string{types.StageChunking, types.StageEmbedding, types.StageMultimodal, types.StagePostProcess} {
+ a.Equal(types.SpanStatusCancelled, got[name],
+ "%s follows the failed docreader and never ran", name)
+ }
+ a.NotContains(got, types.SpanStatusSkipped)
+}