Skip to content

Open-source LLM knowledge platform: turn raw documents into a queryable RAG, an autonomous reasoning agent, and a self-maintaining Wiki.

fix(knowledge): close spans left open on terminal rows and superseded attempts (#3855)

sunfac9ebabf2d4
diff --git a/internal/application/repository/knowledge_span_repo.go b/internal/application/repository/knowledge_span_repo.go
index 76fd8fde2..47634332a 100644
--- a/internal/application/repository/knowledge_span_repo.go
+++ b/internal/application/repository/knowledge_span_repo.go
@@ -39,6 +39,12 @@ type KnowledgeSpanRepository interface {
// running — a tree walk that stops at terminal parents would miss
// those orphan leaves.
CancelAllOpenSpans(ctx context.Context, knowledgeID string, attempt int, errorCode, reason string) (int64, error)
+ // CancelOpenSpansBeforeAttempt flips every pending/running span belonging to
+ // an attempt strictly older than `attempt` to cancelled. Reparse calls it so a
+ // superseded attempt cannot leave a "running" root behind.
+ CancelOpenSpansBeforeAttempt(
+ ctx context.Context, knowledgeID string, attempt int, errorCode, reason string,
+ ) (int64, error)
// CancelOpenSpansByName flips pending/running rows with the given span
// name for (knowledgeID, attempt). Used before re-opening a subspan
// after asynq retry or server restart so the trace tree does not
@@ -247,6 +253,33 @@ func (r *knowledgeSpanRepository) CancelAllOpenSpans(
return res.RowsAffected, nil
}
+// CancelOpenSpansBeforeAttempt is CancelAllOpenSpans widened to every attempt
+// older than `attempt`. A reparse allocates attempt N+1 while attempt N may
+// still hold open spans (its worker died, or the row was failed without
+// finalizing); those would otherwise live forever.
+func (r *knowledgeSpanRepository) CancelOpenSpansBeforeAttempt(
+ ctx context.Context, knowledgeID string, attempt int, errorCode, reason string,
+) (int64, error) {
+ errorCode = common.CleanInvalidUTF8(errorCode)
+ reason = common.CleanInvalidUTF8(reason)
+ now := time.Now()
+ res := r.db.WithContext(ctx).Model(&types.KnowledgeProcessingSpan{}).
+ Where("knowledge_id = ? AND attempt < ? AND status IN ?",
+ knowledgeID, attempt,
+ []string{types.SpanStatusPending, types.SpanStatusRunning}).
+ Updates(map[string]any{
+ "status": types.SpanStatusCancelled,
+ "error_code": errorCode,
+ "error_message": reason,
+ "finished_at": now,
+ "updated_at": now,
+ })
+ if res.Error != nil {
+ return 0, res.Error
+ }
+ return res.RowsAffected, nil
+}
+
func (r *knowledgeSpanRepository) CancelOpenSpansByName(
ctx context.Context, knowledgeID string, attempt int, name, errorCode, reason string,
) (int64, error) {
diff --git a/internal/application/service/knowledge_housekeeping.go b/internal/application/service/knowledge_housekeeping.go
index 4cb126b51..3aa88a149 100644
--- a/internal/application/service/knowledge_housekeeping.go
+++ b/internal/application/service/knowledge_housekeeping.go
@@ -69,6 +69,19 @@ type HousekeepingService struct {
// far more often than a backlog changes.
const queuedProbeTTL = time.Minute
+// openSpanStatuses are the span states meaning "this span never finished".
+var openSpanStatuses = []string{types.SpanStatusPending, types.SpanStatusRunning}
+
+// terminalParseStatuses are knowledge states in which no pipeline can still be
+// advancing, so any open span left on the row is orphaned by definition.
+var terminalParseStatuses = []string{
+ types.ParseStatusCompleted, types.ParseStatusFailed, types.ParseStatusCancelled,
+}
+
+// terminalSpanMessage is the fallback explanation for a span Sweep D closes on
+// a row that carries no error message of its own.
+const terminalSpanMessage = "row reached a terminal state without closing its trace spans"
+
// NewHousekeepingService constructs a HousekeepingService. It does NOT start
// the cron — call Start in the application bootstrap so a misconfigured
// cron schedule cannot prevent the rest of the service from coming up.
@@ -289,6 +302,8 @@ func (h *HousekeepingService) runSweep(ctx context.Context) {
}
}
}
+ // Sweep D: terminal rows still carrying open spans.
+ h.reapTerminalSpans(ctx)
}
// wikiHoldLimit bounds how long a durable Wiki ingest op keeps its row out of
@@ -826,6 +841,62 @@ func (h *HousekeepingService) rearmWikiTriggers(
}
}
+// reapTerminalSpans closes spans still open on rows that already reached a
+// terminal state.
+//
+// Sweep A only selects pending/processing/finalizing, so a row marked terminal
+// by a path that forgets to close its attempt is never revisited: its spans stay
+// "running" and the timeline shows a permanent "进行中". Closing them here makes
+// that whole class of leak self-healing, instead of depending on every failure
+// path remembering to finalize, and it also retires rows stranded by older
+// releases. The row's own error_message is carried over so
+// the timeline still names the real cause rather than a generic stall.
+func (h *HousekeepingService) reapTerminalSpans(ctx context.Context) {
+ type openRow struct {
+ ID int64 `gorm:"column:id"`
+ ErrorMessage *string `gorm:"column:error_message"`
+ }
+ var rows []openRow
+ if err := h.db.WithContext(ctx).
+ Table("knowledge_processing_spans AS s").
+ Select("s.id AS id", "k.error_message AS error_message").
+ Joins("JOIN knowledges k ON k.id = s.knowledge_id").
+ Where("s.status IN ?", openSpanStatuses).
+ Where("k.parse_status IN ?", terminalParseStatuses).
+ Find(&rows).Error; err != nil {
+ logger.Warnf(ctx, "[Housekeeping] terminal span scan failed: %v", err)
+ return
+ }
+ now := time.Now()
+ var closed int64
+ for _, r := range rows {
+ msg := terminalSpanMessage
+ if r.ErrorMessage != nil && strings.TrimSpace(*r.ErrorMessage) != "" {
+ msg = strings.TrimSpace(*r.ErrorMessage)
+ }
+ if len(msg) > 1024 {
+ msg = msg[:1024]
+ }
+ res := h.db.WithContext(ctx).Model(&types.KnowledgeProcessingSpan{}).
+ Where("id = ? AND status IN ?", r.ID, openSpanStatuses).
+ Updates(map[string]any{
+ "status": types.SpanStatusCancelled,
+ "error_code": werrors.ErrCodeTaskStalled,
+ "error_message": msg,
+ "finished_at": now,
+ "updated_at": now,
+ })
+ if res.Error != nil {
+ logger.Warnf(ctx, "[Housekeeping] close terminal span %d failed: %v", r.ID, res.Error)
+ continue
+ }
+ closed += res.RowsAffected
+ }
+ if closed > 0 {
+ logger.Infof(ctx, "[Housekeeping] closed %d orphaned span(s) on terminal row(s)", closed)
+ }
+}
+
// staleThreshold returns how long a "processing" row may sit untouched
// before housekeeping treats it as orphaned. The floor is 1 hour so that a
// genuinely slow large-PDF parse cannot be killed mid-flight; the ceiling
diff --git a/internal/application/service/knowledge_housekeeping_test.go b/internal/application/service/knowledge_housekeeping_test.go
index 1d40f122b..b1517ddef 100644
--- a/internal/application/service/knowledge_housekeeping_test.go
+++ b/internal/application/service/knowledge_housekeeping_test.go
@@ -8,6 +8,7 @@ import (
"time"
"github.com/Tencent/WeKnora/internal/config"
+ werrors "github.com/Tencent/WeKnora/internal/errors"
"github.com/Tencent/WeKnora/internal/types"
"github.com/Tencent/WeKnora/internal/types/interfaces"
"github.com/stretchr/testify/assert"
@@ -687,3 +688,61 @@ func (f *countingTaskInspector) QueuedKnowledgeIDs(context.Context) (map[string]
}
return out, nil
}
+
+// A row that reached a terminal state while its attempt was still open used
+// to keep a "running" root span forever: Sweep A only selects non-terminal
+// rows, so nothing ever revisited it and the document timeline showed a
+// permanent spinner. Sweep D closes those spans.
+func TestHousekeeping_ClosesOrphanedSpansOnTerminalRow(t *testing.T) {
+ db := setupHousekeepingDB(t)
+ svc := newHousekeepingSvcForTest(db)
+ old := time.Now().Add(-72 * time.Hour)
+
+ insertKnowledge(t, db, "kid-failed-open-span", types.ParseStatusFailed, old)
+ require.NoError(t, db.Exec(
+ `UPDATE knowledges SET error_message = '存储空间不足' WHERE id = ?`,
+ "kid-failed-open-span",
+ ).Error)
+ require.NoError(t, db.Exec(
+ `INSERT INTO knowledge_processing_spans
+ (knowledge_id, attempt, span_id, name, kind, status, started_at, updated_at)
+ VALUES (?, 1, 'root', 'knowledge_processing', 'root', 'running', ?, ?)`,
+ "kid-failed-open-span", old, old,
+ ).Error)
+
+ svc.runSweep(context.Background())
+
+ var status, code, msg string
+ require.NoError(t, db.Raw(
+ `SELECT status, error_code, error_message FROM knowledge_processing_spans
+ WHERE knowledge_id = ? AND span_id = 'root'`,
+ "kid-failed-open-span").Row().Scan(&status, &code, &msg))
+ assert.Equal(t, types.SpanStatusCancelled, status)
+ assert.Equal(t, werrors.ErrCodeTaskStalled, code)
+ // The row's own error is carried over so the timeline names the real cause.
+ assert.Equal(t, "存储空间不足", msg)
+}
+
+// Sweep D must not touch a row that is still legitimately in flight: that is
+// Sweep A's call to make, and only after the heartbeat check.
+func TestHousekeeping_KeepsOpenSpansOnInFlightRow(t *testing.T) {
+ db := setupHousekeepingDB(t)
+ svc := newHousekeepingSvcForTest(db)
+ now := time.Now()
+
+ insertKnowledge(t, db, "kid-processing", types.ParseStatusProcessing, now)
+ require.NoError(t, db.Exec(
+ `INSERT INTO knowledge_processing_spans
+ (knowledge_id, attempt, span_id, name, kind, status, started_at, updated_at)
+ VALUES (?, 1, 'root', 'knowledge_processing', 'root', 'running', ?, ?)`,
+ "kid-processing", now, now,
+ ).Error)
+
+ svc.runSweep(context.Background())
+
+ var status string
+ require.NoError(t, db.Raw(
+ `SELECT status FROM knowledge_processing_spans WHERE knowledge_id = ?`,
+ "kid-processing").Row().Scan(&status))
+ assert.Equal(t, types.SpanStatusRunning, status)
+}
diff --git a/internal/application/service/knowledge_span_tracker.go b/internal/application/service/knowledge_span_tracker.go
index 8aef1c05b..100c27d07 100644
--- a/internal/application/service/knowledge_span_tracker.go
+++ b/internal/application/service/knowledge_span_tracker.go
@@ -251,6 +251,23 @@ func (t *spanTracker) OpenAttempt(ctx context.Context, knowledgeID, langfuseTrac
if err != nil {
return nil, 0, err
}
+ // A new attempt supersedes every earlier one: close whatever the previous
+ // attempt left open before opening the new root. A failure path that marks
+ // the row terminal without finalizing its attempt leaves a "running" root
+ // behind; housekeeping's stall sweep never revisits a terminal row, so
+ // without this the old run would appear to progress alongside the new one
+ // and would never be closed.
+ if attempt > 1 {
+ if n, err := t.repo.CancelOpenSpansBeforeAttempt(ctx, knowledgeID, attempt,
+ "ATTEMPT_SUPERSEDED", "superseded by a newer parse attempt"); err != nil {
+ logger.Warnf(ctx, "[SpanTracker] cancel superseded spans failed kid=%s attempt=%d: %v",
+ knowledgeID, attempt, err)
+ } else if n > 0 {
+ logger.Infof(ctx,
+ "[SpanTracker] cancelled %d superseded open span(s) for kid=%s before attempt %d",
+ n, knowledgeID, attempt)
+ }
+ }
now := time.Now()
rootID := newSpanID()
meta := types.JSONMap{}
diff --git a/internal/application/service/knowledge_span_tracker_test.go b/internal/application/service/knowledge_span_tracker_test.go
index 219f94469..ba732fb39 100644
--- a/internal/application/service/knowledge_span_tracker_test.go
+++ b/internal/application/service/knowledge_span_tracker_test.go
@@ -505,3 +505,35 @@ func TestSummaryQuestionPayload_AttemptRoundTrip(t *testing.T) {
require.NoError(t, json.Unmarshal(qBytes, &qOut))
assert.Equal(t, 5, qOut.Attempt)
}
+
+// A reparse must not inherit the previous attempt's open spans. A failure path
+// that marks a row terminal without finalizing its attempt leaves a "running"
+// root behind, and a terminal row is outside housekeeping's stall sweep, so the
+// new attempt has to close it.
+func TestSpanTracker_OpenAttemptCancelsSupersededOpenSpans(t *testing.T) {
+ tracker, db := setupSpanTrackerTest(t)
+ ctx := context.Background()
+
+ _, attempt1, err := tracker.OpenAttempt(ctx, "kid-reparse", "")
+ require.NoError(t, err)
+ require.Equal(t, 1, attempt1)
+
+ _, attempt2, err := tracker.OpenAttempt(ctx, "kid-reparse", "")
+ require.NoError(t, err)
+ require.Equal(t, 2, attempt2)
+
+ var status, code string
+ require.NoError(t, db.Raw(
+ `SELECT status, error_code FROM knowledge_processing_spans
+ WHERE knowledge_id = ? AND attempt = 1 AND kind = 'root'`,
+ "kid-reparse").Row().Scan(&status, &code))
+ assert.Equal(t, types.SpanStatusCancelled, status)
+ assert.Equal(t, "ATTEMPT_SUPERSEDED", code)
+
+ var newStatus string
+ require.NoError(t, db.Raw(
+ `SELECT status FROM knowledge_processing_spans
+ WHERE knowledge_id = ? AND attempt = 2 AND kind = 'root'`,
+ "kid-reparse").Row().Scan(&newStatus))
+ assert.Equal(t, types.SpanStatusRunning, newStatus)
+}