Open-source LLM knowledge platform: turn raw documents into a queryable RAG, an autonomous reasoning agent, and a self-maintaining Wiki.
Go agentagenticaichatbotdsh-pluginembeddingsevaluationgenerative-aigolangknowledge-basellmmulti-tenantollamaopenaiquestion-answeringragrerankingsemantic-searchvector-searchwiki
fix(knowledge): close spans left open on terminal rows and superseded attempts (#3855)
sunfac9ebabf2d4
diff --git a/internal/application/repository/knowledge_span_repo.go b/internal/application/repository/knowledge_span_repo.goindex 76fd8fde2..47634332a 100644--- a/internal/application/repository/knowledge_span_repo.go+++ b/internal/application/repository/knowledge_span_repo.go@@ -39,6 +39,12 @@ type KnowledgeSpanRepository interface {// running — a tree walk that stops at terminal parents would miss// those orphan leaves.CancelAllOpenSpans(ctx context.Context, knowledgeID string, attempt int, errorCode, reason string) (int64, error)+ // CancelOpenSpansBeforeAttempt flips every pending/running span belonging to+ // an attempt strictly older than `attempt` to cancelled. Reparse calls it so a+ // superseded attempt cannot leave a "running" root behind.+ CancelOpenSpansBeforeAttempt(+ ctx context.Context, knowledgeID string, attempt int, errorCode, reason string,+ ) (int64, error)// CancelOpenSpansByName flips pending/running rows with the given span// name for (knowledgeID, attempt). Used before re-opening a subspan// after asynq retry or server restart so the trace tree does not@@ -247,6 +253,33 @@ func (r *knowledgeSpanRepository) CancelAllOpenSpans(return res.RowsAffected, nil}+// CancelOpenSpansBeforeAttempt is CancelAllOpenSpans widened to every attempt+// older than `attempt`. A reparse allocates attempt N+1 while attempt N may+// still hold open spans (its worker died, or the row was failed without+// finalizing); those would otherwise live forever.+func (r *knowledgeSpanRepository) CancelOpenSpansBeforeAttempt(+ ctx context.Context, knowledgeID string, attempt int, errorCode, reason string,+) (int64, error) {+ errorCode = common.CleanInvalidUTF8(errorCode)+ reason = common.CleanInvalidUTF8(reason)+ now := time.Now()+ res := r.db.WithContext(ctx).Model(&types.KnowledgeProcessingSpan{}).+ Where("knowledge_id = ? AND attempt < ? AND status IN ?",+ knowledgeID, attempt,+ []string{types.SpanStatusPending, types.SpanStatusRunning}).+ Updates(map[string]any{+ "status": types.SpanStatusCancelled,+ "error_code": errorCode,+ "error_message": reason,+ "finished_at": now,+ "updated_at": now,+ })+ if res.Error != nil {+ return 0, res.Error+ }+ return res.RowsAffected, nil+}+func (r *knowledgeSpanRepository) CancelOpenSpansByName(ctx context.Context, knowledgeID string, attempt int, name, errorCode, reason string,) (int64, error) {diff --git a/internal/application/service/knowledge_housekeeping.go b/internal/application/service/knowledge_housekeeping.goindex 4cb126b51..3aa88a149 100644--- a/internal/application/service/knowledge_housekeeping.go+++ b/internal/application/service/knowledge_housekeeping.go@@ -69,6 +69,19 @@ type HousekeepingService struct {// far more often than a backlog changes.const queuedProbeTTL = time.Minute+// openSpanStatuses are the span states meaning "this span never finished".+var openSpanStatuses = []string{types.SpanStatusPending, types.SpanStatusRunning}++// terminalParseStatuses are knowledge states in which no pipeline can still be+// advancing, so any open span left on the row is orphaned by definition.+var terminalParseStatuses = []string{+ types.ParseStatusCompleted, types.ParseStatusFailed, types.ParseStatusCancelled,+}++// terminalSpanMessage is the fallback explanation for a span Sweep D closes on+// a row that carries no error message of its own.+const terminalSpanMessage = "row reached a terminal state without closing its trace spans"+// NewHousekeepingService constructs a HousekeepingService. It does NOT start// the cron — call Start in the application bootstrap so a misconfigured// cron schedule cannot prevent the rest of the service from coming up.@@ -289,6 +302,8 @@ func (h *HousekeepingService) runSweep(ctx context.Context) {}}}+ // Sweep D: terminal rows still carrying open spans.+ h.reapTerminalSpans(ctx)}// wikiHoldLimit bounds how long a durable Wiki ingest op keeps its row out of@@ -826,6 +841,62 @@ func (h *HousekeepingService) rearmWikiTriggers(}}+// reapTerminalSpans closes spans still open on rows that already reached a+// terminal state.+//+// Sweep A only selects pending/processing/finalizing, so a row marked terminal+// by a path that forgets to close its attempt is never revisited: its spans stay+// "running" and the timeline shows a permanent "进行中". Closing them here makes+// that whole class of leak self-healing, instead of depending on every failure+// path remembering to finalize, and it also retires rows stranded by older+// releases. The row's own error_message is carried over so+// the timeline still names the real cause rather than a generic stall.+func (h *HousekeepingService) reapTerminalSpans(ctx context.Context) {+ type openRow struct {+ ID int64 `gorm:"column:id"`+ ErrorMessage *string `gorm:"column:error_message"`+ }+ var rows []openRow+ if err := h.db.WithContext(ctx).+ Table("knowledge_processing_spans AS s").+ Select("s.id AS id", "k.error_message AS error_message").+ Joins("JOIN knowledges k ON k.id = s.knowledge_id").+ Where("s.status IN ?", openSpanStatuses).+ Where("k.parse_status IN ?", terminalParseStatuses).+ Find(&rows).Error; err != nil {+ logger.Warnf(ctx, "[Housekeeping] terminal span scan failed: %v", err)+ return+ }+ now := time.Now()+ var closed int64+ for _, r := range rows {+ msg := terminalSpanMessage+ if r.ErrorMessage != nil && strings.TrimSpace(*r.ErrorMessage) != "" {+ msg = strings.TrimSpace(*r.ErrorMessage)+ }+ if len(msg) > 1024 {+ msg = msg[:1024]+ }+ res := h.db.WithContext(ctx).Model(&types.KnowledgeProcessingSpan{}).+ Where("id = ? AND status IN ?", r.ID, openSpanStatuses).+ Updates(map[string]any{+ "status": types.SpanStatusCancelled,+ "error_code": werrors.ErrCodeTaskStalled,+ "error_message": msg,+ "finished_at": now,+ "updated_at": now,+ })+ if res.Error != nil {+ logger.Warnf(ctx, "[Housekeeping] close terminal span %d failed: %v", r.ID, res.Error)+ continue+ }+ closed += res.RowsAffected+ }+ if closed > 0 {+ logger.Infof(ctx, "[Housekeeping] closed %d orphaned span(s) on terminal row(s)", closed)+ }+}+// staleThreshold returns how long a "processing" row may sit untouched// before housekeeping treats it as orphaned. The floor is 1 hour so that a// genuinely slow large-PDF parse cannot be killed mid-flight; the ceilingdiff --git a/internal/application/service/knowledge_housekeeping_test.go b/internal/application/service/knowledge_housekeeping_test.goindex 1d40f122b..b1517ddef 100644--- a/internal/application/service/knowledge_housekeeping_test.go+++ b/internal/application/service/knowledge_housekeeping_test.go@@ -8,6 +8,7 @@ import ("time""github.com/Tencent/WeKnora/internal/config"+ werrors "github.com/Tencent/WeKnora/internal/errors""github.com/Tencent/WeKnora/internal/types""github.com/Tencent/WeKnora/internal/types/interfaces""github.com/stretchr/testify/assert"@@ -687,3 +688,61 @@ func (f *countingTaskInspector) QueuedKnowledgeIDs(context.Context) (map[string]}return out, nil}++// A row that reached a terminal state while its attempt was still open used+// to keep a "running" root span forever: Sweep A only selects non-terminal+// rows, so nothing ever revisited it and the document timeline showed a+// permanent spinner. Sweep D closes those spans.+func TestHousekeeping_ClosesOrphanedSpansOnTerminalRow(t *testing.T) {+ db := setupHousekeepingDB(t)+ svc := newHousekeepingSvcForTest(db)+ old := time.Now().Add(-72 * time.Hour)++ insertKnowledge(t, db, "kid-failed-open-span", types.ParseStatusFailed, old)+ require.NoError(t, db.Exec(+ `UPDATE knowledges SET error_message = '存储空间不足' WHERE id = ?`,+ "kid-failed-open-span",+ ).Error)+ require.NoError(t, db.Exec(+ `INSERT INTO knowledge_processing_spans+ (knowledge_id, attempt, span_id, name, kind, status, started_at, updated_at)+ VALUES (?, 1, 'root', 'knowledge_processing', 'root', 'running', ?, ?)`,+ "kid-failed-open-span", old, old,+ ).Error)++ svc.runSweep(context.Background())++ var status, code, msg string+ require.NoError(t, db.Raw(+ `SELECT status, error_code, error_message FROM knowledge_processing_spans+ WHERE knowledge_id = ? AND span_id = 'root'`,+ "kid-failed-open-span").Row().Scan(&status, &code, &msg))+ assert.Equal(t, types.SpanStatusCancelled, status)+ assert.Equal(t, werrors.ErrCodeTaskStalled, code)+ // The row's own error is carried over so the timeline names the real cause.+ assert.Equal(t, "存储空间不足", msg)+}++// Sweep D must not touch a row that is still legitimately in flight: that is+// Sweep A's call to make, and only after the heartbeat check.+func TestHousekeeping_KeepsOpenSpansOnInFlightRow(t *testing.T) {+ db := setupHousekeepingDB(t)+ svc := newHousekeepingSvcForTest(db)+ now := time.Now()++ insertKnowledge(t, db, "kid-processing", types.ParseStatusProcessing, now)+ require.NoError(t, db.Exec(+ `INSERT INTO knowledge_processing_spans+ (knowledge_id, attempt, span_id, name, kind, status, started_at, updated_at)+ VALUES (?, 1, 'root', 'knowledge_processing', 'root', 'running', ?, ?)`,+ "kid-processing", now, now,+ ).Error)++ svc.runSweep(context.Background())++ var status string+ require.NoError(t, db.Raw(+ `SELECT status FROM knowledge_processing_spans WHERE knowledge_id = ?`,+ "kid-processing").Row().Scan(&status))+ assert.Equal(t, types.SpanStatusRunning, status)+}diff --git a/internal/application/service/knowledge_span_tracker.go b/internal/application/service/knowledge_span_tracker.goindex 8aef1c05b..100c27d07 100644--- a/internal/application/service/knowledge_span_tracker.go+++ b/internal/application/service/knowledge_span_tracker.go@@ -251,6 +251,23 @@ func (t *spanTracker) OpenAttempt(ctx context.Context, knowledgeID, langfuseTracif err != nil {return nil, 0, err}+ // A new attempt supersedes every earlier one: close whatever the previous+ // attempt left open before opening the new root. A failure path that marks+ // the row terminal without finalizing its attempt leaves a "running" root+ // behind; housekeeping's stall sweep never revisits a terminal row, so+ // without this the old run would appear to progress alongside the new one+ // and would never be closed.+ if attempt > 1 {+ if n, err := t.repo.CancelOpenSpansBeforeAttempt(ctx, knowledgeID, attempt,+ "ATTEMPT_SUPERSEDED", "superseded by a newer parse attempt"); err != nil {+ logger.Warnf(ctx, "[SpanTracker] cancel superseded spans failed kid=%s attempt=%d: %v",+ knowledgeID, attempt, err)+ } else if n > 0 {+ logger.Infof(ctx,+ "[SpanTracker] cancelled %d superseded open span(s) for kid=%s before attempt %d",+ n, knowledgeID, attempt)+ }+ }now := time.Now()rootID := newSpanID()meta := types.JSONMap{}diff --git a/internal/application/service/knowledge_span_tracker_test.go b/internal/application/service/knowledge_span_tracker_test.goindex 219f94469..ba732fb39 100644--- a/internal/application/service/knowledge_span_tracker_test.go+++ b/internal/application/service/knowledge_span_tracker_test.go@@ -505,3 +505,35 @@ func TestSummaryQuestionPayload_AttemptRoundTrip(t *testing.T) {require.NoError(t, json.Unmarshal(qBytes, &qOut))assert.Equal(t, 5, qOut.Attempt)}++// A reparse must not inherit the previous attempt's open spans. A failure path+// that marks a row terminal without finalizing its attempt leaves a "running"+// root behind, and a terminal row is outside housekeeping's stall sweep, so the+// new attempt has to close it.+func TestSpanTracker_OpenAttemptCancelsSupersededOpenSpans(t *testing.T) {+ tracker, db := setupSpanTrackerTest(t)+ ctx := context.Background()++ _, attempt1, err := tracker.OpenAttempt(ctx, "kid-reparse", "")+ require.NoError(t, err)+ require.Equal(t, 1, attempt1)++ _, attempt2, err := tracker.OpenAttempt(ctx, "kid-reparse", "")+ require.NoError(t, err)+ require.Equal(t, 2, attempt2)++ var status, code string+ require.NoError(t, db.Raw(+ `SELECT status, error_code FROM knowledge_processing_spans+ WHERE knowledge_id = ? AND attempt = 1 AND kind = 'root'`,+ "kid-reparse").Row().Scan(&status, &code))+ assert.Equal(t, types.SpanStatusCancelled, status)+ assert.Equal(t, "ATTEMPT_SUPERSEDED", code)++ var newStatus string+ require.NoError(t, db.Raw(+ `SELECT status FROM knowledge_processing_spans+ WHERE knowledge_id = ? AND attempt = 2 AND kind = 'root'`,+ "kid-reparse").Row().Scan(&newStatus))+ assert.Equal(t, types.SpanStatusRunning, newStatus)+}