Skip to content

Open-source LLM knowledge platform: turn raw documents into a queryable RAG, an autonomous reasoning agent, and a self-maintaining Wiki.

fix(wiki): stop housekeeping from force-failing documents with durable Wiki backlog

hujiaxin95c3a10ecc54
diff --git a/internal/application/service/knowledge_housekeeping.go b/internal/application/service/knowledge_housekeeping.go
index 95abd863b..4b4b3a968 100644
--- a/internal/application/service/knowledge_housekeeping.go
+++ b/internal/application/service/knowledge_housekeeping.go
@@ -42,8 +42,9 @@ type HousekeepingService struct {
// inspector lets the sweep distinguish a genuinely orphaned row from
// one whose enrichment subtasks are merely backlogged behind a busy
// queue (no span heartbeat yet because no worker has picked them up).
- // nil-safe — a nil inspector disables the queue check and the sweep
- // falls back to the span/updated_at heuristics alone.
+ // nil-safe — a nil inspector disables only the transient queue check.
+ // Durable Wiki ownership in task_pending_ops is always probed through db,
+ // so the sweep never falls back to the span/updated_at heuristics alone.
inspector interfaces.TaskInspector
mu sync.Mutex
@@ -268,23 +269,67 @@ func (h *HousekeepingService) filterByLastSpanActivity(ctx context.Context, cand
return out
}
-// filterOutQueued returns the subset of candidates that have NO task left
-// in the queue backend, plus a count of how many were dropped because a
-// task still references them. A dropped candidate is "backlogged, not
-// orphaned" — its enrichment subtasks are waiting for a worker, so the
-// missing span heartbeat is expected and recovering it would be a false
-// positive. When no inspector is wired (nil) the gate is a pass-through
-// so behaviour matches the pre-existing span-only sweep. On inspector
-// error we fail safe by KEEPING the candidate as stuck (recover it),
-// matching the span heartbeat query's fail-safe direction.
+// filterOutQueued returns the subset of candidates that have NO work left
+// in either the durable pending-op table or the queue backend, plus a count
+// of how many were dropped because work still references them. A dropped
+// candidate is "backlogged, not orphaned" — its enrichment subtasks are
+// waiting for a worker, so the missing span heartbeat is expected and
+// recovering it would be a false positive.
+//
+// The asynq inspector alone cannot answer this for Wiki. Wiki ingest work is
+// durable per document in task_pending_ops (dedup_key = knowledge ID), while
+// asynq only carries a per-KB wake-up trigger whose payload has no knowledge
+// ID — and TypeWikiIngest is deliberately absent from the inspector's
+// taskTypesForKnowledgeCancel set anyway. So HasQueuedTasksForKnowledge
+// structurally reports "nothing queued" for a document whose only outstanding
+// work is a queued Wiki ingest, and the sweep force-fails a perfectly healthy
+// row. Probe the durable table directly before consulting the inspector.
+//
+// When no inspector is wired (nil) the durable gate still applies; only the
+// transient queue check is skipped. On probe error we fail safe, but in
+// opposite directions by design: an inspector error KEEPS the candidate as
+// stuck (matching the span heartbeat query), whereas a durable-table error
+// DEFERS every candidate to the next sweep — we cannot tell backlog from
+// orphan without it, and wrongly failing a live document is not recoverable
+// by the user, while waiting one more interval is.
func (h *HousekeepingService) filterOutQueued(
ctx context.Context, candidates []types.Knowledge,
) (kept []types.Knowledge, skipped int) {
- if h.inspector == nil || len(candidates) == 0 {
+ if len(candidates) == 0 {
return candidates, 0
}
+
+ ids := make([]string, 0, len(candidates))
+ for _, k := range candidates {
+ ids = append(ids, k.ID)
+ }
+ var durableIDs []string
+ if err := h.db.WithContext(ctx).
+ Model(&types.TaskPendingOp{}).
+ Where("task_type = ? AND scope = ? AND op = ? AND dedup_key IN ?",
+ wikiTaskType, wikiTaskScope, WikiOpIngest, ids).
+ Distinct("dedup_key").
+ Pluck("dedup_key", &durableIDs).Error; err != nil {
+ logger.Warnf(ctx,
+ "[Housekeeping] durable queue probe failed: %v (deferring %d candidate(s))",
+ err, len(candidates))
+ return candidates[:0], len(candidates)
+ }
+ durable := make(map[string]struct{}, len(durableIDs))
+ for _, id := range durableIDs {
+ durable[id] = struct{}{}
+ }
+
out := candidates[:0]
for _, k := range candidates {
+ if _, ok := durable[k.ID]; ok {
+ skipped++
+ continue
+ }
+ if h.inspector == nil {
+ out = append(out, k)
+ continue
+ }
queued, err := h.inspector.HasQueuedTasksForKnowledge(ctx, k.ID)
if err != nil {
logger.Warnf(ctx,
diff --git a/internal/application/service/knowledge_housekeeping_test.go b/internal/application/service/knowledge_housekeeping_test.go
index 9c290df2b..7339102d1 100644
--- a/internal/application/service/knowledge_housekeeping_test.go
+++ b/internal/application/service/knowledge_housekeeping_test.go
@@ -68,15 +68,45 @@ CREATE TABLE IF NOT EXISTS knowledge_processing_spans (
);
`
+const housekeepingPendingOpsDDL = `
+CREATE TABLE IF NOT EXISTS task_pending_ops (
+ id INTEGER PRIMARY KEY AUTOINCREMENT,
+ tenant_id INTEGER NOT NULL DEFAULT 0,
+ task_type VARCHAR(64) NOT NULL,
+ scope VARCHAR(32) NOT NULL,
+ scope_id VARCHAR(64) NOT NULL,
+ op VARCHAR(32) NOT NULL,
+ dedup_key VARCHAR(128) NOT NULL DEFAULT '',
+ payload TEXT NOT NULL DEFAULT '{}',
+ fail_count INTEGER NOT NULL DEFAULT 0,
+ enqueued_at DATETIME DEFAULT CURRENT_TIMESTAMP,
+ claimed_at DATETIME
+);
+`
+
func setupHousekeepingDB(t *testing.T) *gorm.DB {
t.Helper()
db, err := gorm.Open(sqlite.Open(":memory:"), &gorm.Config{})
require.NoError(t, err)
require.NoError(t, db.Exec(knowledgeTestDDL).Error)
require.NoError(t, db.Exec(housekeepingSpansDDL).Error)
+ require.NoError(t, db.Exec(housekeepingPendingOpsDDL).Error)
return db
}
+// insertWikiPendingOp mirrors newWikiIngestPendingOp: the durable row is
+// scoped to the KB but deduplicated on the knowledge ID, which is exactly
+// why the per-knowledge asynq probe cannot see it.
+func insertWikiPendingOp(t *testing.T, db *gorm.DB, kbID, knowledgeID string) {
+ t.Helper()
+ require.NoError(t, db.Exec(
+ `INSERT INTO task_pending_ops (task_type, scope, scope_id, op, dedup_key, payload)
+ VALUES (?, ?, ?, ?, ?, ?)`,
+ wikiTaskType, wikiTaskScope, kbID, WikiOpIngest, knowledgeID,
+ `{"op":"ingest","knowledge_id":"`+knowledgeID+`"}`,
+ ).Error)
+}
+
// insertKnowledge writes a knowledge row at the given updated_at. We
// can't pass updated_at through GORM defaults since CURRENT_TIMESTAMP
// would override our test fixture; raw SQL keeps the timestamp.
@@ -270,6 +300,53 @@ func TestHousekeeping_NoFalseKill_TasksStillQueued(t *testing.T) {
"finalizing row with tasks still queued must NOT be flipped to failed")
}
+// A document whose only outstanding work is a queued Wiki ingest is
+// invisible to the asynq probe: the durable op lives in task_pending_ops
+// keyed by knowledge ID, while asynq holds only a per-KB trigger, and
+// TypeWikiIngest is not in taskTypesForKnowledgeCancel either. The
+// inspector below reports "nothing queued" — exactly what production does
+// — so without the durable gate the sweep force-fails a healthy row.
+func TestHousekeeping_NoFalseKill_DurableWikiIngestPending(t *testing.T) {
+ db := setupHousekeepingDB(t)
+ svc := newHousekeepingSvcWithInspector(db, fakeTaskInspector{})
+ stale := time.Now().Add(-3 * time.Hour)
+ insertKnowledge(t, db, "kid-durable-wiki", types.ParseStatusFinalizing, stale)
+ insertSpan(t, db, "kid-durable-wiki", 1, "wiki-1", types.SpanStatusRunning, stale)
+ insertWikiPendingOp(t, db, "kb-1", "kid-durable-wiki")
+
+ svc.runSweep(context.Background())
+
+ var status string
+ require.NoError(t, db.Raw(
+ `SELECT parse_status FROM knowledges WHERE id = ?`, "kid-durable-wiki",
+ ).Row().Scan(&status))
+ assert.Equal(t, types.ParseStatusFinalizing, status,
+ "row with a durable wiki ingest op pending must NOT be flipped to failed")
+}
+
+// The durable gate must not become a blanket amnesty: a stale row with no
+// pending op and nothing in the queue is still genuinely orphaned, and the
+// sweep must keep recovering it. This is the regression guard for the gate
+// itself.
+func TestHousekeeping_StillRecoversWhenNoDurableOp(t *testing.T) {
+ db := setupHousekeepingDB(t)
+ svc := newHousekeepingSvcWithInspector(db, fakeTaskInspector{})
+ stale := time.Now().Add(-3 * time.Hour)
+ insertKnowledge(t, db, "kid-orphan", types.ParseStatusFinalizing, stale)
+ insertSpan(t, db, "kid-orphan", 1, "wiki-1", types.SpanStatusRunning, stale)
+ // A pending op for a DIFFERENT document must not shield this one.
+ insertWikiPendingOp(t, db, "kb-1", "kid-someone-else")
+
+ svc.runSweep(context.Background())
+
+ var status string
+ require.NoError(t, db.Raw(
+ `SELECT parse_status FROM knowledges WHERE id = ?`, "kid-orphan",
+ ).Row().Scan(&status))
+ assert.Equal(t, types.ParseStatusFailed, status,
+ "row with no durable op and nothing queued must still be recovered")
+}
+
// TestHousekeeping_QueueProbeError_FailsSafe confirms the fail-safe
// direction: when the queue probe errors we still recover the row rather
// than leaving it stranded forever.