Skip to content

Open-source LLM knowledge platform: turn raw documents into a queryable RAG, an autonomous reasoning agent, and a self-maintaining Wiki.

fix(knowledge): recover stranded deleting rows and align the KB count (#3367)

Fei.Chenc8ebb9ed96fa
diff --git a/internal/application/repository/knowledge.go b/internal/application/repository/knowledge.go
index 030e1fc91..4b29c5520 100644
--- a/internal/application/repository/knowledge.go
+++ b/internal/application/repository/knowledge.go
@@ -745,8 +745,12 @@ func (r *knowledgeRepository) CountKnowledgeByKnowledgeBaseID(
kbID string,
) (int64, error) {
var count int64
+ // Mirror the document list's view (applyKnowledgeListFilter): rows
+ // mid-deletion are hidden there, so counting them here is what produced
+ // the "4 documents, 3 listed" ghost on the KB card (issues #3338/#3345).
err := r.db.WithContext(ctx).Model(&types.Knowledge{}).
- Where("tenant_id = ? AND knowledge_base_id = ?", tenantID, kbID).
+ Where("tenant_id = ? AND knowledge_base_id = ? AND parse_status <> ?",
+ tenantID, kbID, types.ParseStatusDeleting).
Count(&count).Error
return count, err
}
diff --git a/internal/application/repository/knowledge_duplicate_test.go b/internal/application/repository/knowledge_duplicate_test.go
index 32f188009..56256055c 100644
--- a/internal/application/repository/knowledge_duplicate_test.go
+++ b/internal/application/repository/knowledge_duplicate_test.go
@@ -138,3 +138,28 @@ func TestCheckKnowledgeExists_ParseStatusMatrix(t *testing.T) {
})
}
}
+
+// TestCountKnowledgeByKnowledgeBaseID_ExcludesDeleting pins the count/list
+// alignment (issues #3338/#3345): the document list hides rows mid-deletion,
+// so the KB card count must not include them either — otherwise a stranded
+// delete shows up as "4 documents, 3 listed".
+func TestCountKnowledgeByKnowledgeBaseID_ExcludesDeleting(t *testing.T) {
+ db := setupKnowledgeTestDB(t)
+ repo := NewKnowledgeRepository(db)
+ for _, tc := range []struct{ id, status string }{
+ {"k-done", "completed"},
+ {"k-failed", "failed"},
+ {"k-deleting", "deleting"},
+ } {
+ require.NoError(t, db.Exec(`
+ INSERT INTO knowledges (id, tenant_id, knowledge_base_id, type, file_name, parse_status)
+ VALUES (?, 1, 'kb-cnt', 'file', ?, ?)
+ `, tc.id, tc.id+".md", tc.status).Error)
+ }
+
+ count, err := repo.CountKnowledgeByKnowledgeBaseID(context.Background(), 1, "kb-cnt")
+ require.NoError(t, err)
+ // failed rows stay counted (they are visible, actionable cards); only
+ // the hidden mid-deletion row drops out.
+ require.Equal(t, int64(2), count)
+}
diff --git a/internal/application/service/knowledge_housekeeping.go b/internal/application/service/knowledge_housekeeping.go
index 4b4b3a968..57b9cf7f2 100644
--- a/internal/application/service/knowledge_housekeeping.go
+++ b/internal/application/service/knowledge_housekeeping.go
@@ -210,6 +210,60 @@ func (h *HousekeepingService) runSweep(ctx context.Context) {
} else if resSummary.RowsAffected > 0 {
logger.Infof(ctx, "[Housekeeping] recovered %d stuck summary rows", resSummary.RowsAffected)
}
+
+ // Sweep C: rows stuck in "deleting" whose delete task no longer exists
+ // (issues #3338/#3345). The delete dead-letter callback only recovers
+ // rows that are still "deleting" when retries exhaust; a worker death
+ // mid-delete or a lost queue leaves the row hidden from the document
+ // list forever. Same recovery contract as Sweep A: flip to failed with
+ // a clear error so the row becomes visible and actionable (the user's
+ // delete intent can then be retried; the delete executor is idempotent
+ // and plans by row ID, so a late-arriving task still finishes cleanly).
+ //
+ // Liveness gate: a backlogged delete is not stranded — probe the queue
+ // for a live knowledge:list_delete covering the row before touching it.
+ // A nil inspector (nothing wired at all) defers every candidate to the
+ // next sweep: we cannot tell backlog from orphan without it, and
+ // wrongly failing a live delete is visible to users, while waiting one
+ // more interval is not. A probe error defers that single row.
+ if h.inspector != nil {
+ var strandedDeletes []types.Knowledge
+ if err := h.db.WithContext(ctx).
+ Where("parse_status = ? AND updated_at < ?", types.ParseStatusDeleting, cutoff).
+ Find(&strandedDeletes).Error; err != nil {
+ logger.Warnf(ctx, "[Housekeeping] deleting candidate query failed: %v", err)
+ } else {
+ recoveredDeletes := int64(0)
+ for _, k := range strandedDeletes {
+ queued, err := h.inspector.HasQueuedDeleteTasksForKnowledge(ctx, k.ID)
+ if err != nil {
+ logger.Warnf(ctx,
+ "[Housekeeping] delete-task probe failed for %s: %v (deferring to next sweep)",
+ k.ID, err)
+ continue
+ }
+ if queued {
+ continue
+ }
+ res := h.db.WithContext(ctx).Model(&types.Knowledge{}).
+ Where("id = ? AND parse_status = ?", k.ID, types.ParseStatusDeleting).
+ Updates(map[string]interface{}{
+ "parse_status": types.ParseStatusFailed,
+ "error_message": "delete task stranded (no queued/active delete task) > " + threshold.String() + ", recovered by housekeeping",
+ })
+ if res.Error != nil {
+ logger.Warnf(ctx, "[Housekeeping] delete sweep update failed for %s: %v", k.ID, res.Error)
+ continue
+ }
+ recoveredDeletes += res.RowsAffected
+ }
+ if recoveredDeletes > 0 {
+ logger.Infof(ctx,
+ "[Housekeeping] recovered %d stranded deleting row(s) (threshold=%s)",
+ recoveredDeletes, threshold)
+ }
+ }
+ }
}
// filterByLastSpanActivity returns the subset of candidates whose most
diff --git a/internal/application/service/knowledge_housekeeping_test.go b/internal/application/service/knowledge_housekeeping_test.go
index f0491c606..c0f1938a5 100644
--- a/internal/application/service/knowledge_housekeeping_test.go
+++ b/internal/application/service/knowledge_housekeeping_test.go
@@ -133,7 +133,10 @@ func insertSpan(t *testing.T, db *gorm.DB, kid string, attempt int, spanID, stat
// the probe to fail so the fail-safe branch can be exercised.
type fakeTaskInspector struct {
queued map[string]bool
- err error
+ // deleteQueued independently controls the delete-task liveness probe;
+ // nil means "no delete task alive" for every ID.
+ deleteQueued map[string]bool
+ err error
}
func (f fakeTaskInspector) CancelTasksForKnowledge(
@@ -151,6 +154,15 @@ func (f fakeTaskInspector) HasQueuedTasksForKnowledge(
return f.queued[knowledgeID], nil
}
+func (f fakeTaskInspector) HasQueuedDeleteTasksForKnowledge(
+ _ context.Context, knowledgeID string,
+) (bool, error) {
+ if f.err != nil {
+ return false, f.err
+ }
+ return f.deleteQueued[knowledgeID], nil
+}
+
func (f fakeTaskInspector) QueueStats(
_ context.Context,
) ([]types.QueueStat, bool, error) {
@@ -386,3 +398,88 @@ func TestHousekeeping_PreservesRecentlyTouched(t *testing.T) {
assert.Equal(t, types.ParseStatusProcessing, status,
"knowledge updated within the cutoff must be left alone")
}
+
+// --- Sweep C: stranded "deleting" rows (issues #3338/#3345) --—
+
+func readKnowledgeStatus(t *testing.T, db *gorm.DB, id string) (string, string) {
+ t.Helper()
+ var status, errMsg string
+ require.NoError(t, db.Raw(
+ `SELECT parse_status, COALESCE(error_message, '') FROM knowledges WHERE id = ?`, id,
+ ).Row().Scan(&status, &errMsg))
+ return status, errMsg
+}
+
+func TestHousekeeping_RecoversStrandedDeletingRow(t *testing.T) {
+ db := setupHousekeepingDB(t)
+ // No deleteQueued entry: the delete task is gone (worker death / lost
+ // queue), the dead-letter path never fired, the row is stranded.
+ svc := newHousekeepingSvcForTest(db)
+ stale := time.Now().Add(-3 * time.Hour)
+ insertKnowledge(t, db, "kid-stranded-delete", types.ParseStatusDeleting, stale)
+
+ svc.runSweep(context.Background())
+
+ status, errMsg := readKnowledgeStatus(t, db, "kid-stranded-delete")
+ assert.Equal(t, types.ParseStatusFailed, status)
+ assert.Contains(t, errMsg, "stranded")
+}
+
+func TestHousekeeping_KeepsBackloggedDeletingRow(t *testing.T) {
+ db := setupHousekeepingDB(t)
+ // A live (queued/active/retry) knowledge:list_delete still covers the
+ // row — backpressure, not stranded; must stay untouched.
+ svc := newHousekeepingSvcWithInspector(db, fakeTaskInspector{
+ deleteQueued: map[string]bool{"kid-backlogged-delete": true},
+ })
+ insertKnowledge(t, db, "kid-backlogged-delete", types.ParseStatusDeleting,
+ time.Now().Add(-3*time.Hour))
+
+ svc.runSweep(context.Background())
+
+ status, _ := readKnowledgeStatus(t, db, "kid-backlogged-delete")
+ assert.Equal(t, types.ParseStatusDeleting, status)
+}
+
+func TestHousekeeping_FreshDeletingRowUntouched(t *testing.T) {
+ db := setupHousekeepingDB(t)
+ svc := newHousekeepingSvcForTest(db)
+ // Marked deleting moments ago: below the staleness cutoff, so the sweep
+ // must not even consider it (a delete just handed to the queue).
+ insertKnowledge(t, db, "kid-fresh-delete", types.ParseStatusDeleting, time.Now())
+
+ svc.runSweep(context.Background())
+
+ status, _ := readKnowledgeStatus(t, db, "kid-fresh-delete")
+ assert.Equal(t, types.ParseStatusDeleting, status)
+}
+
+func TestHousekeeping_DeletingProbeErrorDefers(t *testing.T) {
+ db := setupHousekeepingDB(t)
+ // Backend probe error must defer the row to the next sweep rather than
+ // guess: failing a live delete would surface a wrong "failed" card.
+ svc := newHousekeepingSvcWithInspector(db, fakeTaskInspector{
+ err: assert.AnError,
+ })
+ insertKnowledge(t, db, "kid-probe-err-delete", types.ParseStatusDeleting,
+ time.Now().Add(-3*time.Hour))
+
+ svc.runSweep(context.Background())
+
+ status, _ := readKnowledgeStatus(t, db, "kid-probe-err-delete")
+ assert.Equal(t, types.ParseStatusDeleting, status)
+}
+
+func TestHousekeeping_NilInspectorDefersDeletingSweep(t *testing.T) {
+ db := setupHousekeepingDB(t)
+ // Without any inspector wired there is no way to tell backlog from
+ // orphan, so the whole Sweep C defers — never guesses.
+ svc := newHousekeepingSvcWithInspector(db, nil)
+ insertKnowledge(t, db, "kid-nil-inspector", types.ParseStatusDeleting,
+ time.Now().Add(-3*time.Hour))
+
+ svc.runSweep(context.Background())
+
+ status, _ := readKnowledgeStatus(t, db, "kid-nil-inspector")
+ assert.Equal(t, types.ParseStatusDeleting, status)
+}
diff --git a/internal/application/service/knowledgebase_task_cancel_test.go b/internal/application/service/knowledgebase_task_cancel_test.go
index 250b86079..30d775362 100644
--- a/internal/application/service/knowledgebase_task_cancel_test.go
+++ b/internal/application/service/knowledgebase_task_cancel_test.go
@@ -35,6 +35,10 @@ func (r *recordingKBTaskInspector) CancelTasksForKnowledge(
return 0, 0, nil
}
+func (r *recordingKBTaskInspector) HasQueuedDeleteTasksForKnowledge(context.Context, string) (bool, error) {
+ return false, nil
+}
+
func (r *recordingKBTaskInspector) HasQueuedTasksForKnowledge(context.Context, string) (bool, error) {
return false, nil
}
diff --git a/internal/handler/system_runtime_test.go b/internal/handler/system_runtime_test.go
index c7e3c3c83..3dfe7d5de 100644
--- a/internal/handler/system_runtime_test.go
+++ b/internal/handler/system_runtime_test.go
@@ -59,6 +59,10 @@ type runtimeTestInspector struct{}
func (runtimeTestInspector) CancelTasksForKnowledge(context.Context, string) (int, int, error) {
return 0, 0, nil
}
+func (runtimeTestInspector) HasQueuedDeleteTasksForKnowledge(context.Context, string) (bool, error) {
+ return false, nil
+}
+
func (runtimeTestInspector) HasQueuedTasksForKnowledge(context.Context, string) (bool, error) {
return false, nil
}
diff --git a/internal/router/task_inspector.go b/internal/router/task_inspector.go
index bdef9a307..b4404c5dc 100644
--- a/internal/router/task_inspector.go
+++ b/internal/router/task_inspector.go
@@ -159,6 +159,28 @@ func (a *asynqTaskInspector) CancelTasksForKnowledgeBase(
return deleted, cancelled, nil
}
+// matchesKnowledgeListDelete identifies a knowledge:list_delete task whose
+// batch payload covers knowledgeID. Delete tasks carry knowledge_ids (plural)
+// and are deliberately outside taskTypesForKnowledgeCancel, so they need
+// their own matcher for the housekeeping delete sweep's liveness probe.
+func matchesKnowledgeListDelete(taskType string, payload []byte, knowledgeID string) bool {
+ if taskType != types.TypeKnowledgeListDelete {
+ return false
+ }
+ var probe struct {
+ KnowledgeIDs []string `json:"knowledge_ids"`
+ }
+ if err := json.Unmarshal(payload, &probe); err != nil {
+ return false
+ }
+ for _, id := range probe.KnowledgeIDs {
+ if id == knowledgeID {
+ return true
+ }
+ }
+ return false
+}
+
// HasQueuedTasksForKnowledge reports whether any pending / scheduled /
// retry / active task referencing knowledgeID still lives in the queue.
// Read-only counterpart of CancelTasksForKnowledge — the housekeeping
@@ -183,6 +205,29 @@ func (a *asynqTaskInspector) HasQueuedTasksForKnowledge(
return false, nil
}
+// HasQueuedDeleteTasksForKnowledge is the delete-task counterpart of
+// HasQueuedTasksForKnowledge: it matches knowledge:list_delete batch
+// payloads that still cover the knowledge ID. The housekeeping delete
+// sweep uses it to protect backlogged-but-alive deletes from recovery.
+func (a *asynqTaskInspector) HasQueuedDeleteTasksForKnowledge(
+ ctx context.Context, knowledgeID string,
+) (bool, error) {
+ if a == nil || a.inspector == nil || knowledgeID == "" {
+ return false, nil
+ }
+ matcher := func(taskType string, payload []byte) bool {
+ return matchesKnowledgeListDelete(taskType, payload, knowledgeID)
+ }
+ for _, queue := range queuesScanned {
+ for _, state := range a.cancellableTaskStates() {
+ if a.queueStateHasMatch(ctx, queue, state.name, state.list, matcher) {
+ return true, nil
+ }
+ }
+ }
+ return false, nil
+}
+
// QueueStats returns a depth snapshot for every queue this app enqueues
// into. Read-only: it calls Inspector.GetQueueInfo per queue and maps
// the result onto types.QueueStat, attaching static pool/weight metadata
@@ -1081,6 +1126,15 @@ func (noopTaskInspector) HasQueuedTasksForKnowledge(
return false, nil
}
+// HasQueuedDeleteTasksForKnowledge always reports false in Lite mode:
+// deletes also run inline there, so a stale "deleting" row genuinely has
+// no task left and the delete sweep may recover it.
+func (noopTaskInspector) HasQueuedDeleteTasksForKnowledge(
+ ctx context.Context, knowledgeID string,
+) (bool, error) {
+ return false, nil
+}
+
// QueueStats reports "not supported" in Lite mode: there is no Redis /
// asynq backend to inspect, so the runtime dashboard renders an
// "unavailable in this deployment" state instead of an empty table.
diff --git a/internal/router/task_inspector_errors_test.go b/internal/router/task_inspector_errors_test.go
index 2f93356aa..59ca9cc1a 100644
--- a/internal/router/task_inspector_errors_test.go
+++ b/internal/router/task_inspector_errors_test.go
@@ -5,7 +5,9 @@ import (
"fmt"
"testing"
+ "github.com/Tencent/WeKnora/internal/types"
"github.com/hibiken/asynq"
+ "github.com/stretchr/testify/require"
)
func TestIsAsynqQueueNotFound(t *testing.T) {
@@ -35,3 +37,26 @@ func TestIsAsynqQueueNotFound(t *testing.T) {
})
}
}
+
+func TestMatchesKnowledgeListDelete(t *testing.T) {
+ const payload = `{"knowledge_base_id":"kb-1","tenant_id":7,"knowledge_ids":["kid-a","kid-b"]}`
+ cases := []struct {
+ name string
+ taskType string
+ payload string
+ kid string
+ want bool
+ }{
+ {"covered id", types.TypeKnowledgeListDelete, payload, "kid-a", true},
+ {"last id", types.TypeKnowledgeListDelete, payload, "kid-b", true},
+ {"uncovered id", types.TypeKnowledgeListDelete, payload, "kid-c", false},
+ {"other task type", types.TypeDocumentProcess, payload, "kid-a", false},
+ {"bad json", types.TypeKnowledgeListDelete, "{", "kid-a", false},
+ {"empty list", types.TypeKnowledgeListDelete, `{"knowledge_ids":[]}`, "kid-a", false},
+ }
+ for _, tc := range cases {
+ t.Run(tc.name, func(t *testing.T) {
+ require.Equal(t, tc.want, matchesKnowledgeListDelete(tc.taskType, []byte(tc.payload), tc.kid))
+ })
+ }
+}
diff --git a/internal/types/interfaces/task_inspector.go b/internal/types/interfaces/task_inspector.go
index 593c7fe44..6badea746 100644
--- a/internal/types/interfaces/task_inspector.go
+++ b/internal/types/interfaces/task_inspector.go
@@ -44,6 +44,18 @@ type TaskInspector interface {
// span/updated_at checks remain authoritative there.
HasQueuedTasksForKnowledge(ctx context.Context, knowledgeID string) (bool, error)
+ // HasQueuedDeleteTasksForKnowledge reports whether any pending /
+ // scheduled / retry / active knowledge:list_delete task still covers
+ // the given knowledge ID. The delete sweep calls it before recovering
+ // a stranded "deleting" row: the delete task is enqueued with a batch
+ // payload (knowledge_ids list) that the per-parse matcher above cannot
+ // see, so without this probe the sweep could not tell a stranded delete
+ // from one that is merely backlogged behind a busy maintenance queue.
+ //
+ // Same fail-safe contract: (false, err) on backend error, callers defer;
+ // Lite mode always returns false — inline deletes never queue.
+ HasQueuedDeleteTasksForKnowledge(ctx context.Context, knowledgeID string) (bool, error)
+
// QueueStats returns a read-only depth snapshot for every queue this
// application enqueues into, for the System Admin runtime dashboard.
//