Open-source LLM knowledge platform: turn raw documents into a queryable RAG, an autonomous reasoning agent, and a self-maintaining Wiki.
Go agentagenticaichatbotdsh-pluginembeddingsevaluationgenerative-aigolangknowledge-basellmmulti-tenantollamaopenaiquestion-answeringragrerankingsemantic-searchvector-searchwiki
fix(knowledge): recover stranded deleting rows and align the KB count (#3367)
Fei.Chenc8ebb9ed96fa
diff --git a/internal/application/repository/knowledge.go b/internal/application/repository/knowledge.goindex 030e1fc91..4b29c5520 100644--- a/internal/application/repository/knowledge.go+++ b/internal/application/repository/knowledge.go@@ -745,8 +745,12 @@ func (r *knowledgeRepository) CountKnowledgeByKnowledgeBaseID(kbID string,) (int64, error) {var count int64+ // Mirror the document list's view (applyKnowledgeListFilter): rows+ // mid-deletion are hidden there, so counting them here is what produced+ // the "4 documents, 3 listed" ghost on the KB card (issues #3338/#3345).err := r.db.WithContext(ctx).Model(&types.Knowledge{}).- Where("tenant_id = ? AND knowledge_base_id = ?", tenantID, kbID).+ Where("tenant_id = ? AND knowledge_base_id = ? AND parse_status <> ?",+ tenantID, kbID, types.ParseStatusDeleting).Count(&count).Errorreturn count, err}diff --git a/internal/application/repository/knowledge_duplicate_test.go b/internal/application/repository/knowledge_duplicate_test.goindex 32f188009..56256055c 100644--- a/internal/application/repository/knowledge_duplicate_test.go+++ b/internal/application/repository/knowledge_duplicate_test.go@@ -138,3 +138,28 @@ func TestCheckKnowledgeExists_ParseStatusMatrix(t *testing.T) {})}}++// TestCountKnowledgeByKnowledgeBaseID_ExcludesDeleting pins the count/list+// alignment (issues #3338/#3345): the document list hides rows mid-deletion,+// so the KB card count must not include them either — otherwise a stranded+// delete shows up as "4 documents, 3 listed".+func TestCountKnowledgeByKnowledgeBaseID_ExcludesDeleting(t *testing.T) {+ db := setupKnowledgeTestDB(t)+ repo := NewKnowledgeRepository(db)+ for _, tc := range []struct{ id, status string }{+ {"k-done", "completed"},+ {"k-failed", "failed"},+ {"k-deleting", "deleting"},+ } {+ require.NoError(t, db.Exec(`+ INSERT INTO knowledges (id, tenant_id, knowledge_base_id, type, file_name, parse_status)+ VALUES (?, 1, 'kb-cnt', 'file', ?, ?)+ `, tc.id, tc.id+".md", tc.status).Error)+ }++ count, err := repo.CountKnowledgeByKnowledgeBaseID(context.Background(), 1, "kb-cnt")+ require.NoError(t, err)+ // failed rows stay counted (they are visible, actionable cards); only+ // the hidden mid-deletion row drops out.+ require.Equal(t, int64(2), count)+}diff --git a/internal/application/service/knowledge_housekeeping.go b/internal/application/service/knowledge_housekeeping.goindex 4b4b3a968..57b9cf7f2 100644--- a/internal/application/service/knowledge_housekeeping.go+++ b/internal/application/service/knowledge_housekeeping.go@@ -210,6 +210,60 @@ func (h *HousekeepingService) runSweep(ctx context.Context) {} else if resSummary.RowsAffected > 0 {logger.Infof(ctx, "[Housekeeping] recovered %d stuck summary rows", resSummary.RowsAffected)}++ // Sweep C: rows stuck in "deleting" whose delete task no longer exists+ // (issues #3338/#3345). The delete dead-letter callback only recovers+ // rows that are still "deleting" when retries exhaust; a worker death+ // mid-delete or a lost queue leaves the row hidden from the document+ // list forever. Same recovery contract as Sweep A: flip to failed with+ // a clear error so the row becomes visible and actionable (the user's+ // delete intent can then be retried; the delete executor is idempotent+ // and plans by row ID, so a late-arriving task still finishes cleanly).+ //+ // Liveness gate: a backlogged delete is not stranded — probe the queue+ // for a live knowledge:list_delete covering the row before touching it.+ // A nil inspector (nothing wired at all) defers every candidate to the+ // next sweep: we cannot tell backlog from orphan without it, and+ // wrongly failing a live delete is visible to users, while waiting one+ // more interval is not. A probe error defers that single row.+ if h.inspector != nil {+ var strandedDeletes []types.Knowledge+ if err := h.db.WithContext(ctx).+ Where("parse_status = ? AND updated_at < ?", types.ParseStatusDeleting, cutoff).+ Find(&strandedDeletes).Error; err != nil {+ logger.Warnf(ctx, "[Housekeeping] deleting candidate query failed: %v", err)+ } else {+ recoveredDeletes := int64(0)+ for _, k := range strandedDeletes {+ queued, err := h.inspector.HasQueuedDeleteTasksForKnowledge(ctx, k.ID)+ if err != nil {+ logger.Warnf(ctx,+ "[Housekeeping] delete-task probe failed for %s: %v (deferring to next sweep)",+ k.ID, err)+ continue+ }+ if queued {+ continue+ }+ res := h.db.WithContext(ctx).Model(&types.Knowledge{}).+ Where("id = ? AND parse_status = ?", k.ID, types.ParseStatusDeleting).+ Updates(map[string]interface{}{+ "parse_status": types.ParseStatusFailed,+ "error_message": "delete task stranded (no queued/active delete task) > " + threshold.String() + ", recovered by housekeeping",+ })+ if res.Error != nil {+ logger.Warnf(ctx, "[Housekeeping] delete sweep update failed for %s: %v", k.ID, res.Error)+ continue+ }+ recoveredDeletes += res.RowsAffected+ }+ if recoveredDeletes > 0 {+ logger.Infof(ctx,+ "[Housekeeping] recovered %d stranded deleting row(s) (threshold=%s)",+ recoveredDeletes, threshold)+ }+ }+ }}// filterByLastSpanActivity returns the subset of candidates whose mostdiff --git a/internal/application/service/knowledge_housekeeping_test.go b/internal/application/service/knowledge_housekeeping_test.goindex f0491c606..c0f1938a5 100644--- a/internal/application/service/knowledge_housekeeping_test.go+++ b/internal/application/service/knowledge_housekeeping_test.go@@ -133,7 +133,10 @@ func insertSpan(t *testing.T, db *gorm.DB, kid string, attempt int, spanID, stat// the probe to fail so the fail-safe branch can be exercised.type fakeTaskInspector struct {queued map[string]bool- err error+ // deleteQueued independently controls the delete-task liveness probe;+ // nil means "no delete task alive" for every ID.+ deleteQueued map[string]bool+ err error}func (f fakeTaskInspector) CancelTasksForKnowledge(@@ -151,6 +154,15 @@ func (f fakeTaskInspector) HasQueuedTasksForKnowledge(return f.queued[knowledgeID], nil}+func (f fakeTaskInspector) HasQueuedDeleteTasksForKnowledge(+ _ context.Context, knowledgeID string,+) (bool, error) {+ if f.err != nil {+ return false, f.err+ }+ return f.deleteQueued[knowledgeID], nil+}+func (f fakeTaskInspector) QueueStats(_ context.Context,) ([]types.QueueStat, bool, error) {@@ -386,3 +398,88 @@ func TestHousekeeping_PreservesRecentlyTouched(t *testing.T) {assert.Equal(t, types.ParseStatusProcessing, status,"knowledge updated within the cutoff must be left alone")}++// --- Sweep C: stranded "deleting" rows (issues #3338/#3345) --—++func readKnowledgeStatus(t *testing.T, db *gorm.DB, id string) (string, string) {+ t.Helper()+ var status, errMsg string+ require.NoError(t, db.Raw(+ `SELECT parse_status, COALESCE(error_message, '') FROM knowledges WHERE id = ?`, id,+ ).Row().Scan(&status, &errMsg))+ return status, errMsg+}++func TestHousekeeping_RecoversStrandedDeletingRow(t *testing.T) {+ db := setupHousekeepingDB(t)+ // No deleteQueued entry: the delete task is gone (worker death / lost+ // queue), the dead-letter path never fired, the row is stranded.+ svc := newHousekeepingSvcForTest(db)+ stale := time.Now().Add(-3 * time.Hour)+ insertKnowledge(t, db, "kid-stranded-delete", types.ParseStatusDeleting, stale)++ svc.runSweep(context.Background())++ status, errMsg := readKnowledgeStatus(t, db, "kid-stranded-delete")+ assert.Equal(t, types.ParseStatusFailed, status)+ assert.Contains(t, errMsg, "stranded")+}++func TestHousekeeping_KeepsBackloggedDeletingRow(t *testing.T) {+ db := setupHousekeepingDB(t)+ // A live (queued/active/retry) knowledge:list_delete still covers the+ // row — backpressure, not stranded; must stay untouched.+ svc := newHousekeepingSvcWithInspector(db, fakeTaskInspector{+ deleteQueued: map[string]bool{"kid-backlogged-delete": true},+ })+ insertKnowledge(t, db, "kid-backlogged-delete", types.ParseStatusDeleting,+ time.Now().Add(-3*time.Hour))++ svc.runSweep(context.Background())++ status, _ := readKnowledgeStatus(t, db, "kid-backlogged-delete")+ assert.Equal(t, types.ParseStatusDeleting, status)+}++func TestHousekeeping_FreshDeletingRowUntouched(t *testing.T) {+ db := setupHousekeepingDB(t)+ svc := newHousekeepingSvcForTest(db)+ // Marked deleting moments ago: below the staleness cutoff, so the sweep+ // must not even consider it (a delete just handed to the queue).+ insertKnowledge(t, db, "kid-fresh-delete", types.ParseStatusDeleting, time.Now())++ svc.runSweep(context.Background())++ status, _ := readKnowledgeStatus(t, db, "kid-fresh-delete")+ assert.Equal(t, types.ParseStatusDeleting, status)+}++func TestHousekeeping_DeletingProbeErrorDefers(t *testing.T) {+ db := setupHousekeepingDB(t)+ // Backend probe error must defer the row to the next sweep rather than+ // guess: failing a live delete would surface a wrong "failed" card.+ svc := newHousekeepingSvcWithInspector(db, fakeTaskInspector{+ err: assert.AnError,+ })+ insertKnowledge(t, db, "kid-probe-err-delete", types.ParseStatusDeleting,+ time.Now().Add(-3*time.Hour))++ svc.runSweep(context.Background())++ status, _ := readKnowledgeStatus(t, db, "kid-probe-err-delete")+ assert.Equal(t, types.ParseStatusDeleting, status)+}++func TestHousekeeping_NilInspectorDefersDeletingSweep(t *testing.T) {+ db := setupHousekeepingDB(t)+ // Without any inspector wired there is no way to tell backlog from+ // orphan, so the whole Sweep C defers — never guesses.+ svc := newHousekeepingSvcWithInspector(db, nil)+ insertKnowledge(t, db, "kid-nil-inspector", types.ParseStatusDeleting,+ time.Now().Add(-3*time.Hour))++ svc.runSweep(context.Background())++ status, _ := readKnowledgeStatus(t, db, "kid-nil-inspector")+ assert.Equal(t, types.ParseStatusDeleting, status)+}diff --git a/internal/application/service/knowledgebase_task_cancel_test.go b/internal/application/service/knowledgebase_task_cancel_test.goindex 250b86079..30d775362 100644--- a/internal/application/service/knowledgebase_task_cancel_test.go+++ b/internal/application/service/knowledgebase_task_cancel_test.go@@ -35,6 +35,10 @@ func (r *recordingKBTaskInspector) CancelTasksForKnowledge(return 0, 0, nil}+func (r *recordingKBTaskInspector) HasQueuedDeleteTasksForKnowledge(context.Context, string) (bool, error) {+ return false, nil+}+func (r *recordingKBTaskInspector) HasQueuedTasksForKnowledge(context.Context, string) (bool, error) {return false, nil}diff --git a/internal/handler/system_runtime_test.go b/internal/handler/system_runtime_test.goindex c7e3c3c83..3dfe7d5de 100644--- a/internal/handler/system_runtime_test.go+++ b/internal/handler/system_runtime_test.go@@ -59,6 +59,10 @@ type runtimeTestInspector struct{}func (runtimeTestInspector) CancelTasksForKnowledge(context.Context, string) (int, int, error) {return 0, 0, nil}+func (runtimeTestInspector) HasQueuedDeleteTasksForKnowledge(context.Context, string) (bool, error) {+ return false, nil+}+func (runtimeTestInspector) HasQueuedTasksForKnowledge(context.Context, string) (bool, error) {return false, nil}diff --git a/internal/router/task_inspector.go b/internal/router/task_inspector.goindex bdef9a307..b4404c5dc 100644--- a/internal/router/task_inspector.go+++ b/internal/router/task_inspector.go@@ -159,6 +159,28 @@ func (a *asynqTaskInspector) CancelTasksForKnowledgeBase(return deleted, cancelled, nil}+// matchesKnowledgeListDelete identifies a knowledge:list_delete task whose+// batch payload covers knowledgeID. Delete tasks carry knowledge_ids (plural)+// and are deliberately outside taskTypesForKnowledgeCancel, so they need+// their own matcher for the housekeeping delete sweep's liveness probe.+func matchesKnowledgeListDelete(taskType string, payload []byte, knowledgeID string) bool {+ if taskType != types.TypeKnowledgeListDelete {+ return false+ }+ var probe struct {+ KnowledgeIDs []string `json:"knowledge_ids"`+ }+ if err := json.Unmarshal(payload, &probe); err != nil {+ return false+ }+ for _, id := range probe.KnowledgeIDs {+ if id == knowledgeID {+ return true+ }+ }+ return false+}+// HasQueuedTasksForKnowledge reports whether any pending / scheduled /// retry / active task referencing knowledgeID still lives in the queue.// Read-only counterpart of CancelTasksForKnowledge — the housekeeping@@ -183,6 +205,29 @@ func (a *asynqTaskInspector) HasQueuedTasksForKnowledge(return false, nil}+// HasQueuedDeleteTasksForKnowledge is the delete-task counterpart of+// HasQueuedTasksForKnowledge: it matches knowledge:list_delete batch+// payloads that still cover the knowledge ID. The housekeeping delete+// sweep uses it to protect backlogged-but-alive deletes from recovery.+func (a *asynqTaskInspector) HasQueuedDeleteTasksForKnowledge(+ ctx context.Context, knowledgeID string,+) (bool, error) {+ if a == nil || a.inspector == nil || knowledgeID == "" {+ return false, nil+ }+ matcher := func(taskType string, payload []byte) bool {+ return matchesKnowledgeListDelete(taskType, payload, knowledgeID)+ }+ for _, queue := range queuesScanned {+ for _, state := range a.cancellableTaskStates() {+ if a.queueStateHasMatch(ctx, queue, state.name, state.list, matcher) {+ return true, nil+ }+ }+ }+ return false, nil+}+// QueueStats returns a depth snapshot for every queue this app enqueues// into. Read-only: it calls Inspector.GetQueueInfo per queue and maps// the result onto types.QueueStat, attaching static pool/weight metadata@@ -1081,6 +1126,15 @@ func (noopTaskInspector) HasQueuedTasksForKnowledge(return false, nil}+// HasQueuedDeleteTasksForKnowledge always reports false in Lite mode:+// deletes also run inline there, so a stale "deleting" row genuinely has+// no task left and the delete sweep may recover it.+func (noopTaskInspector) HasQueuedDeleteTasksForKnowledge(+ ctx context.Context, knowledgeID string,+) (bool, error) {+ return false, nil+}+// QueueStats reports "not supported" in Lite mode: there is no Redis /// asynq backend to inspect, so the runtime dashboard renders an// "unavailable in this deployment" state instead of an empty table.diff --git a/internal/router/task_inspector_errors_test.go b/internal/router/task_inspector_errors_test.goindex 2f93356aa..59ca9cc1a 100644--- a/internal/router/task_inspector_errors_test.go+++ b/internal/router/task_inspector_errors_test.go@@ -5,7 +5,9 @@ import ("fmt""testing"+ "github.com/Tencent/WeKnora/internal/types""github.com/hibiken/asynq"+ "github.com/stretchr/testify/require")func TestIsAsynqQueueNotFound(t *testing.T) {@@ -35,3 +37,26 @@ func TestIsAsynqQueueNotFound(t *testing.T) {})}}++func TestMatchesKnowledgeListDelete(t *testing.T) {+ const payload = `{"knowledge_base_id":"kb-1","tenant_id":7,"knowledge_ids":["kid-a","kid-b"]}`+ cases := []struct {+ name string+ taskType string+ payload string+ kid string+ want bool+ }{+ {"covered id", types.TypeKnowledgeListDelete, payload, "kid-a", true},+ {"last id", types.TypeKnowledgeListDelete, payload, "kid-b", true},+ {"uncovered id", types.TypeKnowledgeListDelete, payload, "kid-c", false},+ {"other task type", types.TypeDocumentProcess, payload, "kid-a", false},+ {"bad json", types.TypeKnowledgeListDelete, "{", "kid-a", false},+ {"empty list", types.TypeKnowledgeListDelete, `{"knowledge_ids":[]}`, "kid-a", false},+ }+ for _, tc := range cases {+ t.Run(tc.name, func(t *testing.T) {+ require.Equal(t, tc.want, matchesKnowledgeListDelete(tc.taskType, []byte(tc.payload), tc.kid))+ })+ }+}diff --git a/internal/types/interfaces/task_inspector.go b/internal/types/interfaces/task_inspector.goindex 593c7fe44..6badea746 100644--- a/internal/types/interfaces/task_inspector.go+++ b/internal/types/interfaces/task_inspector.go@@ -44,6 +44,18 @@ type TaskInspector interface {// span/updated_at checks remain authoritative there.HasQueuedTasksForKnowledge(ctx context.Context, knowledgeID string) (bool, error)+ // HasQueuedDeleteTasksForKnowledge reports whether any pending /+ // scheduled / retry / active knowledge:list_delete task still covers+ // the given knowledge ID. The delete sweep calls it before recovering+ // a stranded "deleting" row: the delete task is enqueued with a batch+ // payload (knowledge_ids list) that the per-parse matcher above cannot+ // see, so without this probe the sweep could not tell a stranded delete+ // from one that is merely backlogged behind a busy maintenance queue.+ //+ // Same fail-safe contract: (false, err) on backend error, callers defer;+ // Lite mode always returns false — inline deletes never queue.+ HasQueuedDeleteTasksForKnowledge(ctx context.Context, knowledgeID string) (bool, error)+// QueueStats returns a read-only depth snapshot for every queue this// application enqueues into, for the System Admin runtime dashboard.//