Open-source LLM knowledge platform: turn raw documents into a queryable RAG, an autonomous reasoning agent, and a self-maintaining Wiki.
Go agentagenticaichatbotdsh-pluginembeddingsevaluationgenerative-aigolangknowledge-basellmmulti-tenantollamaopenaiquestion-answeringragrerankingsemantic-searchvector-searchwiki
fix(kb-delete): give wiki cleanup its own longer timeout budget (#4292)
tsumon9ae4f3efcd39
diff --git a/internal/application/service/knowledgebase.go b/internal/application/service/knowledgebase.goindex ab4c1bbb7..a41b38ef6 100644--- a/internal/application/service/knowledgebase.go+++ b/internal/application/service/knowledgebase.go@@ -29,6 +29,21 @@ var ErrInvalidTenantID = errors.New("invalid tenant ID")const kbTaskCleanupTimeout = 5 * time.Second+// kbWikiCleanupTimeout bounds the wiki cleanup step (pages, folders,+// revisions, issues) in the ProcessKBDelete defer. It is deliberately much+// longer than kbTaskCleanupTimeout: the wiki DELETE statements themselves are+// fast once unblocked (idx_wiki_pages_kb_id covers them), but the KB's wiki+// workers may still be holding row locks at delete time. With a 5s budget the+// statements were killed while waiting on those locks and all asynq retries+// landed in the same lock window, leaving thousands of wiki rows orphaned+// after MaxRetry was exhausted (see issue #4288).+const kbWikiCleanupTimeout = 2 * time.Minute++// kbDeleteTaskMaxRetry is the asynq retry budget for the KB delete task,+// mirrored here so the defer can detect the final attempt and escalate+// logging when the task is about to land in the dead letter queue.+const kbDeleteTaskMaxRetry = 3+// knowledgeBaseService implements the knowledge base service interfacetype knowledgeBaseService struct {repo interfaces.KnowledgeBaseRepository@@ -1025,7 +1040,7 @@ func (s *knowledgeBaseService) DeleteKnowledgeBase(ctx context.Context, id strin}task := asynq.NewTask(types.TypeKBDelete, payloadBytes,- asynq.Queue(types.QueueMaintenance), asynq.MaxRetry(3), asynq.Timeout(2*time.Hour))+ asynq.Queue(types.QueueMaintenance), asynq.MaxRetry(kbDeleteTaskMaxRetry), asynq.Timeout(2*time.Hour))info, err := s.asynqClient.Enqueue(task)if err != nil {logger.Warnf(ctx, "Failed to enqueue KB delete task: %v", err)@@ -1057,11 +1072,27 @@ func (s *knowledgeBaseService) ProcessKBDelete(ctx context.Context, t *asynq.Tas// Wiki rows are independent of the vector engine. Run this on every// return path — including SkipRetry — with its own timeout so a slow// Redis queue scan cannot starve the SQL deletes.- wikiCtx, cancelWiki := context.WithTimeout(context.WithoutCancel(ctx), kbTaskCleanupTimeout)+ //+ // The budget is kbWikiCleanupTimeout (not the shared 5s+ // kbTaskCleanupTimeout): the wiki DELETEs are fast once unblocked,+ // but in-flight wiki workers may hold row locks at delete time and a+ // 5s budget was being exhausted while waiting on them, orphaning+ // thousands of wiki rows after retries ran out (issue #4288).+ wikiCtx, cancelWiki := context.WithTimeout(context.WithoutCancel(ctx), kbWikiCleanupTimeout)wikiErr := s.cleanupWikiForKnowledgeBase(wikiCtx, tenantID, kbID)cancelWiki()if wikiErr != nil {- logger.Warnf(ctx, "Failed to clean wiki data for KB %s: %v", kbID, wikiErr)+ if retryCount, _ := asynq.GetRetryCount(ctx); retryCount >= kbDeleteTaskMaxRetry {+ // Retries are exhausted; the returned error sends this task+ // to the dead letter queue. Log at ERROR so the residual wiki+ // rows are visible to operators instead of silently orphaned.+ logger.Errorf(ctx,+ "Wiki cleanup for KB %s failed on final retry (attempt %d): %v. "++ "Wiki pages/folders/revisions/issues may be orphaned — manual cleanup required.",+ kbID, retryCount+1, wikiErr)+ } else {+ logger.Warnf(ctx, "Failed to clean wiki data for KB %s: %v", kbID, wikiErr)+ }// SkipRetry would otherwise drop the task while wiki orphans// remain. Promote the wiki error so asynq retries; a later// attempt that succeeds at wiki cleanup can still SkipRetry.diff --git a/internal/application/service/knowledgebase_wiki_cleanup_test.go b/internal/application/service/knowledgebase_wiki_cleanup_test.goindex 75c4636e2..466b6afbc 100644--- a/internal/application/service/knowledgebase_wiki_cleanup_test.go+++ b/internal/application/service/knowledgebase_wiki_cleanup_test.go@@ -4,7 +4,9 @@ import ("context""encoding/json""errors"+ "sync""testing"+ "time""github.com/Tencent/WeKnora/internal/application/service/retriever""github.com/Tencent/WeKnora/internal/types"@@ -254,3 +256,119 @@ func TestProcessKBDeleteCleansWikiWhenVectorStoreUnavailable(t *testing.T) {require.ErrorIs(t, err, retriever.ErrVectorStoreUnavailable)assertWikiCleanup(t, wikiRepo, tenantID, kbID)}++// blockingWikiRepo simulates the issue #4288 scenario: in-flight wiki workers+// still hold row locks when the KB is deleted, so each delete blocks until the+// simulated lock wait elapses or the caller's context deadline fires.+type blockingWikiRepo struct {+ interfaces.WikiPageRepository+ blockFor time.Duration+}++func (r *blockingWikiRepo) wait(ctx context.Context) error {+ timer := time.NewTimer(r.blockFor)+ defer timer.Stop()+ select {+ case <-ctx.Done():+ return ctx.Err()+ case <-timer.C:+ return nil+ }+}++func (r *blockingWikiRepo) DeleteByKnowledgeBaseID(ctx context.Context, _ uint64, _ string) error {+ return r.wait(ctx)+}++func (r *blockingWikiRepo) DeleteFoldersByKnowledgeBaseID(ctx context.Context, _ uint64, _ string) error {+ return r.wait(ctx)+}++func (r *blockingWikiRepo) DeleteRevisionsByKnowledgeBaseID(ctx context.Context, _ uint64, _ string) error {+ return r.wait(ctx)+}++func (r *blockingWikiRepo) DeleteIssuesByKnowledgeBaseID(ctx context.Context, _ uint64, _ string) error {+ return r.wait(ctx)+}++// TestProcessKBDeleteWikiCleanupSurvivesLockContention reproduces issue #4288:+// when wiki deletes stall on row locks held by in-flight wiki workers, the+// old 5s budget killed every attempt (deadline exceeded -> wiki orphans after+// MaxRetry). With the dedicated longer budget a 6s lock wait must succeed.+func TestProcessKBDeleteWikiCleanupSurvivesLockContention(t *testing.T) {+ const kbID = "kb-lock-contention"+ const tenantID uint64 = 1+ wikiRepo := &blockingWikiRepo{blockFor: 6 * time.Second}+ svc := &knowledgeBaseService{+ kgRepo: emptyKBKnowledgeRepo{},+ taskInspector: &recordingKBTaskInspector{},+ taskPendingRepo: &recordingKBPendingRepo{},+ wikiRepo: wikiRepo,+ }++ err := svc.ProcessKBDelete(context.Background(), kbDeletePayload(t, kbID, tenantID))++ require.NoError(t, err, "wiki cleanup must survive a lock wait longer than the old 5s budget")+}++// deadlineCapturingWikiRepo records the context deadline ProcessKBDelete+// hands to the wiki deletes, so the test can verify the dedicated budget.+type deadlineCapturingWikiRepo struct {+ interfaces.WikiPageRepository+ mu sync.Mutex+ deadlines []time.Time+}++func (r *deadlineCapturingWikiRepo) record(ctx context.Context) {+ r.mu.Lock()+ defer r.mu.Unlock()+ if dl, ok := ctx.Deadline(); ok {+ r.deadlines = append(r.deadlines, dl)+ }+}++func (r *deadlineCapturingWikiRepo) DeleteByKnowledgeBaseID(ctx context.Context, _ uint64, _ string) error {+ r.record(ctx)+ return nil+}++func (r *deadlineCapturingWikiRepo) DeleteFoldersByKnowledgeBaseID(ctx context.Context, _ uint64, _ string) error {+ r.record(ctx)+ return nil+}++func (r *deadlineCapturingWikiRepo) DeleteRevisionsByKnowledgeBaseID(ctx context.Context, _ uint64, _ string) error {+ r.record(ctx)+ return nil+}++func (r *deadlineCapturingWikiRepo) DeleteIssuesByKnowledgeBaseID(ctx context.Context, _ uint64, _ string) error {+ r.record(ctx)+ return nil+}++// TestProcessKBDeleteWikiCleanupHasDedicatedBudget verifies the wiki cleanup+// runs under kbWikiCleanupTimeout, not the shared 5s kbTaskCleanupTimeout.+// The remaining budget must be comfortably beyond one minute for every table.+func TestProcessKBDeleteWikiCleanupHasDedicatedBudget(t *testing.T) {+ const kbID = "kb-budget"+ const tenantID uint64 = 1+ wikiRepo := &deadlineCapturingWikiRepo{}+ svc := &knowledgeBaseService{+ kgRepo: emptyKBKnowledgeRepo{},+ taskInspector: &recordingKBTaskInspector{},+ taskPendingRepo: &recordingKBPendingRepo{},+ wikiRepo: wikiRepo,+ }++ require.NoError(t, svc.ProcessKBDelete(context.Background(), kbDeletePayload(t, kbID, tenantID)))++ require.Len(t, wikiRepo.deadlines, 4, "all four wiki deletes must run under the dedicated budget")+ for _, dl := range wikiRepo.deadlines {+ remaining := time.Until(dl)+ assert.Greater(t, remaining, time.Minute,+ "wiki cleanup budget must exceed 1 minute (old shared budget was 5s), got %v", remaining)+ assert.LessOrEqual(t, remaining, kbWikiCleanupTimeout)+ }+}