Skip to content

Open-source LLM knowledge platform: turn raw documents into a queryable RAG, an autonomous reasoning agent, and a self-maintaining Wiki.

fix(kb-delete): give wiki cleanup its own longer timeout budget (#4292)

tsumon9ae4f3efcd39
diff --git a/internal/application/service/knowledgebase.go b/internal/application/service/knowledgebase.go
index ab4c1bbb7..a41b38ef6 100644
--- a/internal/application/service/knowledgebase.go
+++ b/internal/application/service/knowledgebase.go
@@ -29,6 +29,21 @@ var ErrInvalidTenantID = errors.New("invalid tenant ID")
const kbTaskCleanupTimeout = 5 * time.Second
+// kbWikiCleanupTimeout bounds the wiki cleanup step (pages, folders,
+// revisions, issues) in the ProcessKBDelete defer. It is deliberately much
+// longer than kbTaskCleanupTimeout: the wiki DELETE statements themselves are
+// fast once unblocked (idx_wiki_pages_kb_id covers them), but the KB's wiki
+// workers may still be holding row locks at delete time. With a 5s budget the
+// statements were killed while waiting on those locks and all asynq retries
+// landed in the same lock window, leaving thousands of wiki rows orphaned
+// after MaxRetry was exhausted (see issue #4288).
+const kbWikiCleanupTimeout = 2 * time.Minute
+
+// kbDeleteTaskMaxRetry is the asynq retry budget for the KB delete task,
+// mirrored here so the defer can detect the final attempt and escalate
+// logging when the task is about to land in the dead letter queue.
+const kbDeleteTaskMaxRetry = 3
+
// knowledgeBaseService implements the knowledge base service interface
type knowledgeBaseService struct {
repo interfaces.KnowledgeBaseRepository
@@ -1025,7 +1040,7 @@ func (s *knowledgeBaseService) DeleteKnowledgeBase(ctx context.Context, id strin
}
task := asynq.NewTask(types.TypeKBDelete, payloadBytes,
- asynq.Queue(types.QueueMaintenance), asynq.MaxRetry(3), asynq.Timeout(2*time.Hour))
+ asynq.Queue(types.QueueMaintenance), asynq.MaxRetry(kbDeleteTaskMaxRetry), asynq.Timeout(2*time.Hour))
info, err := s.asynqClient.Enqueue(task)
if err != nil {
logger.Warnf(ctx, "Failed to enqueue KB delete task: %v", err)
@@ -1057,11 +1072,27 @@ func (s *knowledgeBaseService) ProcessKBDelete(ctx context.Context, t *asynq.Tas
// Wiki rows are independent of the vector engine. Run this on every
// return path — including SkipRetry — with its own timeout so a slow
// Redis queue scan cannot starve the SQL deletes.
- wikiCtx, cancelWiki := context.WithTimeout(context.WithoutCancel(ctx), kbTaskCleanupTimeout)
+ //
+ // The budget is kbWikiCleanupTimeout (not the shared 5s
+ // kbTaskCleanupTimeout): the wiki DELETEs are fast once unblocked,
+ // but in-flight wiki workers may hold row locks at delete time and a
+ // 5s budget was being exhausted while waiting on them, orphaning
+ // thousands of wiki rows after retries ran out (issue #4288).
+ wikiCtx, cancelWiki := context.WithTimeout(context.WithoutCancel(ctx), kbWikiCleanupTimeout)
wikiErr := s.cleanupWikiForKnowledgeBase(wikiCtx, tenantID, kbID)
cancelWiki()
if wikiErr != nil {
- logger.Warnf(ctx, "Failed to clean wiki data for KB %s: %v", kbID, wikiErr)
+ if retryCount, _ := asynq.GetRetryCount(ctx); retryCount >= kbDeleteTaskMaxRetry {
+ // Retries are exhausted; the returned error sends this task
+ // to the dead letter queue. Log at ERROR so the residual wiki
+ // rows are visible to operators instead of silently orphaned.
+ logger.Errorf(ctx,
+ "Wiki cleanup for KB %s failed on final retry (attempt %d): %v. "+
+ "Wiki pages/folders/revisions/issues may be orphaned — manual cleanup required.",
+ kbID, retryCount+1, wikiErr)
+ } else {
+ logger.Warnf(ctx, "Failed to clean wiki data for KB %s: %v", kbID, wikiErr)
+ }
// SkipRetry would otherwise drop the task while wiki orphans
// remain. Promote the wiki error so asynq retries; a later
// attempt that succeeds at wiki cleanup can still SkipRetry.
diff --git a/internal/application/service/knowledgebase_wiki_cleanup_test.go b/internal/application/service/knowledgebase_wiki_cleanup_test.go
index 75c4636e2..466b6afbc 100644
--- a/internal/application/service/knowledgebase_wiki_cleanup_test.go
+++ b/internal/application/service/knowledgebase_wiki_cleanup_test.go
@@ -4,7 +4,9 @@ import (
"context"
"encoding/json"
"errors"
+ "sync"
"testing"
+ "time"
"github.com/Tencent/WeKnora/internal/application/service/retriever"
"github.com/Tencent/WeKnora/internal/types"
@@ -254,3 +256,119 @@ func TestProcessKBDeleteCleansWikiWhenVectorStoreUnavailable(t *testing.T) {
require.ErrorIs(t, err, retriever.ErrVectorStoreUnavailable)
assertWikiCleanup(t, wikiRepo, tenantID, kbID)
}
+
+// blockingWikiRepo simulates the issue #4288 scenario: in-flight wiki workers
+// still hold row locks when the KB is deleted, so each delete blocks until the
+// simulated lock wait elapses or the caller's context deadline fires.
+type blockingWikiRepo struct {
+ interfaces.WikiPageRepository
+ blockFor time.Duration
+}
+
+func (r *blockingWikiRepo) wait(ctx context.Context) error {
+ timer := time.NewTimer(r.blockFor)
+ defer timer.Stop()
+ select {
+ case <-ctx.Done():
+ return ctx.Err()
+ case <-timer.C:
+ return nil
+ }
+}
+
+func (r *blockingWikiRepo) DeleteByKnowledgeBaseID(ctx context.Context, _ uint64, _ string) error {
+ return r.wait(ctx)
+}
+
+func (r *blockingWikiRepo) DeleteFoldersByKnowledgeBaseID(ctx context.Context, _ uint64, _ string) error {
+ return r.wait(ctx)
+}
+
+func (r *blockingWikiRepo) DeleteRevisionsByKnowledgeBaseID(ctx context.Context, _ uint64, _ string) error {
+ return r.wait(ctx)
+}
+
+func (r *blockingWikiRepo) DeleteIssuesByKnowledgeBaseID(ctx context.Context, _ uint64, _ string) error {
+ return r.wait(ctx)
+}
+
+// TestProcessKBDeleteWikiCleanupSurvivesLockContention reproduces issue #4288:
+// when wiki deletes stall on row locks held by in-flight wiki workers, the
+// old 5s budget killed every attempt (deadline exceeded -> wiki orphans after
+// MaxRetry). With the dedicated longer budget a 6s lock wait must succeed.
+func TestProcessKBDeleteWikiCleanupSurvivesLockContention(t *testing.T) {
+ const kbID = "kb-lock-contention"
+ const tenantID uint64 = 1
+ wikiRepo := &blockingWikiRepo{blockFor: 6 * time.Second}
+ svc := &knowledgeBaseService{
+ kgRepo: emptyKBKnowledgeRepo{},
+ taskInspector: &recordingKBTaskInspector{},
+ taskPendingRepo: &recordingKBPendingRepo{},
+ wikiRepo: wikiRepo,
+ }
+
+ err := svc.ProcessKBDelete(context.Background(), kbDeletePayload(t, kbID, tenantID))
+
+ require.NoError(t, err, "wiki cleanup must survive a lock wait longer than the old 5s budget")
+}
+
+// deadlineCapturingWikiRepo records the context deadline ProcessKBDelete
+// hands to the wiki deletes, so the test can verify the dedicated budget.
+type deadlineCapturingWikiRepo struct {
+ interfaces.WikiPageRepository
+ mu sync.Mutex
+ deadlines []time.Time
+}
+
+func (r *deadlineCapturingWikiRepo) record(ctx context.Context) {
+ r.mu.Lock()
+ defer r.mu.Unlock()
+ if dl, ok := ctx.Deadline(); ok {
+ r.deadlines = append(r.deadlines, dl)
+ }
+}
+
+func (r *deadlineCapturingWikiRepo) DeleteByKnowledgeBaseID(ctx context.Context, _ uint64, _ string) error {
+ r.record(ctx)
+ return nil
+}
+
+func (r *deadlineCapturingWikiRepo) DeleteFoldersByKnowledgeBaseID(ctx context.Context, _ uint64, _ string) error {
+ r.record(ctx)
+ return nil
+}
+
+func (r *deadlineCapturingWikiRepo) DeleteRevisionsByKnowledgeBaseID(ctx context.Context, _ uint64, _ string) error {
+ r.record(ctx)
+ return nil
+}
+
+func (r *deadlineCapturingWikiRepo) DeleteIssuesByKnowledgeBaseID(ctx context.Context, _ uint64, _ string) error {
+ r.record(ctx)
+ return nil
+}
+
+// TestProcessKBDeleteWikiCleanupHasDedicatedBudget verifies the wiki cleanup
+// runs under kbWikiCleanupTimeout, not the shared 5s kbTaskCleanupTimeout.
+// The remaining budget must be comfortably beyond one minute for every table.
+func TestProcessKBDeleteWikiCleanupHasDedicatedBudget(t *testing.T) {
+ const kbID = "kb-budget"
+ const tenantID uint64 = 1
+ wikiRepo := &deadlineCapturingWikiRepo{}
+ svc := &knowledgeBaseService{
+ kgRepo: emptyKBKnowledgeRepo{},
+ taskInspector: &recordingKBTaskInspector{},
+ taskPendingRepo: &recordingKBPendingRepo{},
+ wikiRepo: wikiRepo,
+ }
+
+ require.NoError(t, svc.ProcessKBDelete(context.Background(), kbDeletePayload(t, kbID, tenantID)))
+
+ require.Len(t, wikiRepo.deadlines, 4, "all four wiki deletes must run under the dedicated budget")
+ for _, dl := range wikiRepo.deadlines {
+ remaining := time.Until(dl)
+ assert.Greater(t, remaining, time.Minute,
+ "wiki cleanup budget must exceed 1 minute (old shared budget was 5s), got %v", remaining)
+ assert.LessOrEqual(t, remaining, kbWikiCleanupTimeout)
+ }
+}