Open-source LLM knowledge platform: turn raw documents into a queryable RAG, an autonomous reasoning agent, and a self-maintaining Wiki.
Go agentagenticaichatbotdsh-pluginembeddingsevaluationgenerative-aigolangknowledge-basellmmulti-tenantollamaopenaiquestion-answeringragrerankingsemantic-searchvector-searchwiki
fix(tenant): stop wiki tasks of soft-deleted tenants from issuing model requests (#3602)
LuckVdc57766cc7606
diff --git a/internal/application/repository/task_queue.go b/internal/application/repository/task_queue.goindex 1823732cd..f35b72128 100644--- a/internal/application/repository/task_queue.go+++ b/internal/application/repository/task_queue.go@@ -53,11 +53,13 @@ func preparePendingOp(op *types.TaskPendingOp) error {}// EnqueueIfKnowledgeBaseActive prevents detached wiki cleanup from writing new-// durable work after a KB was soft-deleted. On Postgres the share lock-// serializes this check+insert transaction against the row update performed by-// soft deletion: whichever operation acquires the row first determines the-// order, and the deletion path's subsequent scope scrub removes any insert-// that committed before it.+// durable work after a KB was soft-deleted. The tenant must also be alive: a+// tenant soft-deletion removes the workspace without touching its knowledge+// bases, and a deleted tenant must never accrue new model-backed work. On+// Postgres the share lock serializes this check+insert transaction against+// the row update performed by soft deletion: whichever operation acquires the+// row first determines the order, and the deletion path's subsequent scope+// scrub removes any insert that committed before it.func (r *taskPendingOpsRepository) EnqueueIfKnowledgeBaseActive(ctx context.Context,op *types.TaskPendingOp,@@ -84,6 +86,10 @@ func (r *taskPendingOpsRepository) EnqueueIfKnowledgeBaseActive(}return err}+ active, err := tenantActiveWithinTx(tx, op.TenantID)+ if err != nil || !active {+ return err+ }if err := tx.Create(op).Error; err != nil {return err}@@ -93,6 +99,31 @@ func (r *taskPendingOpsRepository) EnqueueIfKnowledgeBaseActive(return accepted, err}+// tenantActiveWithinTx reports whether the tenant row exists and is not+// soft-deleted, inside the caller's transaction. Callers that cannot see a+// tenants table (legacy test doubles) fail closed.+func tenantActiveWithinTx(tx *gorm.DB, tenantID uint64) (bool, error) {+ if tenantID == 0 {+ return false, nil+ }+ var tenant types.Tenant+ err := tx.Model(&types.Tenant{}).Select("id").Where("id = ?", tenantID).Take(&tenant).Error+ if errors.Is(err, gorm.ErrRecordNotFound) {+ return false, nil+ }+ if err != nil {+ return false, err+ }+ return true, nil+}++// HasActiveTenant reports whether the tenant exists and has not been+// soft-deleted. Wiki task consumers call this before doing durable or+// model-backed work on the tenant's behalf (#3593).+func (r *taskPendingOpsRepository) HasActiveTenant(ctx context.Context, tenantID uint64) (bool, error) {+ return tenantActiveWithinTx(r.db.WithContext(ctx), tenantID)+}+// SeedKnowledgeFinalizingWithPendingOp commits the finalizing counter and the// durable operation that owns one slot in the same transaction. This closes// the crash window where a knowledge row could enter finalizing before itsdiff --git a/internal/application/repository/task_queue_test.go b/internal/application/repository/task_queue_test.goindex 87d9bf18f..dc8fe547a 100644--- a/internal/application/repository/task_queue_test.go+++ b/internal/application/repository/task_queue_test.go@@ -392,15 +392,32 @@ func TestTaskPendingOps_EnqueueIfKnowledgeBaseActive(t *testing.T) {deleted_at DATETIME)`).Error)require.NoError(t, db.Exec(- "INSERT INTO knowledge_bases (id, tenant_id, deleted_at) VALUES (?, ?, NULL), (?, ?, ?)",- "kb-active", 1, "kb-deleted", 1, time.Now(),+ "INSERT INTO knowledge_bases (id, tenant_id, deleted_at) VALUES (?, ?, NULL), (?, ?, ?), (?, ?, NULL)",+ "kb-active", 1, "kb-deleted", 1, time.Now(), "kb-t2", 2,+ ).Error)+ require.NoError(t, db.Exec(`CREATE TABLE tenants (+ id INTEGER PRIMARY KEY,+ deleted_at DATETIME+ )`).Error)+ require.NoError(t, db.Exec(+ "INSERT INTO tenants (id, deleted_at) VALUES (?, NULL), (?, ?)",+ 1, 2, time.Now(),).Error)repo := NewTaskPendingOpsRepository(db)guard, ok := repo.(interfaces.TaskPendingOpsKnowledgeBaseGuard)require.True(t, ok)+ liveness, ok := repo.(interfaces.TaskPendingOpsTenantLiveness)+ require.True(t, ok, "task pending repository must expose tenant liveness for wiki task guards")ctx := context.Background()+ activeTenant, err := liveness.HasActiveTenant(ctx, 1)+ require.NoError(t, err)+ assert.True(t, activeTenant)+ deletedTenant, err := liveness.HasActiveTenant(ctx, 2)+ require.NoError(t, err)+ assert.False(t, deletedTenant)+accepted, err := guard.EnqueueIfKnowledgeBaseActive(ctx,makePendingOp(types.TypeWikiIngest, types.TaskScopeKnowledgeBase, "kb-active", "ingest", "active", nil))require.NoError(t, err)@@ -426,6 +443,10 @@ func TestTaskPendingOps_EnqueueIfKnowledgeBaseActive(t *testing.T) {TenantID: 2, TaskType: types.TypeWikiIngest, Scope: types.TaskScopeKnowledgeBase,ScopeID: "kb-active", Op: "ingest", DedupKey: "wrong-tenant",}},+ {name: "deleted tenant with live KB", op: &types.TaskPendingOp{+ TenantID: 2, TaskType: types.TypeWikiIngest, Scope: types.TaskScopeKnowledgeBase,+ ScopeID: "kb-t2", Op: "ingest", DedupKey: "deleted-tenant",+ }},} {t.Run(tc.name, func(t *testing.T) {accepted, err := guard.EnqueueIfKnowledgeBaseActive(ctx, tc.op)diff --git a/internal/application/service/wiki_deleted_tenant_guard_test.go b/internal/application/service/wiki_deleted_tenant_guard_test.gonew file mode 100644index 000000000..d84c69443--- /dev/null+++ b/internal/application/service/wiki_deleted_tenant_guard_test.go@@ -0,0 +1,93 @@+package service++import (+ "context"+ "encoding/json"+ "testing"++ "github.com/Tencent/WeKnora/internal/types"+ "github.com/Tencent/WeKnora/internal/types/interfaces"+ "github.com/hibiken/asynq"+ "github.com/stretchr/testify/assert"+ "github.com/stretchr/testify/require"+)++// wikiTenantGuardPendingRepo stubs the slice of TaskPendingOpsRepository the+// tenant-liveness guard touches: the liveness lookup and the scope cleanup.+type wikiTenantGuardPendingRepo struct {+ interfaces.TaskPendingOpsRepository+ active bool+ lookupErr error+ lookups []uint64+ deletedKBs []string+}++func (r *wikiTenantGuardPendingRepo) HasActiveTenant(_ context.Context, tenantID uint64) (bool, error) {+ r.lookups = append(r.lookups, tenantID)+ return r.active, r.lookupErr+}++func (r *wikiTenantGuardPendingRepo) DeleteByScope(_ context.Context, scope, scopeID string) error {+ if scope == types.TaskScopeKnowledgeBase {+ r.deletedKBs = append(r.deletedKBs, scopeID)+ }+ return nil+}++func TestWikiTaskSkipsDeletedTenant(t *testing.T) {+ payload, err := json.Marshal(WikiIngestPayload{+ TenantID: 7,+ KnowledgeBaseID: "kb-orphaned",+ Language: "en",+ })+ require.NoError(t, err)++ t.Run("ingest clears queue and does not proceed", func(t *testing.T) {+ repo := &wikiTenantGuardPendingRepo{active: false}+ svc := &wikiIngestService{pendingRepo: repo}++ err := svc.ProcessWikiIngest(context.Background(), asynq.NewTask(types.TypeWikiIngest, payload))+ require.NoError(t, err)+ require.Len(t, repo.lookups, 1)+ assert.Equal(t, uint64(7), repo.lookups[0])+ assert.Equal(t, []string{"kb-orphaned"}, repo.deletedKBs)+ })++ t.Run("finalize clears queue and does not proceed", func(t *testing.T) {+ repo := &wikiTenantGuardPendingRepo{active: false}+ svc := &wikiIngestService{pendingRepo: repo}++ err := svc.ProcessWikiFinalize(context.Background(), asynq.NewTask(types.TypeWikiFinalize, payload))+ require.NoError(t, err)+ require.Len(t, repo.lookups, 1)+ assert.Equal(t, []string{"kb-orphaned"}, repo.deletedKBs)+ })+}++func TestWikiTenantGuardFailsOpenOnLookupError(t *testing.T) {+ repo := &wikiTenantGuardPendingRepo{active: false, lookupErr: assert.AnError}+ svc := &wikiIngestService{pendingRepo: repo}++ // A transient liveness-lookup failure must not be read as "deleted":+ // the task keeps its retry budget and the enqueue guard / startup+ // recovery enforce the same invariant on their own DB access.+ assert.False(t, svc.tenantIsDeleted(context.Background(), 7))+ assert.Empty(t, repo.deletedKBs)+}++func TestWikiTenantGuardWithoutLivenessCapability(t *testing.T) {+ // A pending repo without the liveness extension (legacy test doubles,+ // alternate implementations) is never treated as a deleted tenant.+ svc := &wikiIngestService{pendingRepo: &wikiKBGuardPendingRepo{}}++ assert.False(t, svc.tenantIsDeleted(context.Background(), 7))+}++func TestWikiTenantGuardIgnoresZeroTenant(t *testing.T) {+ repo := &wikiTenantGuardPendingRepo{active: false}+ svc := &wikiIngestService{pendingRepo: repo}++ assert.False(t, svc.tenantIsDeleted(context.Background(), 0))+ assert.Empty(t, repo.lookups)+ _ = interfaces.TaskPendingOpsTenantLiveness(nil)+}diff --git a/internal/application/service/wiki_ingest.go b/internal/application/service/wiki_ingest.goindex da081a5ae..097d85f8a 100644--- a/internal/application/service/wiki_ingest.go+++ b/internal/application/service/wiki_ingest.go@@ -720,6 +720,31 @@ func (s *wikiIngestService) clearDeletedKnowledgeBasePendingOps(ctx context.Contreturn cleaner.DeleteByScope(cleanupCtx, types.TaskScopeKnowledgeBase, kbID)}+// tenantIsDeleted reports whether the payload's tenant has been soft-deleted.+// A tenant deletion removes the workspace but leaves its knowledge bases and+// durable pending ops in place, so wiki tasks restored from those ops would+// otherwise keep issuing model requests for a tenant nobody can reach (#3593).+//+// Fail-open on transient lookup errors and when the pending repo does not+// expose tenant liveness (legacy test doubles): the retry machinery still+// covers the task, and the guarded enqueue / startup recovery paths enforce+// the same invariant on their own DB access.+func (s *wikiIngestService) tenantIsDeleted(ctx context.Context, tenantID uint64) bool {+ if tenantID == 0 {+ return false+ }+ checker, ok := s.pendingRepo.(interfaces.TaskPendingOpsTenantLiveness)+ if !ok {+ return false+ }+ active, err := checker.HasActiveTenant(ctx, tenantID)+ if err != nil {+ logger.Warnf(ctx, "wiki: tenant liveness lookup failed for tenant %d: %v (failing open)", tenantID, err)+ return false+ }+ return !active+}+// releaseIngestForUnavailableWiki drops the KB's queued ingest ops when the// wiki cannot run for a reason retries will not fix, and releases each// document's wiki slot in the same transaction so it leaves "finalizing".diff --git a/internal/application/service/wiki_ingest_batch.go b/internal/application/service/wiki_ingest_batch.goindex d2d79414b..f1d92ad5f 100644--- a/internal/application/service/wiki_ingest_batch.go+++ b/internal/application/service/wiki_ingest_batch.go@@ -254,6 +254,18 @@ func (s *wikiIngestService) ProcessWikiIngest(ctx context.Context, t *asynq.Taskctx = context.WithValue(ctx, types.LanguageContextKey, payload.Language)}+ // A soft-deleted tenant owns no reachable workspace anymore; its KBs and+ // durable pending ops survive the deletion, so without this guard the+ // batch would keep issuing model requests (and finalize would rebuild+ // index pages) for a tenant nobody can see (#3593). Drop the KB's queue.+ if s.tenantIsDeleted(ctx, payload.TenantID) {+ exitStatus = "tenant_deleted"+ if err := s.clearDeletedKnowledgeBasePendingOps(ctx, payload.KnowledgeBaseID); err != nil {+ return fmt.Errorf("wiki ingest: clear deleted tenant queue: %w", err)+ }+ return nil+ }+// Concurrency model (Phase 3)://// - Standard (Redis) mode: NO exclusive per-KB lock. Multiple batches@@ -931,6 +943,16 @@ func (s *wikiIngestService) ProcessWikiFinalize(ctx context.Context, t *asynq.Tareturn nil}+ // Same tenant-liveness guard as the ingest batch: finalize calls the+ // synthesis model to rebuild the index page, and a deleted tenant must+ // never accrue new model requests (#3593). Drop the KB's queue.+ if s.tenantIsDeleted(ctx, payload.TenantID) {+ if err := s.clearDeletedKnowledgeBasePendingOps(ctx, payload.KnowledgeBaseID); err != nil {+ return fmt.Errorf("wiki finalize: clear deleted tenant queue: %w", err)+ }+ return nil+ }+// Per-KB finalize lock, separate from the ingest active lock so finalize// and ingest batches never block each other. Coalescing via asynq.TaskID// already keeps at most one finalize pending per KB; this guards thediff --git a/internal/container/recover_pending_wiki_tasks.go b/internal/container/recover_pending_wiki_tasks.goindex 1867e055e..adf242020 100644--- a/internal/container/recover_pending_wiki_tasks.go+++ b/internal/container/recover_pending_wiki_tasks.go@@ -34,16 +34,24 @@ func recoverPendingWikiTasks(db *gorm.DB, task interfaces.TaskEnqueuer) {return}ctx := context.Background()- const activeKnowledgeBase = `EXISTS (+ // The outer parentheses matter: the cleanup below negates this whole+ // predicate with NOT, and `NOT EXISTS(...) AND EXISTS(...)` would parse+ // as `(NOT EXISTS(...)) AND EXISTS(...)` — silently matching nothing.+ const activeKnowledgeBase = `(EXISTS (SELECT 1 FROM knowledge_bases kbWHERE kb.id = task_pending_ops.scope_idAND kb.tenant_id = task_pending_ops.tenant_idAND kb.deleted_at IS NULL- )`+ ) AND EXISTS (+ SELECT 1 FROM tenants t+ WHERE t.id = task_pending_ops.tenant_id+ AND t.deleted_at IS NULL+ ))`wikiTaskTypes := []string{types.TypeWikiIngest, types.TypeWikiFinalize}- // Durable rows for a deleted/missing KB must not recreate ephemeral- // triggers at startup. Fail closed if this cleanup cannot be verified.+ // Durable rows for a deleted/missing KB — or a soft-deleted tenant —+ // must not recreate ephemeral triggers at startup. Fail closed if this+ // cleanup cannot be verified.cleanup := db.WithContext(ctx).Where("scope = ? AND task_type IN ?", types.TaskScopeKnowledgeBase, wikiTaskTypes).Where("NOT " + activeKnowledgeBase).@@ -53,7 +61,9 @@ func recoverPendingWikiTasks(db *gorm.DB, task interfaces.TaskEnqueuer) {return}if cleanup.RowsAffected > 0 {- logger.Infof(ctx, "[WikiRecovery] removed %d pending row(s) for deleted knowledge bases", cleanup.RowsAffected)+ logger.Infof(ctx,+ "[WikiRecovery] removed %d pending row(s) for deleted knowledge bases or tenants",+ cleanup.RowsAffected)}var scopes []pendingWikiScopediff --git a/internal/container/reset_pending_tasks_test.go b/internal/container/reset_pending_tasks_test.goindex 47d4c0248..4df02aa0b 100644--- a/internal/container/reset_pending_tasks_test.go+++ b/internal/container/reset_pending_tasks_test.go@@ -89,6 +89,13 @@ CREATE TABLE IF NOT EXISTS knowledge_bases ();`+var resetPendingTenantsDDL = `+CREATE TABLE IF NOT EXISTS tenants (+ id INTEGER PRIMARY KEY,+ deleted_at DATETIME+);+`+func setupResetPendingDB(t *testing.T) *gorm.DB {t.Helper()db, err := gorm.Open(sqlite.Open(":memory:"), &gorm.Config{})@@ -98,6 +105,14 @@ func setupResetPendingDB(t *testing.T) *gorm.DB {require.NoError(t, db.Exec(resetPendingSpansDDL).Error)require.NoError(t, db.Exec(resetPendingOpsDDL).Error)require.NoError(t, db.Exec(resetPendingKnowledgeBasesDDL).Error)+ require.NoError(t, db.Exec(resetPendingTenantsDDL).Error)+ // Tenants 7 and 8 are alive; tenant 9 is soft-deleted — its KB-scoped+ // pending rows must be purged by recovery even though the KB row is+ // still present (#3593).+ require.NoError(t, db.Exec(+ "INSERT INTO tenants (id, deleted_at) VALUES (?, NULL), (?, NULL), (?, ?)",+ 7, 8, 9, time.Now(),+ ).Error)return db}@@ -309,8 +324,8 @@ func TestRecoverPendingWikiTasks_RecreatesOneTriggerPerLaneAndKB(t *testing.T) {db := setupResetPendingDB(t)require.NoError(t, db.Exec(`INSERT INTO knowledge_bases (id, tenant_id, deleted_at)- VALUES (?, ?, NULL), (?, ?, NULL), (?, ?, ?)`,- "kb-a", 7, "kb-b", 8, "kb-deleted", 9, time.Now(),+ VALUES (?, ?, NULL), (?, ?, NULL), (?, ?, ?), (?, ?, NULL)`,+ "kb-a", 7, "kb-b", 8, "kb-deleted", 9, time.Now(), "kb-t9", 9,).Error)rows := []struct {tenantID uint64@@ -323,6 +338,7 @@ func TestRecoverPendingWikiTasks_RecreatesOneTriggerPerLaneAndKB(t *testing.T) {{7, types.TypeWikiFinalize, "kb-a", "slug-a"},{8, types.TypeWikiIngest, "kb-b", "k-3"},{9, types.TypeWikiIngest, "kb-deleted", "k-deleted"},+ {9, types.TypeWikiIngest, "kb-t9", "k-t9"}, // live KB of a deleted tenant{10, types.TypeWikiFinalize, "kb-missing", "k-missing"},}for _, row := range rows {@@ -350,7 +366,14 @@ func TestRecoverPendingWikiTasks_RecreatesOneTriggerPerLaneAndKB(t *testing.T) {var orphaned int64require.NoError(t, db.Model(&types.TaskPendingOp{}).- Where("scope_id IN ?", []string{"kb-deleted", "kb-missing"}).+ Where("scope_id IN ?", []string{"kb-deleted", "kb-missing", "kb-t9"}).Count(&orphaned).Error)assert.Zero(t, orphaned)++ // The live KB of a soft-deleted tenant must be purged too: the KB row+ // survives the tenant deletion, so the KB-only predicate would keep+ // re-arming its triggers at every startup (#3593).+ var remaining int64+ require.NoError(t, db.Model(&types.TaskPendingOp{}).Count(&remaining).Error)+ assert.Equal(t, int64(4), remaining)}diff --git a/internal/types/interfaces/task_queue.go b/internal/types/interfaces/task_queue.goindex c2fae3f0c..e6d8dfd75 100644--- a/internal/types/interfaces/task_queue.go+++ b/internal/types/interfaces/task_queue.go@@ -114,6 +114,16 @@ type TaskPendingOpsKnowledgeBaseGuard interface {EnqueueIfKnowledgeBaseActive(ctx context.Context, op *types.TaskPendingOp) (accepted bool, err error)}+// TaskPendingOpsTenantLiveness reports whether a tenant is still alive (not+// soft-deleted). Wiki task consumers use it to guarantee a deleted tenant+// never triggers new model requests: the check runs at task entry (ingest+// and finalize) and inside the guarded enqueue, alongside the KB-active+// check — tenant soft-deletion removes the workspace without touching its+// knowledge bases or durable pending ops.+type TaskPendingOpsTenantLiveness interface {+ HasActiveTenant(ctx context.Context, tenantID uint64) (bool, error)+}+// TaskPendingOpsFinalizingSeeder atomically hands a processing knowledge row// to the asynchronous finalizing pipeline while persisting the durable// pending operation that owns one of its subtask slots.