Skip to content

Open-source LLM knowledge platform: turn raw documents into a queryable RAG, an autonomous reasoning agent, and a self-maintaining Wiki.

fix(tenant): stop wiki tasks of soft-deleted tenants from issuing model requests (#3602)

LuckVdc57766cc7606
diff --git a/internal/application/repository/task_queue.go b/internal/application/repository/task_queue.go
index 1823732cd..f35b72128 100644
--- a/internal/application/repository/task_queue.go
+++ b/internal/application/repository/task_queue.go
@@ -53,11 +53,13 @@ func preparePendingOp(op *types.TaskPendingOp) error {
}
// EnqueueIfKnowledgeBaseActive prevents detached wiki cleanup from writing new
-// durable work after a KB was soft-deleted. On Postgres the share lock
-// serializes this check+insert transaction against the row update performed by
-// soft deletion: whichever operation acquires the row first determines the
-// order, and the deletion path's subsequent scope scrub removes any insert
-// that committed before it.
+// durable work after a KB was soft-deleted. The tenant must also be alive: a
+// tenant soft-deletion removes the workspace without touching its knowledge
+// bases, and a deleted tenant must never accrue new model-backed work. On
+// Postgres the share lock serializes this check+insert transaction against
+// the row update performed by soft deletion: whichever operation acquires the
+// row first determines the order, and the deletion path's subsequent scope
+// scrub removes any insert that committed before it.
func (r *taskPendingOpsRepository) EnqueueIfKnowledgeBaseActive(
ctx context.Context,
op *types.TaskPendingOp,
@@ -84,6 +86,10 @@ func (r *taskPendingOpsRepository) EnqueueIfKnowledgeBaseActive(
}
return err
}
+ active, err := tenantActiveWithinTx(tx, op.TenantID)
+ if err != nil || !active {
+ return err
+ }
if err := tx.Create(op).Error; err != nil {
return err
}
@@ -93,6 +99,31 @@ func (r *taskPendingOpsRepository) EnqueueIfKnowledgeBaseActive(
return accepted, err
}
+// tenantActiveWithinTx reports whether the tenant row exists and is not
+// soft-deleted, inside the caller's transaction. Callers that cannot see a
+// tenants table (legacy test doubles) fail closed.
+func tenantActiveWithinTx(tx *gorm.DB, tenantID uint64) (bool, error) {
+ if tenantID == 0 {
+ return false, nil
+ }
+ var tenant types.Tenant
+ err := tx.Model(&types.Tenant{}).Select("id").Where("id = ?", tenantID).Take(&tenant).Error
+ if errors.Is(err, gorm.ErrRecordNotFound) {
+ return false, nil
+ }
+ if err != nil {
+ return false, err
+ }
+ return true, nil
+}
+
+// HasActiveTenant reports whether the tenant exists and has not been
+// soft-deleted. Wiki task consumers call this before doing durable or
+// model-backed work on the tenant's behalf (#3593).
+func (r *taskPendingOpsRepository) HasActiveTenant(ctx context.Context, tenantID uint64) (bool, error) {
+ return tenantActiveWithinTx(r.db.WithContext(ctx), tenantID)
+}
+
// SeedKnowledgeFinalizingWithPendingOp commits the finalizing counter and the
// durable operation that owns one slot in the same transaction. This closes
// the crash window where a knowledge row could enter finalizing before its
diff --git a/internal/application/repository/task_queue_test.go b/internal/application/repository/task_queue_test.go
index 87d9bf18f..dc8fe547a 100644
--- a/internal/application/repository/task_queue_test.go
+++ b/internal/application/repository/task_queue_test.go
@@ -392,15 +392,32 @@ func TestTaskPendingOps_EnqueueIfKnowledgeBaseActive(t *testing.T) {
deleted_at DATETIME
)`).Error)
require.NoError(t, db.Exec(
- "INSERT INTO knowledge_bases (id, tenant_id, deleted_at) VALUES (?, ?, NULL), (?, ?, ?)",
- "kb-active", 1, "kb-deleted", 1, time.Now(),
+ "INSERT INTO knowledge_bases (id, tenant_id, deleted_at) VALUES (?, ?, NULL), (?, ?, ?), (?, ?, NULL)",
+ "kb-active", 1, "kb-deleted", 1, time.Now(), "kb-t2", 2,
+ ).Error)
+ require.NoError(t, db.Exec(`CREATE TABLE tenants (
+ id INTEGER PRIMARY KEY,
+ deleted_at DATETIME
+ )`).Error)
+ require.NoError(t, db.Exec(
+ "INSERT INTO tenants (id, deleted_at) VALUES (?, NULL), (?, ?)",
+ 1, 2, time.Now(),
).Error)
repo := NewTaskPendingOpsRepository(db)
guard, ok := repo.(interfaces.TaskPendingOpsKnowledgeBaseGuard)
require.True(t, ok)
+ liveness, ok := repo.(interfaces.TaskPendingOpsTenantLiveness)
+ require.True(t, ok, "task pending repository must expose tenant liveness for wiki task guards")
ctx := context.Background()
+ activeTenant, err := liveness.HasActiveTenant(ctx, 1)
+ require.NoError(t, err)
+ assert.True(t, activeTenant)
+ deletedTenant, err := liveness.HasActiveTenant(ctx, 2)
+ require.NoError(t, err)
+ assert.False(t, deletedTenant)
+
accepted, err := guard.EnqueueIfKnowledgeBaseActive(ctx,
makePendingOp(types.TypeWikiIngest, types.TaskScopeKnowledgeBase, "kb-active", "ingest", "active", nil))
require.NoError(t, err)
@@ -426,6 +443,10 @@ func TestTaskPendingOps_EnqueueIfKnowledgeBaseActive(t *testing.T) {
TenantID: 2, TaskType: types.TypeWikiIngest, Scope: types.TaskScopeKnowledgeBase,
ScopeID: "kb-active", Op: "ingest", DedupKey: "wrong-tenant",
}},
+ {name: "deleted tenant with live KB", op: &types.TaskPendingOp{
+ TenantID: 2, TaskType: types.TypeWikiIngest, Scope: types.TaskScopeKnowledgeBase,
+ ScopeID: "kb-t2", Op: "ingest", DedupKey: "deleted-tenant",
+ }},
} {
t.Run(tc.name, func(t *testing.T) {
accepted, err := guard.EnqueueIfKnowledgeBaseActive(ctx, tc.op)
diff --git a/internal/application/service/wiki_deleted_tenant_guard_test.go b/internal/application/service/wiki_deleted_tenant_guard_test.go
new file mode 100644
index 000000000..d84c69443
--- /dev/null
+++ b/internal/application/service/wiki_deleted_tenant_guard_test.go
@@ -0,0 +1,93 @@
+package service
+
+import (
+ "context"
+ "encoding/json"
+ "testing"
+
+ "github.com/Tencent/WeKnora/internal/types"
+ "github.com/Tencent/WeKnora/internal/types/interfaces"
+ "github.com/hibiken/asynq"
+ "github.com/stretchr/testify/assert"
+ "github.com/stretchr/testify/require"
+)
+
+// wikiTenantGuardPendingRepo stubs the slice of TaskPendingOpsRepository the
+// tenant-liveness guard touches: the liveness lookup and the scope cleanup.
+type wikiTenantGuardPendingRepo struct {
+ interfaces.TaskPendingOpsRepository
+ active bool
+ lookupErr error
+ lookups []uint64
+ deletedKBs []string
+}
+
+func (r *wikiTenantGuardPendingRepo) HasActiveTenant(_ context.Context, tenantID uint64) (bool, error) {
+ r.lookups = append(r.lookups, tenantID)
+ return r.active, r.lookupErr
+}
+
+func (r *wikiTenantGuardPendingRepo) DeleteByScope(_ context.Context, scope, scopeID string) error {
+ if scope == types.TaskScopeKnowledgeBase {
+ r.deletedKBs = append(r.deletedKBs, scopeID)
+ }
+ return nil
+}
+
+func TestWikiTaskSkipsDeletedTenant(t *testing.T) {
+ payload, err := json.Marshal(WikiIngestPayload{
+ TenantID: 7,
+ KnowledgeBaseID: "kb-orphaned",
+ Language: "en",
+ })
+ require.NoError(t, err)
+
+ t.Run("ingest clears queue and does not proceed", func(t *testing.T) {
+ repo := &wikiTenantGuardPendingRepo{active: false}
+ svc := &wikiIngestService{pendingRepo: repo}
+
+ err := svc.ProcessWikiIngest(context.Background(), asynq.NewTask(types.TypeWikiIngest, payload))
+ require.NoError(t, err)
+ require.Len(t, repo.lookups, 1)
+ assert.Equal(t, uint64(7), repo.lookups[0])
+ assert.Equal(t, []string{"kb-orphaned"}, repo.deletedKBs)
+ })
+
+ t.Run("finalize clears queue and does not proceed", func(t *testing.T) {
+ repo := &wikiTenantGuardPendingRepo{active: false}
+ svc := &wikiIngestService{pendingRepo: repo}
+
+ err := svc.ProcessWikiFinalize(context.Background(), asynq.NewTask(types.TypeWikiFinalize, payload))
+ require.NoError(t, err)
+ require.Len(t, repo.lookups, 1)
+ assert.Equal(t, []string{"kb-orphaned"}, repo.deletedKBs)
+ })
+}
+
+func TestWikiTenantGuardFailsOpenOnLookupError(t *testing.T) {
+ repo := &wikiTenantGuardPendingRepo{active: false, lookupErr: assert.AnError}
+ svc := &wikiIngestService{pendingRepo: repo}
+
+ // A transient liveness-lookup failure must not be read as "deleted":
+ // the task keeps its retry budget and the enqueue guard / startup
+ // recovery enforce the same invariant on their own DB access.
+ assert.False(t, svc.tenantIsDeleted(context.Background(), 7))
+ assert.Empty(t, repo.deletedKBs)
+}
+
+func TestWikiTenantGuardWithoutLivenessCapability(t *testing.T) {
+ // A pending repo without the liveness extension (legacy test doubles,
+ // alternate implementations) is never treated as a deleted tenant.
+ svc := &wikiIngestService{pendingRepo: &wikiKBGuardPendingRepo{}}
+
+ assert.False(t, svc.tenantIsDeleted(context.Background(), 7))
+}
+
+func TestWikiTenantGuardIgnoresZeroTenant(t *testing.T) {
+ repo := &wikiTenantGuardPendingRepo{active: false}
+ svc := &wikiIngestService{pendingRepo: repo}
+
+ assert.False(t, svc.tenantIsDeleted(context.Background(), 0))
+ assert.Empty(t, repo.lookups)
+ _ = interfaces.TaskPendingOpsTenantLiveness(nil)
+}
diff --git a/internal/application/service/wiki_ingest.go b/internal/application/service/wiki_ingest.go
index da081a5ae..097d85f8a 100644
--- a/internal/application/service/wiki_ingest.go
+++ b/internal/application/service/wiki_ingest.go
@@ -720,6 +720,31 @@ func (s *wikiIngestService) clearDeletedKnowledgeBasePendingOps(ctx context.Cont
return cleaner.DeleteByScope(cleanupCtx, types.TaskScopeKnowledgeBase, kbID)
}
+// tenantIsDeleted reports whether the payload's tenant has been soft-deleted.
+// A tenant deletion removes the workspace but leaves its knowledge bases and
+// durable pending ops in place, so wiki tasks restored from those ops would
+// otherwise keep issuing model requests for a tenant nobody can reach (#3593).
+//
+// Fail-open on transient lookup errors and when the pending repo does not
+// expose tenant liveness (legacy test doubles): the retry machinery still
+// covers the task, and the guarded enqueue / startup recovery paths enforce
+// the same invariant on their own DB access.
+func (s *wikiIngestService) tenantIsDeleted(ctx context.Context, tenantID uint64) bool {
+ if tenantID == 0 {
+ return false
+ }
+ checker, ok := s.pendingRepo.(interfaces.TaskPendingOpsTenantLiveness)
+ if !ok {
+ return false
+ }
+ active, err := checker.HasActiveTenant(ctx, tenantID)
+ if err != nil {
+ logger.Warnf(ctx, "wiki: tenant liveness lookup failed for tenant %d: %v (failing open)", tenantID, err)
+ return false
+ }
+ return !active
+}
+
// releaseIngestForUnavailableWiki drops the KB's queued ingest ops when the
// wiki cannot run for a reason retries will not fix, and releases each
// document's wiki slot in the same transaction so it leaves "finalizing".
diff --git a/internal/application/service/wiki_ingest_batch.go b/internal/application/service/wiki_ingest_batch.go
index d2d79414b..f1d92ad5f 100644
--- a/internal/application/service/wiki_ingest_batch.go
+++ b/internal/application/service/wiki_ingest_batch.go
@@ -254,6 +254,18 @@ func (s *wikiIngestService) ProcessWikiIngest(ctx context.Context, t *asynq.Task
ctx = context.WithValue(ctx, types.LanguageContextKey, payload.Language)
}
+ // A soft-deleted tenant owns no reachable workspace anymore; its KBs and
+ // durable pending ops survive the deletion, so without this guard the
+ // batch would keep issuing model requests (and finalize would rebuild
+ // index pages) for a tenant nobody can see (#3593). Drop the KB's queue.
+ if s.tenantIsDeleted(ctx, payload.TenantID) {
+ exitStatus = "tenant_deleted"
+ if err := s.clearDeletedKnowledgeBasePendingOps(ctx, payload.KnowledgeBaseID); err != nil {
+ return fmt.Errorf("wiki ingest: clear deleted tenant queue: %w", err)
+ }
+ return nil
+ }
+
// Concurrency model (Phase 3):
//
// - Standard (Redis) mode: NO exclusive per-KB lock. Multiple batches
@@ -931,6 +943,16 @@ func (s *wikiIngestService) ProcessWikiFinalize(ctx context.Context, t *asynq.Ta
return nil
}
+ // Same tenant-liveness guard as the ingest batch: finalize calls the
+ // synthesis model to rebuild the index page, and a deleted tenant must
+ // never accrue new model requests (#3593). Drop the KB's queue.
+ if s.tenantIsDeleted(ctx, payload.TenantID) {
+ if err := s.clearDeletedKnowledgeBasePendingOps(ctx, payload.KnowledgeBaseID); err != nil {
+ return fmt.Errorf("wiki finalize: clear deleted tenant queue: %w", err)
+ }
+ return nil
+ }
+
// Per-KB finalize lock, separate from the ingest active lock so finalize
// and ingest batches never block each other. Coalescing via asynq.TaskID
// already keeps at most one finalize pending per KB; this guards the
diff --git a/internal/container/recover_pending_wiki_tasks.go b/internal/container/recover_pending_wiki_tasks.go
index 1867e055e..adf242020 100644
--- a/internal/container/recover_pending_wiki_tasks.go
+++ b/internal/container/recover_pending_wiki_tasks.go
@@ -34,16 +34,24 @@ func recoverPendingWikiTasks(db *gorm.DB, task interfaces.TaskEnqueuer) {
return
}
ctx := context.Background()
- const activeKnowledgeBase = `EXISTS (
+ // The outer parentheses matter: the cleanup below negates this whole
+ // predicate with NOT, and `NOT EXISTS(...) AND EXISTS(...)` would parse
+ // as `(NOT EXISTS(...)) AND EXISTS(...)` — silently matching nothing.
+ const activeKnowledgeBase = `(EXISTS (
SELECT 1 FROM knowledge_bases kb
WHERE kb.id = task_pending_ops.scope_id
AND kb.tenant_id = task_pending_ops.tenant_id
AND kb.deleted_at IS NULL
- )`
+ ) AND EXISTS (
+ SELECT 1 FROM tenants t
+ WHERE t.id = task_pending_ops.tenant_id
+ AND t.deleted_at IS NULL
+ ))`
wikiTaskTypes := []string{types.TypeWikiIngest, types.TypeWikiFinalize}
- // Durable rows for a deleted/missing KB must not recreate ephemeral
- // triggers at startup. Fail closed if this cleanup cannot be verified.
+ // Durable rows for a deleted/missing KB — or a soft-deleted tenant —
+ // must not recreate ephemeral triggers at startup. Fail closed if this
+ // cleanup cannot be verified.
cleanup := db.WithContext(ctx).
Where("scope = ? AND task_type IN ?", types.TaskScopeKnowledgeBase, wikiTaskTypes).
Where("NOT " + activeKnowledgeBase).
@@ -53,7 +61,9 @@ func recoverPendingWikiTasks(db *gorm.DB, task interfaces.TaskEnqueuer) {
return
}
if cleanup.RowsAffected > 0 {
- logger.Infof(ctx, "[WikiRecovery] removed %d pending row(s) for deleted knowledge bases", cleanup.RowsAffected)
+ logger.Infof(ctx,
+ "[WikiRecovery] removed %d pending row(s) for deleted knowledge bases or tenants",
+ cleanup.RowsAffected)
}
var scopes []pendingWikiScope
diff --git a/internal/container/reset_pending_tasks_test.go b/internal/container/reset_pending_tasks_test.go
index 47d4c0248..4df02aa0b 100644
--- a/internal/container/reset_pending_tasks_test.go
+++ b/internal/container/reset_pending_tasks_test.go
@@ -89,6 +89,13 @@ CREATE TABLE IF NOT EXISTS knowledge_bases (
);
`
+var resetPendingTenantsDDL = `
+CREATE TABLE IF NOT EXISTS tenants (
+ id INTEGER PRIMARY KEY,
+ deleted_at DATETIME
+);
+`
+
func setupResetPendingDB(t *testing.T) *gorm.DB {
t.Helper()
db, err := gorm.Open(sqlite.Open(":memory:"), &gorm.Config{})
@@ -98,6 +105,14 @@ func setupResetPendingDB(t *testing.T) *gorm.DB {
require.NoError(t, db.Exec(resetPendingSpansDDL).Error)
require.NoError(t, db.Exec(resetPendingOpsDDL).Error)
require.NoError(t, db.Exec(resetPendingKnowledgeBasesDDL).Error)
+ require.NoError(t, db.Exec(resetPendingTenantsDDL).Error)
+ // Tenants 7 and 8 are alive; tenant 9 is soft-deleted — its KB-scoped
+ // pending rows must be purged by recovery even though the KB row is
+ // still present (#3593).
+ require.NoError(t, db.Exec(
+ "INSERT INTO tenants (id, deleted_at) VALUES (?, NULL), (?, NULL), (?, ?)",
+ 7, 8, 9, time.Now(),
+ ).Error)
return db
}
@@ -309,8 +324,8 @@ func TestRecoverPendingWikiTasks_RecreatesOneTriggerPerLaneAndKB(t *testing.T) {
db := setupResetPendingDB(t)
require.NoError(t, db.Exec(
`INSERT INTO knowledge_bases (id, tenant_id, deleted_at)
- VALUES (?, ?, NULL), (?, ?, NULL), (?, ?, ?)`,
- "kb-a", 7, "kb-b", 8, "kb-deleted", 9, time.Now(),
+ VALUES (?, ?, NULL), (?, ?, NULL), (?, ?, ?), (?, ?, NULL)`,
+ "kb-a", 7, "kb-b", 8, "kb-deleted", 9, time.Now(), "kb-t9", 9,
).Error)
rows := []struct {
tenantID uint64
@@ -323,6 +338,7 @@ func TestRecoverPendingWikiTasks_RecreatesOneTriggerPerLaneAndKB(t *testing.T) {
{7, types.TypeWikiFinalize, "kb-a", "slug-a"},
{8, types.TypeWikiIngest, "kb-b", "k-3"},
{9, types.TypeWikiIngest, "kb-deleted", "k-deleted"},
+ {9, types.TypeWikiIngest, "kb-t9", "k-t9"}, // live KB of a deleted tenant
{10, types.TypeWikiFinalize, "kb-missing", "k-missing"},
}
for _, row := range rows {
@@ -350,7 +366,14 @@ func TestRecoverPendingWikiTasks_RecreatesOneTriggerPerLaneAndKB(t *testing.T) {
var orphaned int64
require.NoError(t, db.Model(&types.TaskPendingOp{}).
- Where("scope_id IN ?", []string{"kb-deleted", "kb-missing"}).
+ Where("scope_id IN ?", []string{"kb-deleted", "kb-missing", "kb-t9"}).
Count(&orphaned).Error)
assert.Zero(t, orphaned)
+
+ // The live KB of a soft-deleted tenant must be purged too: the KB row
+ // survives the tenant deletion, so the KB-only predicate would keep
+ // re-arming its triggers at every startup (#3593).
+ var remaining int64
+ require.NoError(t, db.Model(&types.TaskPendingOp{}).Count(&remaining).Error)
+ assert.Equal(t, int64(4), remaining)
}
diff --git a/internal/types/interfaces/task_queue.go b/internal/types/interfaces/task_queue.go
index c2fae3f0c..e6d8dfd75 100644
--- a/internal/types/interfaces/task_queue.go
+++ b/internal/types/interfaces/task_queue.go
@@ -114,6 +114,16 @@ type TaskPendingOpsKnowledgeBaseGuard interface {
EnqueueIfKnowledgeBaseActive(ctx context.Context, op *types.TaskPendingOp) (accepted bool, err error)
}
+// TaskPendingOpsTenantLiveness reports whether a tenant is still alive (not
+// soft-deleted). Wiki task consumers use it to guarantee a deleted tenant
+// never triggers new model requests: the check runs at task entry (ingest
+// and finalize) and inside the guarded enqueue, alongside the KB-active
+// check — tenant soft-deletion removes the workspace without touching its
+// knowledge bases or durable pending ops.
+type TaskPendingOpsTenantLiveness interface {
+ HasActiveTenant(ctx context.Context, tenantID uint64) (bool, error)
+}
+
// TaskPendingOpsFinalizingSeeder atomically hands a processing knowledge row
// to the asynchronous finalizing pipeline while persisting the durable
// pending operation that owns one of its subtask slots.