Open-source LLM knowledge platform: turn raw documents into a queryable RAG, an autonomous reasoning agent, and a self-maintaining Wiki.
Go agentagenticaichatbotdsh-pluginembeddingsevaluationgenerative-aigolangknowledge-basellmmulti-tenantollamaopenaiquestion-answeringragrerankingsemantic-searchvector-searchwiki
fix(wiki): bound outstanding follow-up tasks
wizardchenc819abd9589c
diff --git a/internal/application/repository/task_queue.go b/internal/application/repository/task_queue.goindex f35b72128..9df8fd76d 100644--- a/internal/application/repository/task_queue.go+++ b/internal/application/repository/task_queue.go@@ -480,6 +480,25 @@ func (r *taskPendingOpsRepository) PendingCount(return n, nil}+// ClaimableCount uses ClaimBatch's eligibility and grouping rules so running+// documents and multiple queued operations for one document do not inflate+// the number of batches a scheduler needs to dispatch.+func (r *taskPendingOpsRepository) ClaimableCount(+ ctx context.Context, taskType, scope, scopeID string, staleBefore time.Time,+) (int64, error) {+ freshKeys := r.db.WithContext(ctx).Model(&types.TaskPendingOp{}).+ Select("dedup_key").+ Where("task_type = ? AND scope = ? AND scope_id = ?", taskType, scope, scopeID).+ Where("claimed_at IS NOT NULL AND claimed_at >= ?", staleBefore)+ var n int64+ err := r.db.WithContext(ctx).Model(&types.TaskPendingOp{}).+ Where("task_type = ? AND scope = ? AND scope_id = ?", taskType, scope, scopeID).+ Where("claimed_at IS NULL OR claimed_at < ?", staleBefore).+ Where("dedup_key NOT IN (?)", freshKeys).+ Distinct("dedup_key").Count(&n).Error+ return n, err+}+// DeleteByDedupKey drops rows in the tuple whose dedup_key matches.// If `op` is non-empty, only rows with the matching op are dropped;// otherwise every matching row is removed. Empty dedup_key is rejecteddiff --git a/internal/application/repository/task_queue_test.go b/internal/application/repository/task_queue_test.goindex dc8fe547a..08aaafd9f 100644--- a/internal/application/repository/task_queue_test.go+++ b/internal/application/repository/task_queue_test.go@@ -1037,3 +1037,44 @@ func TestTaskPendingOps_DrainUnclaimedAndReleaseRollsBackOnReleaseFailure(t *tesrequire.NoError(t, db.Model(&types.TaskPendingOp{}).Count(&count).Error)assert.Equal(t, int64(1), count)}++func TestTaskPendingOps_ClaimableCountMatchesClaimBatch(t *testing.T) {+ db := setupTaskQueueTestDB(t)+ repo := NewTaskPendingOpsRepository(db)+ counter := repo.(interfaces.TaskPendingOpsClaimableCounter)+ ctx := context.Background()+ staleBefore := time.Now().Add(-time.Hour)+ for _, key := range []string{"ready", "ready", "stale", "stale", "busy", "busy"} {+ require.NoError(t, repo.Enqueue(ctx,+ makePendingOp("wiki:ingest", "knowledge_base", "kb-1", "ingest", key, nil)))+ }+ require.NoError(t, db.Model(&types.TaskPendingOp{}).Where("dedup_key = ?", "stale").+ Update("claimed_at", staleBefore.Add(-time.Second)).Error)+ require.NoError(t, db.Model(&types.TaskPendingOp{}).Where("dedup_key = ?", "busy").+ Update("claimed_at", time.Now()).Error)+ // An unclaimed late sibling of a busy key is not eligible on its own.+ require.NoError(t, repo.Enqueue(ctx,+ makePendingOp("wiki:ingest", "knowledge_base", "kb-1", "retract", "busy", nil)))+ // Identical keys in another tuple must not exclude the ready document.+ other := makePendingOp("wiki:ingest", "knowledge_base", "other-kb", "ingest", "ready", nil)+ require.NoError(t, repo.Enqueue(ctx, other))+ require.NoError(t, db.Model(&types.TaskPendingOp{}).Where("scope_id = ?", "other-kb").+ Update("claimed_at", time.Now()).Error)+ n, err := counter.ClaimableCount(ctx, "wiki:ingest", "knowledge_base", "kb-1", staleBefore)+ require.NoError(t, err)+ require.Equal(t, int64(2), n)+ rows, err := repo.ClaimBatch(ctx, "wiki:ingest", "knowledge_base", "kb-1", 2, staleBefore)+ require.NoError(t, err)+ require.Len(t, rows, 4)+ keys := map[string]bool{}+ for _, row := range rows {+ keys[row.DedupKey] = true+ }+ require.Len(t, keys, int(n))+ n, err = counter.ClaimableCount(ctx, "wiki:ingest", "knowledge_base", "kb-1", staleBefore)+ require.NoError(t, err)+ require.Zero(t, n)+ pending, err := repo.PendingCount(ctx, "wiki:ingest", "knowledge_base", "kb-1")+ require.NoError(t, err)+ require.Equal(t, int64(7), pending, "crash recovery must still see claimed rows")+}diff --git a/internal/application/service/wiki_ingest_batch.go b/internal/application/service/wiki_ingest_batch.goindex 796f7c0a0..e9c743971 100644--- a/internal/application/service/wiki_ingest_batch.go+++ b/internal/application/service/wiki_ingest_batch.go@@ -18,26 +18,16 @@ import ("github.com/Tencent/WeKnora/internal/models/chat""github.com/Tencent/WeKnora/internal/tracing/langfuse""github.com/Tencent/WeKnora/internal/types"+ "github.com/Tencent/WeKnora/internal/types/interfaces""github.com/google/uuid""github.com/hibiken/asynq""golang.org/x/sync/errgroup")-// followUpTriggerCount decides how many follow-up triggers a completed batch-// should re-arm for its KB (#3461). Since Phase 3 the consumption side-// (disjoint claiming, per-slug locks, the inflight cap) fully supports-// concurrent batches, but nothing created more than one trigger for a KB with-// a standing backlog — so once uploads stopped, the backlog drained at-// 1 batch × batch-size per cycle regardless of ingest_max_inflight, leaving-// the wiki pool and the model server idle.-//-// The count is the smaller of (a) how many batches the remaining rows still-// need and (b) how many slots could be free by the time the triggers fire:-// the caller's own slot releases right after it returns, hence the +1.-// activeSlots < 0 means "unknown" (no Redis / Redis error) and maxInflight-// <= 1 cannot host more than one batch anyway; both collapse to 1, matching-// the pre-fan-out behaviour. Over-issued triggers are harmless: the ones that-// cannot get a slot are coalesced by scheduleCappedRetry's fixed TaskID.+// followUpTriggerCount estimates useful new work from claimable documents and+// running slots. reserveFollowUpTasks additionally bounds all outstanding+// follow-ups (scheduled, queued, running and retrying) across batch completions.+// The +1 allows the completing batch to replace itself before releasing its slot.func followUpTriggerCount(pending, batchSize, activeSlots, maxInflight int) int {if pending <= 0 || batchSize <= 0 {return 0@@ -53,31 +43,9 @@ func followUpTriggerCount(pending, batchSize, activeSlots, maxInflight int) intreturn min(batchesNeeded, freeSlots)}-// scheduleFollowUp enqueues asynq trigger tasks if there are still pending-// ops in task_pending_ops for this KB. Returns true when at least one-// follow-up was scheduled.-//-// Post-Phase-3 this backstops the case where a batch drained its claimed-// window but more rows remain and no other trigger is pending (e.g. a steady-// trickle of uploads). Standard mode already fans a KB's backlog across-// concurrent claiming batches, so the short delay is normally just a light-// debounce rather than a lock-release wait.-//-// How many triggers are re-armed follows followUpTriggerCount above: enough-// to drain the backlog, capped by the free inflight slots, so the concurrent-// batch count converges on ingest_max_inflight instead of idling at 1.-//-// Fan-out is deliberately skipped when rateLimited: widening concurrency-// during a 429 storm only amplifies the backoff. Lite mode (no Redis) also-// never fans out — liteLocks serializes batches per KB, so extra triggers-// would only bounce off ErrWikiIngestConcurrent.-//-// `delay` is the ProcessIn before each follow-up fires. Callers pass-// wikiFollowUpDelay for the normal case and wikiRateLimitBackoff when the-// batch tripped an upstream rate limit — the released failed rows are-// eligible immediately, but nothing claims them until a trigger fires, so-// stretching the follow-up interval is what actually paces retries down-// during a 429 storm.+// scheduleFollowUp replenishes the KB's bounded supply of follow-up tasks.+// Lite mode and rate-limited completions request only one successor. A 429+// delays this batch's successor; it does not pause other batches in the KB.func (s *wikiIngestService) scheduleFollowUp(ctx context.Context,payload WikiIngestPayload,@@ -93,38 +61,74 @@ func (s *wikiIngestService) scheduleFollowUp(return false}+ if s.redisClient != nil {+ if counter, ok := s.pendingRepo.(interfaces.TaskPendingOpsClaimableCounter); ok {+ count, err = counter.ClaimableCount(ctx, wikiTaskType, wikiTaskScope,+ payload.KnowledgeBaseID, time.Now().Add(-wikiClaimStaleAfter))+ if err != nil {+ logger.Warnf(ctx, "wiki ingest: claimable count failed: %v", err)+ return s.scheduleFollowUpRecheck(ctx, payload)+ }+ if count == 0 {+ return s.scheduleStaleClaimRecheck(ctx, payload)+ }+ }+ // ClaimBatch caps the number of distinct keys at 1000.+ batchSize = min(batchSize, 1000)+ }+// activeSlots stays -1 (unknown → single follow-up) in Lite mode and on- // a rate-limited exit; -1 also shows up in the log line as "unknown".+ // a rate-limited exit.activeSlots := -1if !rateLimited {activeSlots = s.activeInflightSlots(ctx, payload.KnowledgeBaseID)}n := followUpTriggerCount(int(count), batchSize, activeSlots, maxInflight)- activeStr := fmt.Sprintf("%d", activeSlots)- if activeSlots < 0 {- activeStr = "unknown"+ var taskIDs []string+ if s.redisClient != nil {+ taskIDs, err = s.reserveFollowUpTasks(ctx, payload.KnowledgeBaseID, n, maxInflight)+ if err != nil {+ logger.Warnf(ctx, "wiki ingest: follow-up reservation failed: %v", err)+ // Keep the pre-fan-out behavior while Redis is unavailable.+ n = 1+ } else {+ n = len(taskIDs)+ if n == 0 {+ return s.scheduleFollowUpRecheck(ctx, payload)+ }+ }}- logger.Infof(ctx,- "wiki ingest: %d more documents pending for KB %s, scheduling %d follow-up(s) in %s (active slots %s/%d)",- count, payload.KnowledgeBaseID, n, delay, activeStr, maxInflight)+ logger.Infof(ctx, "wiki ingest: %d claimable documents for KB %s, scheduling %d follow-up(s) in %s",+ count, payload.KnowledgeBaseID, n, delay)langfuse.InjectTracing(ctx, &payload)payloadBytes, _ := json.Marshal(payload)scheduled := false+ needsRecheck := falsefor i := 0; i < n; i++ {- t := asynq.NewTask(types.TypeWikiIngest, payloadBytes,+ opts := []asynq.Option{asynq.Queue(types.QueueWiki),asynq.MaxRetry(wikiIngestMaxRetry),- asynq.Timeout(60*time.Minute),- asynq.ProcessIn(delay), // debounce (or rate-limit backoff) before draining the remainder- )+ asynq.Timeout(60 * time.Minute),+ asynq.ProcessIn(delay),+ }+ if len(taskIDs) > 0 {+ opts = append(opts, asynq.TaskID(taskIDs[i]))+ }+ t := asynq.NewTask(types.TypeWikiIngest, payloadBytes, opts...)if _, err := s.task.Enqueue(t); err != nil {logger.Warnf(ctx, "wiki ingest: follow-up enqueue %d/%d failed: %v", i+1, n, err)+ // A timeout may mean the enqueue succeeded. Keep its reservation+ // until reconciliation can establish the actual queue state.+ needsRecheck = truecontinue}scheduled = true}+ if needsRecheck {+ scheduled = s.scheduleFollowUpRecheck(ctx, payload) || scheduled+ }return scheduled}@@ -253,7 +257,7 @@ func (s *wikiIngestService) newWikiBatchContext(}}-func (s *wikiIngestService) ProcessWikiIngest(ctx context.Context, t *asynq.Task) error {+func (s *wikiIngestService) ProcessWikiIngest(ctx context.Context, t *asynq.Task) (retErr error) {taskStartedAt := time.Now()retryCount, _ := asynq.GetRetryCount(ctx)maxRetry, _ := asynq.GetMaxRetry(ctx)@@ -312,6 +316,16 @@ func (s *wikiIngestService) ProcessWikiIngest(ctx context.Context, t *asynq.Taskreturn fmt.Errorf("wiki ingest: unmarshal payload: %w", err)}+ defer func() {+ if p := recover(); p != nil {+ // Asynq retries panics too: retain the outstanding reservation.+ panic(p)+ }+ if retErr == nil {+ s.releaseFollowUpTask(ctx, payload.KnowledgeBaseID)+ }+ }()+// Inject contextctx = context.WithValue(ctx, types.TenantIDContextKey, payload.TenantID)if payload.Language != "" {diff --git a/internal/application/service/wiki_ingest_followup.go b/internal/application/service/wiki_ingest_followup.gonew file mode 100644index 000000000..3bcd96752--- /dev/null+++ b/internal/application/service/wiki_ingest_followup.go@@ -0,0 +1,152 @@+package service++import (+ "context"+ "encoding/json"+ "errors"+ "strconv"+ "time"++ "github.com/Tencent/WeKnora/internal/logger"+ "github.com/Tencent/WeKnora/internal/tracing/langfuse"+ "github.com/Tencent/WeKnora/internal/types"+ "github.com/google/uuid"+ "github.com/hibiken/asynq"+ "github.com/redis/go-redis/v9"+)++const (+ wikiFollowUpPrefix = "wiki:followup:"+ // This is a reconciliation interval, not a queue lease. A task can wait+ // arbitrarily long for a worker without losing its reserved capacity.+ // Missing tasks are retained for this interval to allow an enqueue in+ // progress to finish, including when its caller times out.+ wikiFollowUpReconcileAfter = 2 * time.Minute+)++// Reservations cover the entire asynq task lifetime, including delayed tasks+// and retries. Keep one spare beyond the running cap so the last completing+// task can enqueue its successor before its own reservation is released.+// Atomic ZCARD + ZADD makes concurrent completions share the same budget.+const wikiFollowUpReserveScript = `+local available = math.max(0, tonumber(ARGV[2]) - redis.call('ZCARD', KEYS[1]))+local n = math.min(tonumber(ARGV[1]), available)+local ids = {}+for i = 1, n do+ local id = ARGV[3] .. ':' .. i+ redis.call('ZADD', KEYS[1], ARGV[4], id)+ ids[#ids + 1] = id+end+return ids+`++// Do not expire a reservation just because the task has waited a long time.+// Inspect asynq first, then remove/refresh only if another reconciler hasn't+// already updated it. ZADD XX also avoids resurrecting a completed task.+const wikiFollowUpReconcileScript = `+local score = redis.call('ZSCORE', KEYS[1], ARGV[1])+if score and tonumber(score) == tonumber(ARGV[2]) then+ if ARGV[3] == 'remove' then+ redis.call('ZREM', KEYS[1], ARGV[1])+ else+ redis.call('ZADD', KEYS[1], 'XX', ARGV[3], ARGV[1])+ end+end+return 1+`++func (s *wikiIngestService) reserveFollowUpTasks(+ ctx context.Context, kbID string, requested, maxInflight int,+) ([]string, error) {+ if requested <= 0 {+ return nil, nil+ }+ if err := s.reconcileFollowUpTasks(ctx, kbID); err != nil {+ return nil, err+ }+ return s.redisClient.Eval(ctx, wikiFollowUpReserveScript,+ []string{wikiFollowUpPrefix + kbID}, requested, max(1, maxInflight)+1,+ "wiki-followup-"+uuid.NewString(), time.Now().Add(wikiFollowUpReconcileAfter).UnixMilli(),+ ).StringSlice()+}++func (s *wikiIngestService) reconcileFollowUpTasks(ctx context.Context, kbID string) error {+ key := wikiFollowUpPrefix + kbID+ now := time.Now()+ entries, err := s.redisClient.ZRangeByScoreWithScores(ctx, key, &redis.ZRangeBy{+ Min: "-inf", Max: strconv.FormatInt(now.UnixMilli(), 10),+ }).Result()+ if err != nil || len(entries) == 0 {+ return err+ }+ // Shares the service's Redis connection; Inspector.Close must not close it.+ inspector := asynq.NewInspectorFromRedisClient(s.redisClient)+ for _, entry := range entries {+ if err := ctx.Err(); err != nil {+ return err+ }+ id, ok := entry.Member.(string)+ if !ok {+ continue+ }+ info, err := inspector.GetTaskInfo(types.QueueWiki, id)+ missing := errors.Is(err, asynq.ErrTaskNotFound) || errors.Is(err, asynq.ErrQueueNotFound)+ if err != nil && !missing {+ return err+ }+ action := strconv.FormatInt(now.Add(wikiFollowUpReconcileAfter).UnixMilli(), 10)+ if missing || info.State == asynq.TaskStateArchived || info.State == asynq.TaskStateCompleted {+ action = "remove"+ }+ if err := s.redisClient.Eval(ctx, wikiFollowUpReconcileScript,+ []string{key}, id, entry.Score, action).Err(); err != nil {+ return err+ }+ }+ return nil+}++func (s *wikiIngestService) releaseFollowUpTask(ctx context.Context, kbID string) {+ if s.redisClient == nil {+ return+ }+ id, ok := asynq.GetTaskID(ctx)+ if !ok {+ return+ }+ cleanupCtx, cancel := wikiIngestCleanupContext(ctx)+ defer cancel()+ if err := s.redisClient.ZRem(cleanupCtx, wikiFollowUpPrefix+kbID, id).Err(); err != nil {+ logger.Warnf(ctx, "wiki ingest: follow-up reservation release failed: %v", err)+ }+}++// A coalesced probe keeps progress possible when all capacity is reserved but+// an enqueue failed or a worker died. It also revisits reservations whose+// cleanup failed. It is outside the reservation budget and contributes at+// most two probes per KB, independent of the backlog size.+func (s *wikiIngestService) scheduleFollowUpRecheck(ctx context.Context, payload WikiIngestPayload) bool {+ langfuse.InjectTracing(ctx, &payload)+ b, _ := json.Marshal(payload)+ id := "wiki-followup-recheck-" + payload.KnowledgeBaseID+ if current, _ := asynq.GetTaskID(ctx); current == id {+ // The current task ID remains occupied until asynq acknowledges it.+ // Alternate between two IDs so the probe can re-arm itself.+ id += "-next"+ }+ t := asynq.NewTask(types.TypeWikiIngest, b,+ asynq.Queue(types.QueueWiki),+ asynq.MaxRetry(wikiIngestMaxRetry),+ asynq.Timeout(60*time.Minute),+ asynq.ProcessIn(wikiFollowUpReconcileAfter+wikiFollowUpDelay),+ asynq.TaskID(id),+ )+ if _, err := s.task.Enqueue(t); err != nil {+ if errors.Is(err, asynq.ErrTaskIDConflict) || errors.Is(err, asynq.ErrDuplicateTask) {+ return true+ }+ logger.Warnf(ctx, "wiki ingest: follow-up recheck enqueue failed: %v", err)+ return false+ }+ return true+}diff --git a/internal/application/service/wiki_ingest_followup_test.go b/internal/application/service/wiki_ingest_followup_test.goindex 2536661e3..6657edabb 100644--- a/internal/application/service/wiki_ingest_followup_test.go+++ b/internal/application/service/wiki_ingest_followup_test.go@@ -4,6 +4,7 @@ import ("context""encoding/json""errors"+ "fmt""sync""testing""time"@@ -17,10 +18,12 @@ import (// --- test doubles -----------------------------------------------------------// wikiPendingRepoForFollowUpTest reuses the cleanup-test repo and makes-// PendingCount configurable, which is the only input scheduleFollowUp reads.+// PendingCount and the eligible-document count configurable.type wikiPendingRepoForFollowUpTest struct {wikiPendingRepoForCleanupTest- pending int64+ pending int64+ claimable *int64+ claimableErr error}func (r *wikiPendingRepoForFollowUpTest) PendingCount(@@ -29,6 +32,15 @@ func (r *wikiPendingRepoForFollowUpTest) PendingCount(return r.pending, nil}+func (r *wikiPendingRepoForFollowUpTest) ClaimableCount(+ context.Context, string, string, string, time.Time,+) (int64, error) {+ if r.claimable != nil {+ return *r.claimable, r.claimableErr+ }+ return r.pending, r.claimableErr+}+// followUpEnqueuerRecorder captures every trigger scheduleFollowUp enqueues.// failFrom >= 0 makes enqueues from that index onward return failErr so the// partial-failure path can be exercised.@@ -248,47 +260,274 @@ func TestScheduleFollowUpPartialEnqueueFailureStillReportsScheduled(t *testing.Tgot := svc.scheduleFollowUp(context.Background(),WikiIngestPayload{KnowledgeBaseID: "kb-1"}, wikiFollowUpDelay, 10, 4, false)require.True(t, got)- require.Equal(t, 4, rec.count())-}--// --- convergence --------------------------------------------------------------// TestFollowUpFanOutConvergesUnderCap simulates successive batch completions-// fanning out on a board whose slots never free (worst case for over-issue)-// and asserts the per-completion trigger count never exceeds the free slots,-// i.e. the supply side alone cannot push concurrency past ingest_max_inflight.-func TestFollowUpFanOutConvergesUnderCap(t *testing.T) {- const (- maxInflight = 8- batchSize = 10- )- var (- pending = 1000- activeSlots = 1 // the completing batch itself; releases on return- )- seen := map[int]bool{}- for round := 0; round < 6; round++ {- n := followUpTriggerCount(pending, batchSize, activeSlots, maxInflight)- require.Greater(t, n, 0)- require.LessOrEqual(t, n, maxInflight-activeSlots+1,- "round %d: supply must not exceed free slots", round)- if activeSlots == maxInflight {- require.Equal(t, 1, n,- "round %d: a full board must clamp supply to the follow-up chain", round)- }- seen[n] = true-- // The n triggers fire: n-1 of them claim slots (one bounces off the- // cap into scheduleCappedRetry when the board is full), each claims a- // batch of batchSize rows.- newBatches := min(n, maxInflight-activeSlots)- pending -= newBatches * batchSize- activeSlots = maxInflight- if pending <= 0 {- pending = 1 // keep the loop meaningful; count stays capped below- }+ require.Equal(t, 5, rec.count()) // four follow-ups and one recovery probe+}++// Use the actual asynq client so these tests exercise atomic reservations and+// queue state, rather than merely checking the single-completer arithmetic.+func newQueuedFollowUpTestService(t *testing.T) (*wikiIngestService, *asynq.Inspector) {+ t.Helper()+ mr := miniredis.RunT(t)+ rdb := redis.NewClient(&redis.Options{Addr: mr.Addr()})+ t.Cleanup(func() { _ = rdb.Close() })+ client := asynq.NewClient(asynq.RedisClientOpt{Addr: mr.Addr()})+ t.Cleanup(func() { _ = client.Close() })+ return &wikiIngestService{+ redisClient: rdb,+ task: client,+ pendingRepo: &wikiPendingRepoForFollowUpTest{pending: 24624},+ }, asynq.NewInspectorFromRedisClient(rdb)+}++func expireFollowUpChecks(t *testing.T, svc *wikiIngestService) {+ t.Helper()+ ctx := context.Background()+ ids, err := svc.redisClient.ZRange(ctx, wikiFollowUpPrefix+"kb-1", 0, -1).Result()+ require.NoError(t, err)+ for _, id := range ids {+ require.NoError(t, svc.redisClient.ZAdd(ctx, wikiFollowUpPrefix+"kb-1", redis.Z{+ Score: float64(time.Now().Add(-time.Minute).UnixMilli()), Member: id,+ }).Err())+ }+}++func TestScheduleFollowUpWorkerPoolBelowCap(t *testing.T) {+ svc, inspector := newQueuedFollowUpTestService(t)+ ctx := context.Background()+ payload := WikiIngestPayload{KnowledgeBaseID: "kb-1"}+ // Startup recovery has one running batch, whose completion fans out.+ release, ok := svc.reserveInflightSlot(ctx, "kb-1", 32)+ require.True(t, ok)+ require.True(t, svc.scheduleFollowUp(ctx, payload, wikiFollowUpDelay, 10, 32, false))+ release()+ queued, err := inspector.ListScheduledTasks("wiki", asynq.PageSize(100))+ require.NoError(t, err)+ require.Len(t, queued, 32)++ // Model eight workers dequeuing tasks. Keep their follow-up reservations+ // until completion, just as the handler does, and consume one queued task+ // for each finished batch. No model, database or external Redis is used.+ runningIDs := make([]string, 8)+ for i := range runningIDs {+ runningIDs[i] = queued[i].ID+ require.NoError(t, inspector.DeleteTask("wiki", queued[i].ID))+ }+ reserveSlots(t, svc, "kb-1", 8, 32)+ for completed := 0; completed < 100; completed++ {+ svc.pendingRepo.(*wikiPendingRepoForFollowUpTest).pending -= 10+ require.True(t, svc.scheduleFollowUp(ctx, payload, wikiFollowUpDelay, 10, 32, false))+ worker := completed % len(runningIDs)+ require.NoError(t, svc.redisClient.ZRem(ctx, wikiFollowUpPrefix+"kb-1", runningIDs[worker]).Err())+ queued, err = inspector.ListScheduledTasks("wiki", asynq.PageSize(100))+ require.NoError(t, err)+ require.NotEmpty(t, queued)+ runningIDs[worker] = queued[0].ID+ require.NoError(t, inspector.DeleteTask("wiki", queued[0].ID))+ info, err := inspector.GetQueueInfo("wiki")+ require.NoError(t, err)+ require.LessOrEqual(t, info.Scheduled, 25, "completion %d must not multiply the queue", completed)}- // The first round must have fanned out beyond the follow-up chain —- // otherwise this would not be testing convergence at all.- require.Greater(t, len(seen), 1)+}++func TestScheduleFollowUpConcurrentCompletionsShareBudget(t *testing.T) {+ svc, inspector := newQueuedFollowUpTestService(t)+ reserveSlots(t, svc, "kb-1", 16, 32)+ start := make(chan struct{})+ var wg sync.WaitGroup+ for i := 0; i < 16; i++ {+ wg.Add(1)+ go func() {+ defer wg.Done()+ <-start+ svc.scheduleFollowUp(context.Background(), WikiIngestPayload{KnowledgeBaseID: "kb-1"},+ wikiFollowUpDelay, 10, 32, false)+ }()+ }+ close(start)+ wg.Wait()+ info, err := inspector.GetQueueInfo("wiki")+ require.NoError(t, err)+ // Cap + one successor spare + at most one coalesced recovery probe.+ require.LessOrEqual(t, info.Scheduled, 34)+ require.Equal(t, int64(33), svc.redisClient.ZCard(context.Background(), wikiFollowUpPrefix+"kb-1").Val())+}++func TestScheduleFollowUpClaimedRowsOnlyKeepsCrashRecovery(t *testing.T) {+ svc, inspector := newQueuedFollowUpTestService(t)+ zero := int64(0)+ svc.pendingRepo.(*wikiPendingRepoForFollowUpTest).claimable = &zero+ require.True(t, svc.scheduleFollowUp(context.Background(), WikiIngestPayload{KnowledgeBaseID: "kb-1"},+ wikiFollowUpDelay, 10, 32, false))+ tasks, err := inspector.ListScheduledTasks("wiki")+ require.NoError(t, err)+ require.Len(t, tasks, 1)+ require.Equal(t, "wiki-ingest-recheck-kb-1", tasks[0].ID)+ require.Greater(t, time.Until(tasks[0].NextProcessAt), wikiClaimStaleAfter-time.Second)+}++func TestFollowUpReservationsKeepLongQueuedTasks(t *testing.T) {+ svc, inspector := newQueuedFollowUpTestService(t)+ ctx := context.Background()+ payload := WikiIngestPayload{KnowledgeBaseID: "kb-1"}+ require.True(t, svc.scheduleFollowUp(ctx, payload, wikiFollowUpDelay, 10, 4, false))+ for i := 0; i < 5; i++ {+ expireFollowUpChecks(t, svc)+ require.True(t, svc.scheduleFollowUp(ctx, payload, wikiFollowUpDelay, 10, 4, false))+ }+ info, err := inspector.GetQueueInfo("wiki")+ require.NoError(t, err)+ require.LessOrEqual(t, info.Scheduled, 6) // five reserved tasks plus one probe+ require.Equal(t, int64(5), svc.redisClient.ZCard(ctx, wikiFollowUpPrefix+"kb-1").Val())+}++func TestFollowUpReservationsRecoverMissingAndArchivedTasks(t *testing.T) {+ svc, inspector := newQueuedFollowUpTestService(t)+ ctx := context.Background()+ require.True(t, svc.scheduleFollowUp(ctx, WikiIngestPayload{KnowledgeBaseID: "kb-1"},+ wikiFollowUpDelay, 10, 4, false))+ tasks, err := inspector.ListScheduledTasks("wiki")+ require.NoError(t, err)+ require.NoError(t, inspector.DeleteTask("wiki", tasks[0].ID))+ require.NoError(t, inspector.ArchiveTask("wiki", tasks[1].ID))+ expireFollowUpChecks(t, svc)+ require.NoError(t, svc.reconcileFollowUpTasks(ctx, "kb-1"))+ ids, err := svc.redisClient.ZRange(ctx, wikiFollowUpPrefix+"kb-1", 0, -1).Result()+ require.NoError(t, err)+ require.NotContains(t, ids, tasks[0].ID)+ require.NotContains(t, ids, tasks[1].ID)+ require.Len(t, ids, len(tasks)-2)+}++func TestFollowUpReservationsProtectEnqueueInProgress(t *testing.T) {+ svc, _ := newQueuedFollowUpTestService(t)+ ctx := context.Background()+ ids, err := svc.reserveFollowUpTasks(ctx, "kb-1", 5, 4)+ require.NoError(t, err)+ require.Len(t, ids, 5)+ // The task does not exist yet. Another completer must not recycle its+ // capacity while the first enqueue is still in flight.+ more, err := svc.reserveFollowUpTasks(ctx, "kb-1", 5, 4)+ require.NoError(t, err)+ require.Empty(t, more)+ // A producer that died before enqueueing must not consume capacity forever.+ expireFollowUpChecks(t, svc)+ more, err = svc.reserveFollowUpTasks(ctx, "kb-1", 5, 4)+ require.NoError(t, err)+ require.Len(t, more, 5)+}++// An enqueue can be committed by Redis even when its response is lost.+type lostFollowUpEnqueueResponse struct{ client *asynq.Client }++func (q lostFollowUpEnqueueResponse) Enqueue(task *asynq.Task, opts ...asynq.Option) (*asynq.TaskInfo, error) {+ if _, err := q.client.Enqueue(task, opts...); err != nil {+ return nil, err+ }+ return nil, errors.New("enqueue response lost")+}++func TestFollowUpReservationsSurviveAmbiguousEnqueueFailure(t *testing.T) {+ svc, inspector := newQueuedFollowUpTestService(t)+ client := svc.task.(*asynq.Client)+ svc.task = lostFollowUpEnqueueResponse{client: client}+ svc.scheduleFollowUp(context.Background(), WikiIngestPayload{KnowledgeBaseID: "kb-1"},+ wikiFollowUpDelay, 10, 4, false)+ svc.task = client+ expireFollowUpChecks(t, svc)+ require.NoError(t, svc.reconcileFollowUpTasks(context.Background(), "kb-1"))+ require.Equal(t, int64(5), svc.redisClient.ZCard(context.Background(), wikiFollowUpPrefix+"kb-1").Val())+ info, err := inspector.GetQueueInfo("wiki")+ require.NoError(t, err)+ require.Equal(t, 6, info.Scheduled) // five real tasks and the recovery probe+}++func TestFollowUpReservationHandlerLifecycle(t *testing.T) {+ for _, fail := range []bool{false, true} {+ t.Run(fmt.Sprintf("retry=%t", fail), func(t *testing.T) {+ svc, inspector := newQueuedFollowUpTestService(t)+ kbService := &wikiGuardKBService{}+ if fail {+ kbService.err = errors.New("temporary KB lookup failure")+ }+ svc.kbService = kbService+ ctx := context.Background()+ ids, err := svc.reserveFollowUpTasks(ctx, "kb-1", 1, 1)+ require.NoError(t, err)+ require.Len(t, ids, 1)+ server := asynq.NewServer(asynq.RedisClientOpt{Addr: svc.redisClient.Options().Addr}, asynq.Config{+ Concurrency: 1, Queues: map[string]int{"wiki": 1},+ TaskCheckInterval: 10 * time.Millisecond, ShutdownTimeout: time.Second,+ RetryDelayFunc: func(int, error, *asynq.Task) time.Duration { return time.Hour },+ LogLevel: asynq.ErrorLevel,+ })+ require.NoError(t, server.Start(asynq.HandlerFunc(svc.ProcessWikiIngest)))+ t.Cleanup(server.Shutdown)+ payload, err := json.Marshal(WikiIngestPayload{KnowledgeBaseID: "kb-1"})+ require.NoError(t, err)+ _, err = svc.task.Enqueue(asynq.NewTask("wiki:ingest", payload,+ asynq.Queue("wiki"), asynq.TaskID(ids[0])))+ require.NoError(t, err)+ if fail {+ require.Eventually(t, func() bool {+ info, err := inspector.GetTaskInfo("wiki", ids[0])+ return err == nil && info.State == asynq.TaskStateRetry+ }, 5*time.Second, 10*time.Millisecond)+ expireFollowUpChecks(t, svc)+ require.NoError(t, svc.reconcileFollowUpTasks(ctx, "kb-1"))+ require.Equal(t, int64(1), svc.redisClient.ZCard(ctx, wikiFollowUpPrefix+"kb-1").Val())+ } else {+ require.Eventually(t, func() bool {+ return svc.redisClient.ZCard(ctx, wikiFollowUpPrefix+"kb-1").Val() == 0+ }, 5*time.Second, 10*time.Millisecond, "successful handler must release its reservation")+ }+ })+ }+}++func TestFollowUpRecheckCanRearmItself(t *testing.T) {+ svc, inspector := newQueuedFollowUpTestService(t)+ server := asynq.NewServer(asynq.RedisClientOpt{Addr: svc.redisClient.Options().Addr}, asynq.Config{+ Concurrency: 1, Queues: map[string]int{"wiki": 1},+ TaskCheckInterval: 10 * time.Millisecond, ShutdownTimeout: time.Second,+ LogLevel: asynq.ErrorLevel,+ })+ require.NoError(t, server.Start(asynq.HandlerFunc(func(ctx context.Context, _ *asynq.Task) error {+ if !svc.scheduleFollowUpRecheck(ctx, WikiIngestPayload{KnowledgeBaseID: "kb-1"}) {+ return errors.New("could not re-arm probe")+ }+ return nil+ })))+ t.Cleanup(server.Shutdown)+ _, err := svc.task.Enqueue(asynq.NewTask("wiki:ingest", nil,+ asynq.Queue("wiki"), asynq.TaskID("wiki-followup-recheck-kb-1")))+ require.NoError(t, err)+ require.Eventually(t, func() bool {+ info, err := inspector.GetTaskInfo("wiki", "wiki-followup-recheck-kb-1-next")+ return err == nil && info.State == asynq.TaskStateScheduled+ }, 5*time.Second, 10*time.Millisecond)+}++func TestFollowUpReservationsKeepCapacitySeparateForEachKB(t *testing.T) {+ svc, _ := newQueuedFollowUpTestService(t)+ ctx := context.Background()+ first, err := svc.reserveFollowUpTasks(ctx, "kb-1", 5, 4)+ require.NoError(t, err)+ require.Len(t, first, 5)+ second, err := svc.reserveFollowUpTasks(ctx, "kb-2", 5, 4)+ require.NoError(t, err)+ require.Len(t, second, 5)+ more, err := svc.reserveFollowUpTasks(ctx, "kb-1", 5, 4)+ require.NoError(t, err)+ require.Empty(t, more)+}++func TestScheduleFollowUpUsesRepositoryBatchLimit(t *testing.T) {+ svc, inspector := newQueuedFollowUpTestService(t)+ svc.pendingRepo.(*wikiPendingRepoForFollowUpTest).pending = 1500+ reserveSlots(t, svc, "kb-1", 1, 4)+ require.True(t, svc.scheduleFollowUp(context.Background(), WikiIngestPayload{KnowledgeBaseID: "kb-1"},+ wikiFollowUpDelay, 2000, 4, false))+ info, err := inspector.GetQueueInfo("wiki")+ require.NoError(t, err)+ require.Equal(t, 2, info.Scheduled, "ClaimBatch can claim at most 1000 documents per batch")}diff --git a/internal/types/interfaces/task_queue.go b/internal/types/interfaces/task_queue.goindex e6d8dfd75..c50eb324d 100644--- a/internal/types/interfaces/task_queue.go+++ b/internal/types/interfaces/task_queue.go@@ -86,6 +86,13 @@ type TaskPendingOpsRepository interface {DeleteByDedupKey(ctx context.Context, taskType, scope, scopeID, dedupKey, op string) error}+// TaskPendingOpsClaimableCounter counts the distinct keys ClaimBatch can+// claim, excluding every key with a fresh claim (including late siblings).+// PendingCount remains the count of all rows for crash-recovery consumers.+type TaskPendingOpsClaimableCounter interface {+ ClaimableCount(ctx context.Context, taskType, scope, scopeID string, staleBefore time.Time) (int64, error)+}+// TaskPendingOpsScopeCleaner is an optional extension for callers that need// to discard every durable pending operation owned by a deleted scope. It is// intentionally separate from TaskPendingOpsRepository so alternate queue