Skip to content

Open-source LLM knowledge platform: turn raw documents into a queryable RAG, an autonomous reasoning agent, and a self-maintaining Wiki.

fix(wiki): bound outstanding follow-up tasks

wizardchenc819abd9589c
diff --git a/internal/application/repository/task_queue.go b/internal/application/repository/task_queue.go
index f35b72128..9df8fd76d 100644
--- a/internal/application/repository/task_queue.go
+++ b/internal/application/repository/task_queue.go
@@ -480,6 +480,25 @@ func (r *taskPendingOpsRepository) PendingCount(
return n, nil
}
+// ClaimableCount uses ClaimBatch's eligibility and grouping rules so running
+// documents and multiple queued operations for one document do not inflate
+// the number of batches a scheduler needs to dispatch.
+func (r *taskPendingOpsRepository) ClaimableCount(
+ ctx context.Context, taskType, scope, scopeID string, staleBefore time.Time,
+) (int64, error) {
+ freshKeys := r.db.WithContext(ctx).Model(&types.TaskPendingOp{}).
+ Select("dedup_key").
+ Where("task_type = ? AND scope = ? AND scope_id = ?", taskType, scope, scopeID).
+ Where("claimed_at IS NOT NULL AND claimed_at >= ?", staleBefore)
+ var n int64
+ err := r.db.WithContext(ctx).Model(&types.TaskPendingOp{}).
+ Where("task_type = ? AND scope = ? AND scope_id = ?", taskType, scope, scopeID).
+ Where("claimed_at IS NULL OR claimed_at < ?", staleBefore).
+ Where("dedup_key NOT IN (?)", freshKeys).
+ Distinct("dedup_key").Count(&n).Error
+ return n, err
+}
+
// DeleteByDedupKey drops rows in the tuple whose dedup_key matches.
// If `op` is non-empty, only rows with the matching op are dropped;
// otherwise every matching row is removed. Empty dedup_key is rejected
diff --git a/internal/application/repository/task_queue_test.go b/internal/application/repository/task_queue_test.go
index dc8fe547a..08aaafd9f 100644
--- a/internal/application/repository/task_queue_test.go
+++ b/internal/application/repository/task_queue_test.go
@@ -1037,3 +1037,44 @@ func TestTaskPendingOps_DrainUnclaimedAndReleaseRollsBackOnReleaseFailure(t *tes
require.NoError(t, db.Model(&types.TaskPendingOp{}).Count(&count).Error)
assert.Equal(t, int64(1), count)
}
+
+func TestTaskPendingOps_ClaimableCountMatchesClaimBatch(t *testing.T) {
+ db := setupTaskQueueTestDB(t)
+ repo := NewTaskPendingOpsRepository(db)
+ counter := repo.(interfaces.TaskPendingOpsClaimableCounter)
+ ctx := context.Background()
+ staleBefore := time.Now().Add(-time.Hour)
+ for _, key := range []string{"ready", "ready", "stale", "stale", "busy", "busy"} {
+ require.NoError(t, repo.Enqueue(ctx,
+ makePendingOp("wiki:ingest", "knowledge_base", "kb-1", "ingest", key, nil)))
+ }
+ require.NoError(t, db.Model(&types.TaskPendingOp{}).Where("dedup_key = ?", "stale").
+ Update("claimed_at", staleBefore.Add(-time.Second)).Error)
+ require.NoError(t, db.Model(&types.TaskPendingOp{}).Where("dedup_key = ?", "busy").
+ Update("claimed_at", time.Now()).Error)
+ // An unclaimed late sibling of a busy key is not eligible on its own.
+ require.NoError(t, repo.Enqueue(ctx,
+ makePendingOp("wiki:ingest", "knowledge_base", "kb-1", "retract", "busy", nil)))
+ // Identical keys in another tuple must not exclude the ready document.
+ other := makePendingOp("wiki:ingest", "knowledge_base", "other-kb", "ingest", "ready", nil)
+ require.NoError(t, repo.Enqueue(ctx, other))
+ require.NoError(t, db.Model(&types.TaskPendingOp{}).Where("scope_id = ?", "other-kb").
+ Update("claimed_at", time.Now()).Error)
+ n, err := counter.ClaimableCount(ctx, "wiki:ingest", "knowledge_base", "kb-1", staleBefore)
+ require.NoError(t, err)
+ require.Equal(t, int64(2), n)
+ rows, err := repo.ClaimBatch(ctx, "wiki:ingest", "knowledge_base", "kb-1", 2, staleBefore)
+ require.NoError(t, err)
+ require.Len(t, rows, 4)
+ keys := map[string]bool{}
+ for _, row := range rows {
+ keys[row.DedupKey] = true
+ }
+ require.Len(t, keys, int(n))
+ n, err = counter.ClaimableCount(ctx, "wiki:ingest", "knowledge_base", "kb-1", staleBefore)
+ require.NoError(t, err)
+ require.Zero(t, n)
+ pending, err := repo.PendingCount(ctx, "wiki:ingest", "knowledge_base", "kb-1")
+ require.NoError(t, err)
+ require.Equal(t, int64(7), pending, "crash recovery must still see claimed rows")
+}
diff --git a/internal/application/service/wiki_ingest_batch.go b/internal/application/service/wiki_ingest_batch.go
index 796f7c0a0..e9c743971 100644
--- a/internal/application/service/wiki_ingest_batch.go
+++ b/internal/application/service/wiki_ingest_batch.go
@@ -18,26 +18,16 @@ import (
"github.com/Tencent/WeKnora/internal/models/chat"
"github.com/Tencent/WeKnora/internal/tracing/langfuse"
"github.com/Tencent/WeKnora/internal/types"
+ "github.com/Tencent/WeKnora/internal/types/interfaces"
"github.com/google/uuid"
"github.com/hibiken/asynq"
"golang.org/x/sync/errgroup"
)
-// followUpTriggerCount decides how many follow-up triggers a completed batch
-// should re-arm for its KB (#3461). Since Phase 3 the consumption side
-// (disjoint claiming, per-slug locks, the inflight cap) fully supports
-// concurrent batches, but nothing created more than one trigger for a KB with
-// a standing backlog — so once uploads stopped, the backlog drained at
-// 1 batch × batch-size per cycle regardless of ingest_max_inflight, leaving
-// the wiki pool and the model server idle.
-//
-// The count is the smaller of (a) how many batches the remaining rows still
-// need and (b) how many slots could be free by the time the triggers fire:
-// the caller's own slot releases right after it returns, hence the +1.
-// activeSlots < 0 means "unknown" (no Redis / Redis error) and maxInflight
-// <= 1 cannot host more than one batch anyway; both collapse to 1, matching
-// the pre-fan-out behaviour. Over-issued triggers are harmless: the ones that
-// cannot get a slot are coalesced by scheduleCappedRetry's fixed TaskID.
+// followUpTriggerCount estimates useful new work from claimable documents and
+// running slots. reserveFollowUpTasks additionally bounds all outstanding
+// follow-ups (scheduled, queued, running and retrying) across batch completions.
+// The +1 allows the completing batch to replace itself before releasing its slot.
func followUpTriggerCount(pending, batchSize, activeSlots, maxInflight int) int {
if pending <= 0 || batchSize <= 0 {
return 0
@@ -53,31 +43,9 @@ func followUpTriggerCount(pending, batchSize, activeSlots, maxInflight int) int
return min(batchesNeeded, freeSlots)
}
-// scheduleFollowUp enqueues asynq trigger tasks if there are still pending
-// ops in task_pending_ops for this KB. Returns true when at least one
-// follow-up was scheduled.
-//
-// Post-Phase-3 this backstops the case where a batch drained its claimed
-// window but more rows remain and no other trigger is pending (e.g. a steady
-// trickle of uploads). Standard mode already fans a KB's backlog across
-// concurrent claiming batches, so the short delay is normally just a light
-// debounce rather than a lock-release wait.
-//
-// How many triggers are re-armed follows followUpTriggerCount above: enough
-// to drain the backlog, capped by the free inflight slots, so the concurrent
-// batch count converges on ingest_max_inflight instead of idling at 1.
-//
-// Fan-out is deliberately skipped when rateLimited: widening concurrency
-// during a 429 storm only amplifies the backoff. Lite mode (no Redis) also
-// never fans out — liteLocks serializes batches per KB, so extra triggers
-// would only bounce off ErrWikiIngestConcurrent.
-//
-// `delay` is the ProcessIn before each follow-up fires. Callers pass
-// wikiFollowUpDelay for the normal case and wikiRateLimitBackoff when the
-// batch tripped an upstream rate limit — the released failed rows are
-// eligible immediately, but nothing claims them until a trigger fires, so
-// stretching the follow-up interval is what actually paces retries down
-// during a 429 storm.
+// scheduleFollowUp replenishes the KB's bounded supply of follow-up tasks.
+// Lite mode and rate-limited completions request only one successor. A 429
+// delays this batch's successor; it does not pause other batches in the KB.
func (s *wikiIngestService) scheduleFollowUp(
ctx context.Context,
payload WikiIngestPayload,
@@ -93,38 +61,74 @@ func (s *wikiIngestService) scheduleFollowUp(
return false
}
+ if s.redisClient != nil {
+ if counter, ok := s.pendingRepo.(interfaces.TaskPendingOpsClaimableCounter); ok {
+ count, err = counter.ClaimableCount(ctx, wikiTaskType, wikiTaskScope,
+ payload.KnowledgeBaseID, time.Now().Add(-wikiClaimStaleAfter))
+ if err != nil {
+ logger.Warnf(ctx, "wiki ingest: claimable count failed: %v", err)
+ return s.scheduleFollowUpRecheck(ctx, payload)
+ }
+ if count == 0 {
+ return s.scheduleStaleClaimRecheck(ctx, payload)
+ }
+ }
+ // ClaimBatch caps the number of distinct keys at 1000.
+ batchSize = min(batchSize, 1000)
+ }
+
// activeSlots stays -1 (unknown → single follow-up) in Lite mode and on
- // a rate-limited exit; -1 also shows up in the log line as "unknown".
+ // a rate-limited exit.
activeSlots := -1
if !rateLimited {
activeSlots = s.activeInflightSlots(ctx, payload.KnowledgeBaseID)
}
n := followUpTriggerCount(int(count), batchSize, activeSlots, maxInflight)
- activeStr := fmt.Sprintf("%d", activeSlots)
- if activeSlots < 0 {
- activeStr = "unknown"
+ var taskIDs []string
+ if s.redisClient != nil {
+ taskIDs, err = s.reserveFollowUpTasks(ctx, payload.KnowledgeBaseID, n, maxInflight)
+ if err != nil {
+ logger.Warnf(ctx, "wiki ingest: follow-up reservation failed: %v", err)
+ // Keep the pre-fan-out behavior while Redis is unavailable.
+ n = 1
+ } else {
+ n = len(taskIDs)
+ if n == 0 {
+ return s.scheduleFollowUpRecheck(ctx, payload)
+ }
+ }
}
- logger.Infof(ctx,
- "wiki ingest: %d more documents pending for KB %s, scheduling %d follow-up(s) in %s (active slots %s/%d)",
- count, payload.KnowledgeBaseID, n, delay, activeStr, maxInflight)
+ logger.Infof(ctx, "wiki ingest: %d claimable documents for KB %s, scheduling %d follow-up(s) in %s",
+ count, payload.KnowledgeBaseID, n, delay)
langfuse.InjectTracing(ctx, &payload)
payloadBytes, _ := json.Marshal(payload)
scheduled := false
+ needsRecheck := false
for i := 0; i < n; i++ {
- t := asynq.NewTask(types.TypeWikiIngest, payloadBytes,
+ opts := []asynq.Option{
asynq.Queue(types.QueueWiki),
asynq.MaxRetry(wikiIngestMaxRetry),
- asynq.Timeout(60*time.Minute),
- asynq.ProcessIn(delay), // debounce (or rate-limit backoff) before draining the remainder
- )
+ asynq.Timeout(60 * time.Minute),
+ asynq.ProcessIn(delay),
+ }
+ if len(taskIDs) > 0 {
+ opts = append(opts, asynq.TaskID(taskIDs[i]))
+ }
+ t := asynq.NewTask(types.TypeWikiIngest, payloadBytes, opts...)
if _, err := s.task.Enqueue(t); err != nil {
logger.Warnf(ctx, "wiki ingest: follow-up enqueue %d/%d failed: %v", i+1, n, err)
+ // A timeout may mean the enqueue succeeded. Keep its reservation
+ // until reconciliation can establish the actual queue state.
+ needsRecheck = true
continue
}
scheduled = true
}
+ if needsRecheck {
+ scheduled = s.scheduleFollowUpRecheck(ctx, payload) || scheduled
+ }
return scheduled
}
@@ -253,7 +257,7 @@ func (s *wikiIngestService) newWikiBatchContext(
}
}
-func (s *wikiIngestService) ProcessWikiIngest(ctx context.Context, t *asynq.Task) error {
+func (s *wikiIngestService) ProcessWikiIngest(ctx context.Context, t *asynq.Task) (retErr error) {
taskStartedAt := time.Now()
retryCount, _ := asynq.GetRetryCount(ctx)
maxRetry, _ := asynq.GetMaxRetry(ctx)
@@ -312,6 +316,16 @@ func (s *wikiIngestService) ProcessWikiIngest(ctx context.Context, t *asynq.Task
return fmt.Errorf("wiki ingest: unmarshal payload: %w", err)
}
+ defer func() {
+ if p := recover(); p != nil {
+ // Asynq retries panics too: retain the outstanding reservation.
+ panic(p)
+ }
+ if retErr == nil {
+ s.releaseFollowUpTask(ctx, payload.KnowledgeBaseID)
+ }
+ }()
+
// Inject context
ctx = context.WithValue(ctx, types.TenantIDContextKey, payload.TenantID)
if payload.Language != "" {
diff --git a/internal/application/service/wiki_ingest_followup.go b/internal/application/service/wiki_ingest_followup.go
new file mode 100644
index 000000000..3bcd96752
--- /dev/null
+++ b/internal/application/service/wiki_ingest_followup.go
@@ -0,0 +1,152 @@
+package service
+
+import (
+ "context"
+ "encoding/json"
+ "errors"
+ "strconv"
+ "time"
+
+ "github.com/Tencent/WeKnora/internal/logger"
+ "github.com/Tencent/WeKnora/internal/tracing/langfuse"
+ "github.com/Tencent/WeKnora/internal/types"
+ "github.com/google/uuid"
+ "github.com/hibiken/asynq"
+ "github.com/redis/go-redis/v9"
+)
+
+const (
+ wikiFollowUpPrefix = "wiki:followup:"
+ // This is a reconciliation interval, not a queue lease. A task can wait
+ // arbitrarily long for a worker without losing its reserved capacity.
+ // Missing tasks are retained for this interval to allow an enqueue in
+ // progress to finish, including when its caller times out.
+ wikiFollowUpReconcileAfter = 2 * time.Minute
+)
+
+// Reservations cover the entire asynq task lifetime, including delayed tasks
+// and retries. Keep one spare beyond the running cap so the last completing
+// task can enqueue its successor before its own reservation is released.
+// Atomic ZCARD + ZADD makes concurrent completions share the same budget.
+const wikiFollowUpReserveScript = `
+local available = math.max(0, tonumber(ARGV[2]) - redis.call('ZCARD', KEYS[1]))
+local n = math.min(tonumber(ARGV[1]), available)
+local ids = {}
+for i = 1, n do
+ local id = ARGV[3] .. ':' .. i
+ redis.call('ZADD', KEYS[1], ARGV[4], id)
+ ids[#ids + 1] = id
+end
+return ids
+`
+
+// Do not expire a reservation just because the task has waited a long time.
+// Inspect asynq first, then remove/refresh only if another reconciler hasn't
+// already updated it. ZADD XX also avoids resurrecting a completed task.
+const wikiFollowUpReconcileScript = `
+local score = redis.call('ZSCORE', KEYS[1], ARGV[1])
+if score and tonumber(score) == tonumber(ARGV[2]) then
+ if ARGV[3] == 'remove' then
+ redis.call('ZREM', KEYS[1], ARGV[1])
+ else
+ redis.call('ZADD', KEYS[1], 'XX', ARGV[3], ARGV[1])
+ end
+end
+return 1
+`
+
+func (s *wikiIngestService) reserveFollowUpTasks(
+ ctx context.Context, kbID string, requested, maxInflight int,
+) ([]string, error) {
+ if requested <= 0 {
+ return nil, nil
+ }
+ if err := s.reconcileFollowUpTasks(ctx, kbID); err != nil {
+ return nil, err
+ }
+ return s.redisClient.Eval(ctx, wikiFollowUpReserveScript,
+ []string{wikiFollowUpPrefix + kbID}, requested, max(1, maxInflight)+1,
+ "wiki-followup-"+uuid.NewString(), time.Now().Add(wikiFollowUpReconcileAfter).UnixMilli(),
+ ).StringSlice()
+}
+
+func (s *wikiIngestService) reconcileFollowUpTasks(ctx context.Context, kbID string) error {
+ key := wikiFollowUpPrefix + kbID
+ now := time.Now()
+ entries, err := s.redisClient.ZRangeByScoreWithScores(ctx, key, &redis.ZRangeBy{
+ Min: "-inf", Max: strconv.FormatInt(now.UnixMilli(), 10),
+ }).Result()
+ if err != nil || len(entries) == 0 {
+ return err
+ }
+ // Shares the service's Redis connection; Inspector.Close must not close it.
+ inspector := asynq.NewInspectorFromRedisClient(s.redisClient)
+ for _, entry := range entries {
+ if err := ctx.Err(); err != nil {
+ return err
+ }
+ id, ok := entry.Member.(string)
+ if !ok {
+ continue
+ }
+ info, err := inspector.GetTaskInfo(types.QueueWiki, id)
+ missing := errors.Is(err, asynq.ErrTaskNotFound) || errors.Is(err, asynq.ErrQueueNotFound)
+ if err != nil && !missing {
+ return err
+ }
+ action := strconv.FormatInt(now.Add(wikiFollowUpReconcileAfter).UnixMilli(), 10)
+ if missing || info.State == asynq.TaskStateArchived || info.State == asynq.TaskStateCompleted {
+ action = "remove"
+ }
+ if err := s.redisClient.Eval(ctx, wikiFollowUpReconcileScript,
+ []string{key}, id, entry.Score, action).Err(); err != nil {
+ return err
+ }
+ }
+ return nil
+}
+
+func (s *wikiIngestService) releaseFollowUpTask(ctx context.Context, kbID string) {
+ if s.redisClient == nil {
+ return
+ }
+ id, ok := asynq.GetTaskID(ctx)
+ if !ok {
+ return
+ }
+ cleanupCtx, cancel := wikiIngestCleanupContext(ctx)
+ defer cancel()
+ if err := s.redisClient.ZRem(cleanupCtx, wikiFollowUpPrefix+kbID, id).Err(); err != nil {
+ logger.Warnf(ctx, "wiki ingest: follow-up reservation release failed: %v", err)
+ }
+}
+
+// A coalesced probe keeps progress possible when all capacity is reserved but
+// an enqueue failed or a worker died. It also revisits reservations whose
+// cleanup failed. It is outside the reservation budget and contributes at
+// most two probes per KB, independent of the backlog size.
+func (s *wikiIngestService) scheduleFollowUpRecheck(ctx context.Context, payload WikiIngestPayload) bool {
+ langfuse.InjectTracing(ctx, &payload)
+ b, _ := json.Marshal(payload)
+ id := "wiki-followup-recheck-" + payload.KnowledgeBaseID
+ if current, _ := asynq.GetTaskID(ctx); current == id {
+ // The current task ID remains occupied until asynq acknowledges it.
+ // Alternate between two IDs so the probe can re-arm itself.
+ id += "-next"
+ }
+ t := asynq.NewTask(types.TypeWikiIngest, b,
+ asynq.Queue(types.QueueWiki),
+ asynq.MaxRetry(wikiIngestMaxRetry),
+ asynq.Timeout(60*time.Minute),
+ asynq.ProcessIn(wikiFollowUpReconcileAfter+wikiFollowUpDelay),
+ asynq.TaskID(id),
+ )
+ if _, err := s.task.Enqueue(t); err != nil {
+ if errors.Is(err, asynq.ErrTaskIDConflict) || errors.Is(err, asynq.ErrDuplicateTask) {
+ return true
+ }
+ logger.Warnf(ctx, "wiki ingest: follow-up recheck enqueue failed: %v", err)
+ return false
+ }
+ return true
+}
diff --git a/internal/application/service/wiki_ingest_followup_test.go b/internal/application/service/wiki_ingest_followup_test.go
index 2536661e3..6657edabb 100644
--- a/internal/application/service/wiki_ingest_followup_test.go
+++ b/internal/application/service/wiki_ingest_followup_test.go
@@ -4,6 +4,7 @@ import (
"context"
"encoding/json"
"errors"
+ "fmt"
"sync"
"testing"
"time"
@@ -17,10 +18,12 @@ import (
// --- test doubles -----------------------------------------------------------
// wikiPendingRepoForFollowUpTest reuses the cleanup-test repo and makes
-// PendingCount configurable, which is the only input scheduleFollowUp reads.
+// PendingCount and the eligible-document count configurable.
type wikiPendingRepoForFollowUpTest struct {
wikiPendingRepoForCleanupTest
- pending int64
+ pending int64
+ claimable *int64
+ claimableErr error
}
func (r *wikiPendingRepoForFollowUpTest) PendingCount(
@@ -29,6 +32,15 @@ func (r *wikiPendingRepoForFollowUpTest) PendingCount(
return r.pending, nil
}
+func (r *wikiPendingRepoForFollowUpTest) ClaimableCount(
+ context.Context, string, string, string, time.Time,
+) (int64, error) {
+ if r.claimable != nil {
+ return *r.claimable, r.claimableErr
+ }
+ return r.pending, r.claimableErr
+}
+
// followUpEnqueuerRecorder captures every trigger scheduleFollowUp enqueues.
// failFrom >= 0 makes enqueues from that index onward return failErr so the
// partial-failure path can be exercised.
@@ -248,47 +260,274 @@ func TestScheduleFollowUpPartialEnqueueFailureStillReportsScheduled(t *testing.T
got := svc.scheduleFollowUp(context.Background(),
WikiIngestPayload{KnowledgeBaseID: "kb-1"}, wikiFollowUpDelay, 10, 4, false)
require.True(t, got)
- require.Equal(t, 4, rec.count())
-}
-
-// --- convergence ------------------------------------------------------------
-
-// TestFollowUpFanOutConvergesUnderCap simulates successive batch completions
-// fanning out on a board whose slots never free (worst case for over-issue)
-// and asserts the per-completion trigger count never exceeds the free slots,
-// i.e. the supply side alone cannot push concurrency past ingest_max_inflight.
-func TestFollowUpFanOutConvergesUnderCap(t *testing.T) {
- const (
- maxInflight = 8
- batchSize = 10
- )
- var (
- pending = 1000
- activeSlots = 1 // the completing batch itself; releases on return
- )
- seen := map[int]bool{}
- for round := 0; round < 6; round++ {
- n := followUpTriggerCount(pending, batchSize, activeSlots, maxInflight)
- require.Greater(t, n, 0)
- require.LessOrEqual(t, n, maxInflight-activeSlots+1,
- "round %d: supply must not exceed free slots", round)
- if activeSlots == maxInflight {
- require.Equal(t, 1, n,
- "round %d: a full board must clamp supply to the follow-up chain", round)
- }
- seen[n] = true
-
- // The n triggers fire: n-1 of them claim slots (one bounces off the
- // cap into scheduleCappedRetry when the board is full), each claims a
- // batch of batchSize rows.
- newBatches := min(n, maxInflight-activeSlots)
- pending -= newBatches * batchSize
- activeSlots = maxInflight
- if pending <= 0 {
- pending = 1 // keep the loop meaningful; count stays capped below
- }
+ require.Equal(t, 5, rec.count()) // four follow-ups and one recovery probe
+}
+
+// Use the actual asynq client so these tests exercise atomic reservations and
+// queue state, rather than merely checking the single-completer arithmetic.
+func newQueuedFollowUpTestService(t *testing.T) (*wikiIngestService, *asynq.Inspector) {
+ t.Helper()
+ mr := miniredis.RunT(t)
+ rdb := redis.NewClient(&redis.Options{Addr: mr.Addr()})
+ t.Cleanup(func() { _ = rdb.Close() })
+ client := asynq.NewClient(asynq.RedisClientOpt{Addr: mr.Addr()})
+ t.Cleanup(func() { _ = client.Close() })
+ return &wikiIngestService{
+ redisClient: rdb,
+ task: client,
+ pendingRepo: &wikiPendingRepoForFollowUpTest{pending: 24624},
+ }, asynq.NewInspectorFromRedisClient(rdb)
+}
+
+func expireFollowUpChecks(t *testing.T, svc *wikiIngestService) {
+ t.Helper()
+ ctx := context.Background()
+ ids, err := svc.redisClient.ZRange(ctx, wikiFollowUpPrefix+"kb-1", 0, -1).Result()
+ require.NoError(t, err)
+ for _, id := range ids {
+ require.NoError(t, svc.redisClient.ZAdd(ctx, wikiFollowUpPrefix+"kb-1", redis.Z{
+ Score: float64(time.Now().Add(-time.Minute).UnixMilli()), Member: id,
+ }).Err())
+ }
+}
+
+func TestScheduleFollowUpWorkerPoolBelowCap(t *testing.T) {
+ svc, inspector := newQueuedFollowUpTestService(t)
+ ctx := context.Background()
+ payload := WikiIngestPayload{KnowledgeBaseID: "kb-1"}
+ // Startup recovery has one running batch, whose completion fans out.
+ release, ok := svc.reserveInflightSlot(ctx, "kb-1", 32)
+ require.True(t, ok)
+ require.True(t, svc.scheduleFollowUp(ctx, payload, wikiFollowUpDelay, 10, 32, false))
+ release()
+ queued, err := inspector.ListScheduledTasks("wiki", asynq.PageSize(100))
+ require.NoError(t, err)
+ require.Len(t, queued, 32)
+
+ // Model eight workers dequeuing tasks. Keep their follow-up reservations
+ // until completion, just as the handler does, and consume one queued task
+ // for each finished batch. No model, database or external Redis is used.
+ runningIDs := make([]string, 8)
+ for i := range runningIDs {
+ runningIDs[i] = queued[i].ID
+ require.NoError(t, inspector.DeleteTask("wiki", queued[i].ID))
+ }
+ reserveSlots(t, svc, "kb-1", 8, 32)
+ for completed := 0; completed < 100; completed++ {
+ svc.pendingRepo.(*wikiPendingRepoForFollowUpTest).pending -= 10
+ require.True(t, svc.scheduleFollowUp(ctx, payload, wikiFollowUpDelay, 10, 32, false))
+ worker := completed % len(runningIDs)
+ require.NoError(t, svc.redisClient.ZRem(ctx, wikiFollowUpPrefix+"kb-1", runningIDs[worker]).Err())
+ queued, err = inspector.ListScheduledTasks("wiki", asynq.PageSize(100))
+ require.NoError(t, err)
+ require.NotEmpty(t, queued)
+ runningIDs[worker] = queued[0].ID
+ require.NoError(t, inspector.DeleteTask("wiki", queued[0].ID))
+ info, err := inspector.GetQueueInfo("wiki")
+ require.NoError(t, err)
+ require.LessOrEqual(t, info.Scheduled, 25, "completion %d must not multiply the queue", completed)
}
- // The first round must have fanned out beyond the follow-up chain —
- // otherwise this would not be testing convergence at all.
- require.Greater(t, len(seen), 1)
+}
+
+func TestScheduleFollowUpConcurrentCompletionsShareBudget(t *testing.T) {
+ svc, inspector := newQueuedFollowUpTestService(t)
+ reserveSlots(t, svc, "kb-1", 16, 32)
+ start := make(chan struct{})
+ var wg sync.WaitGroup
+ for i := 0; i < 16; i++ {
+ wg.Add(1)
+ go func() {
+ defer wg.Done()
+ <-start
+ svc.scheduleFollowUp(context.Background(), WikiIngestPayload{KnowledgeBaseID: "kb-1"},
+ wikiFollowUpDelay, 10, 32, false)
+ }()
+ }
+ close(start)
+ wg.Wait()
+ info, err := inspector.GetQueueInfo("wiki")
+ require.NoError(t, err)
+ // Cap + one successor spare + at most one coalesced recovery probe.
+ require.LessOrEqual(t, info.Scheduled, 34)
+ require.Equal(t, int64(33), svc.redisClient.ZCard(context.Background(), wikiFollowUpPrefix+"kb-1").Val())
+}
+
+func TestScheduleFollowUpClaimedRowsOnlyKeepsCrashRecovery(t *testing.T) {
+ svc, inspector := newQueuedFollowUpTestService(t)
+ zero := int64(0)
+ svc.pendingRepo.(*wikiPendingRepoForFollowUpTest).claimable = &zero
+ require.True(t, svc.scheduleFollowUp(context.Background(), WikiIngestPayload{KnowledgeBaseID: "kb-1"},
+ wikiFollowUpDelay, 10, 32, false))
+ tasks, err := inspector.ListScheduledTasks("wiki")
+ require.NoError(t, err)
+ require.Len(t, tasks, 1)
+ require.Equal(t, "wiki-ingest-recheck-kb-1", tasks[0].ID)
+ require.Greater(t, time.Until(tasks[0].NextProcessAt), wikiClaimStaleAfter-time.Second)
+}
+
+func TestFollowUpReservationsKeepLongQueuedTasks(t *testing.T) {
+ svc, inspector := newQueuedFollowUpTestService(t)
+ ctx := context.Background()
+ payload := WikiIngestPayload{KnowledgeBaseID: "kb-1"}
+ require.True(t, svc.scheduleFollowUp(ctx, payload, wikiFollowUpDelay, 10, 4, false))
+ for i := 0; i < 5; i++ {
+ expireFollowUpChecks(t, svc)
+ require.True(t, svc.scheduleFollowUp(ctx, payload, wikiFollowUpDelay, 10, 4, false))
+ }
+ info, err := inspector.GetQueueInfo("wiki")
+ require.NoError(t, err)
+ require.LessOrEqual(t, info.Scheduled, 6) // five reserved tasks plus one probe
+ require.Equal(t, int64(5), svc.redisClient.ZCard(ctx, wikiFollowUpPrefix+"kb-1").Val())
+}
+
+func TestFollowUpReservationsRecoverMissingAndArchivedTasks(t *testing.T) {
+ svc, inspector := newQueuedFollowUpTestService(t)
+ ctx := context.Background()
+ require.True(t, svc.scheduleFollowUp(ctx, WikiIngestPayload{KnowledgeBaseID: "kb-1"},
+ wikiFollowUpDelay, 10, 4, false))
+ tasks, err := inspector.ListScheduledTasks("wiki")
+ require.NoError(t, err)
+ require.NoError(t, inspector.DeleteTask("wiki", tasks[0].ID))
+ require.NoError(t, inspector.ArchiveTask("wiki", tasks[1].ID))
+ expireFollowUpChecks(t, svc)
+ require.NoError(t, svc.reconcileFollowUpTasks(ctx, "kb-1"))
+ ids, err := svc.redisClient.ZRange(ctx, wikiFollowUpPrefix+"kb-1", 0, -1).Result()
+ require.NoError(t, err)
+ require.NotContains(t, ids, tasks[0].ID)
+ require.NotContains(t, ids, tasks[1].ID)
+ require.Len(t, ids, len(tasks)-2)
+}
+
+func TestFollowUpReservationsProtectEnqueueInProgress(t *testing.T) {
+ svc, _ := newQueuedFollowUpTestService(t)
+ ctx := context.Background()
+ ids, err := svc.reserveFollowUpTasks(ctx, "kb-1", 5, 4)
+ require.NoError(t, err)
+ require.Len(t, ids, 5)
+ // The task does not exist yet. Another completer must not recycle its
+ // capacity while the first enqueue is still in flight.
+ more, err := svc.reserveFollowUpTasks(ctx, "kb-1", 5, 4)
+ require.NoError(t, err)
+ require.Empty(t, more)
+ // A producer that died before enqueueing must not consume capacity forever.
+ expireFollowUpChecks(t, svc)
+ more, err = svc.reserveFollowUpTasks(ctx, "kb-1", 5, 4)
+ require.NoError(t, err)
+ require.Len(t, more, 5)
+}
+
+// An enqueue can be committed by Redis even when its response is lost.
+type lostFollowUpEnqueueResponse struct{ client *asynq.Client }
+
+func (q lostFollowUpEnqueueResponse) Enqueue(task *asynq.Task, opts ...asynq.Option) (*asynq.TaskInfo, error) {
+ if _, err := q.client.Enqueue(task, opts...); err != nil {
+ return nil, err
+ }
+ return nil, errors.New("enqueue response lost")
+}
+
+func TestFollowUpReservationsSurviveAmbiguousEnqueueFailure(t *testing.T) {
+ svc, inspector := newQueuedFollowUpTestService(t)
+ client := svc.task.(*asynq.Client)
+ svc.task = lostFollowUpEnqueueResponse{client: client}
+ svc.scheduleFollowUp(context.Background(), WikiIngestPayload{KnowledgeBaseID: "kb-1"},
+ wikiFollowUpDelay, 10, 4, false)
+ svc.task = client
+ expireFollowUpChecks(t, svc)
+ require.NoError(t, svc.reconcileFollowUpTasks(context.Background(), "kb-1"))
+ require.Equal(t, int64(5), svc.redisClient.ZCard(context.Background(), wikiFollowUpPrefix+"kb-1").Val())
+ info, err := inspector.GetQueueInfo("wiki")
+ require.NoError(t, err)
+ require.Equal(t, 6, info.Scheduled) // five real tasks and the recovery probe
+}
+
+func TestFollowUpReservationHandlerLifecycle(t *testing.T) {
+ for _, fail := range []bool{false, true} {
+ t.Run(fmt.Sprintf("retry=%t", fail), func(t *testing.T) {
+ svc, inspector := newQueuedFollowUpTestService(t)
+ kbService := &wikiGuardKBService{}
+ if fail {
+ kbService.err = errors.New("temporary KB lookup failure")
+ }
+ svc.kbService = kbService
+ ctx := context.Background()
+ ids, err := svc.reserveFollowUpTasks(ctx, "kb-1", 1, 1)
+ require.NoError(t, err)
+ require.Len(t, ids, 1)
+ server := asynq.NewServer(asynq.RedisClientOpt{Addr: svc.redisClient.Options().Addr}, asynq.Config{
+ Concurrency: 1, Queues: map[string]int{"wiki": 1},
+ TaskCheckInterval: 10 * time.Millisecond, ShutdownTimeout: time.Second,
+ RetryDelayFunc: func(int, error, *asynq.Task) time.Duration { return time.Hour },
+ LogLevel: asynq.ErrorLevel,
+ })
+ require.NoError(t, server.Start(asynq.HandlerFunc(svc.ProcessWikiIngest)))
+ t.Cleanup(server.Shutdown)
+ payload, err := json.Marshal(WikiIngestPayload{KnowledgeBaseID: "kb-1"})
+ require.NoError(t, err)
+ _, err = svc.task.Enqueue(asynq.NewTask("wiki:ingest", payload,
+ asynq.Queue("wiki"), asynq.TaskID(ids[0])))
+ require.NoError(t, err)
+ if fail {
+ require.Eventually(t, func() bool {
+ info, err := inspector.GetTaskInfo("wiki", ids[0])
+ return err == nil && info.State == asynq.TaskStateRetry
+ }, 5*time.Second, 10*time.Millisecond)
+ expireFollowUpChecks(t, svc)
+ require.NoError(t, svc.reconcileFollowUpTasks(ctx, "kb-1"))
+ require.Equal(t, int64(1), svc.redisClient.ZCard(ctx, wikiFollowUpPrefix+"kb-1").Val())
+ } else {
+ require.Eventually(t, func() bool {
+ return svc.redisClient.ZCard(ctx, wikiFollowUpPrefix+"kb-1").Val() == 0
+ }, 5*time.Second, 10*time.Millisecond, "successful handler must release its reservation")
+ }
+ })
+ }
+}
+
+func TestFollowUpRecheckCanRearmItself(t *testing.T) {
+ svc, inspector := newQueuedFollowUpTestService(t)
+ server := asynq.NewServer(asynq.RedisClientOpt{Addr: svc.redisClient.Options().Addr}, asynq.Config{
+ Concurrency: 1, Queues: map[string]int{"wiki": 1},
+ TaskCheckInterval: 10 * time.Millisecond, ShutdownTimeout: time.Second,
+ LogLevel: asynq.ErrorLevel,
+ })
+ require.NoError(t, server.Start(asynq.HandlerFunc(func(ctx context.Context, _ *asynq.Task) error {
+ if !svc.scheduleFollowUpRecheck(ctx, WikiIngestPayload{KnowledgeBaseID: "kb-1"}) {
+ return errors.New("could not re-arm probe")
+ }
+ return nil
+ })))
+ t.Cleanup(server.Shutdown)
+ _, err := svc.task.Enqueue(asynq.NewTask("wiki:ingest", nil,
+ asynq.Queue("wiki"), asynq.TaskID("wiki-followup-recheck-kb-1")))
+ require.NoError(t, err)
+ require.Eventually(t, func() bool {
+ info, err := inspector.GetTaskInfo("wiki", "wiki-followup-recheck-kb-1-next")
+ return err == nil && info.State == asynq.TaskStateScheduled
+ }, 5*time.Second, 10*time.Millisecond)
+}
+
+func TestFollowUpReservationsKeepCapacitySeparateForEachKB(t *testing.T) {
+ svc, _ := newQueuedFollowUpTestService(t)
+ ctx := context.Background()
+ first, err := svc.reserveFollowUpTasks(ctx, "kb-1", 5, 4)
+ require.NoError(t, err)
+ require.Len(t, first, 5)
+ second, err := svc.reserveFollowUpTasks(ctx, "kb-2", 5, 4)
+ require.NoError(t, err)
+ require.Len(t, second, 5)
+ more, err := svc.reserveFollowUpTasks(ctx, "kb-1", 5, 4)
+ require.NoError(t, err)
+ require.Empty(t, more)
+}
+
+func TestScheduleFollowUpUsesRepositoryBatchLimit(t *testing.T) {
+ svc, inspector := newQueuedFollowUpTestService(t)
+ svc.pendingRepo.(*wikiPendingRepoForFollowUpTest).pending = 1500
+ reserveSlots(t, svc, "kb-1", 1, 4)
+ require.True(t, svc.scheduleFollowUp(context.Background(), WikiIngestPayload{KnowledgeBaseID: "kb-1"},
+ wikiFollowUpDelay, 2000, 4, false))
+ info, err := inspector.GetQueueInfo("wiki")
+ require.NoError(t, err)
+ require.Equal(t, 2, info.Scheduled, "ClaimBatch can claim at most 1000 documents per batch")
}
diff --git a/internal/types/interfaces/task_queue.go b/internal/types/interfaces/task_queue.go
index e6d8dfd75..c50eb324d 100644
--- a/internal/types/interfaces/task_queue.go
+++ b/internal/types/interfaces/task_queue.go
@@ -86,6 +86,13 @@ type TaskPendingOpsRepository interface {
DeleteByDedupKey(ctx context.Context, taskType, scope, scopeID, dedupKey, op string) error
}
+// TaskPendingOpsClaimableCounter counts the distinct keys ClaimBatch can
+// claim, excluding every key with a fresh claim (including late siblings).
+// PendingCount remains the count of all rows for crash-recovery consumers.
+type TaskPendingOpsClaimableCounter interface {
+ ClaimableCount(ctx context.Context, taskType, scope, scopeID string, staleBefore time.Time) (int64, error)
+}
+
// TaskPendingOpsScopeCleaner is an optional extension for callers that need
// to discard every durable pending operation owned by a deleted scope. It is
// intentionally separate from TaskPendingOpsRepository so alternate queue