Skip to content

Open-source LLM knowledge platform: turn raw documents into a queryable RAG, an autonomous reasoning agent, and a self-maintaining Wiki.

fix(wiki): fan out follow-up triggers so backlogs drain at ingest_max_inflight

Duke4ba85e2edf71
diff --git a/internal/application/service/wiki_ingest.go b/internal/application/service/wiki_ingest.go
index 4df7d57f6..172033025 100644
--- a/internal/application/service/wiki_ingest.go
+++ b/internal/application/service/wiki_ingest.go
@@ -1112,6 +1112,34 @@ func (s *wikiIngestService) reserveInflightSlot(ctx context.Context, kbID string
}, true
}
+// wikiInflightCountScript purges expired slots exactly like the reserve
+// script does, then returns the live count. Read-mostly companion to
+// wikiInflightReserveScript so a fan-out decision sees the same view of the
+// cap a reserver would.
+const wikiInflightCountScript = `
+redis.call('ZREMRANGEBYSCORE', KEYS[1], 0, tonumber(ARGV[1]))
+return redis.call('ZCARD', KEYS[1])
+`
+
+// activeInflightSlots reports how many batches currently hold one of KB's
+// in-flight slots (standard/Redis mode). Returns -1 when the count is
+// unavailable (no Redis client, or a Redis error); callers must treat that as
+// "unknown" and stay conservative instead of scaling on a stale number.
+func (s *wikiIngestService) activeInflightSlots(ctx context.Context, kbID string) int {
+ if s.redisClient == nil {
+ return -1
+ }
+ res, err := s.redisClient.Eval(ctx, wikiInflightCountScript,
+ []string{wikiInflightPrefix + kbID},
+ time.Now().UnixMilli(),
+ ).Int()
+ if err != nil {
+ logger.Warnf(ctx, "wiki ingest: inflight count failed for KB %s: %v", kbID, err)
+ return -1
+ }
+ return res
+}
+
// scheduleCappedRetry enqueues a single coalesced follow-up trigger after a
// batch was turned away by the in-flight cap. asynq.TaskID collapses all
// turned-away triggers for one KB into a single pending retry (no thundering
diff --git a/internal/application/service/wiki_ingest_batch.go b/internal/application/service/wiki_ingest_batch.go
index c16c17582..796f7c0a0 100644
--- a/internal/application/service/wiki_ingest_batch.go
+++ b/internal/application/service/wiki_ingest_batch.go
@@ -23,23 +23,68 @@ import (
"golang.org/x/sync/errgroup"
)
-// scheduleFollowUp enqueues another asynq trigger task if there are
-// still pending ops in task_pending_ops for this KB. Returns true when
-// a follow-up was scheduled.
+// followUpTriggerCount decides how many follow-up triggers a completed batch
+// should re-arm for its KB (#3461). Since Phase 3 the consumption side
+// (disjoint claiming, per-slug locks, the inflight cap) fully supports
+// concurrent batches, but nothing created more than one trigger for a KB with
+// a standing backlog — so once uploads stopped, the backlog drained at
+// 1 batch × batch-size per cycle regardless of ingest_max_inflight, leaving
+// the wiki pool and the model server idle.
//
-// Post-Phase-3 this only backstops the case where a batch drained its
-// claimed window but more rows remain and no other trigger is pending
-// (e.g. steady trickle of uploads). Standard mode already fans a KB's
-// backlog across concurrent claiming batches, so the short delay is
-// normally just a light debounce rather than a lock-release wait.
+// The count is the smaller of (a) how many batches the remaining rows still
+// need and (b) how many slots could be free by the time the triggers fire:
+// the caller's own slot releases right after it returns, hence the +1.
+// activeSlots < 0 means "unknown" (no Redis / Redis error) and maxInflight
+// <= 1 cannot host more than one batch anyway; both collapse to 1, matching
+// the pre-fan-out behaviour. Over-issued triggers are harmless: the ones that
+// cannot get a slot are coalesced by scheduleCappedRetry's fixed TaskID.
+func followUpTriggerCount(pending, batchSize, activeSlots, maxInflight int) int {
+ if pending <= 0 || batchSize <= 0 {
+ return 0
+ }
+ batchesNeeded := (pending + batchSize - 1) / batchSize
+ if activeSlots < 0 || maxInflight <= 1 {
+ return 1
+ }
+ freeSlots := maxInflight - activeSlots + 1 // caller's slot releases on return
+ if freeSlots < 1 {
+ freeSlots = 1
+ }
+ return min(batchesNeeded, freeSlots)
+}
+
+// scheduleFollowUp enqueues asynq trigger tasks if there are still pending
+// ops in task_pending_ops for this KB. Returns true when at least one
+// follow-up was scheduled.
+//
+// Post-Phase-3 this backstops the case where a batch drained its claimed
+// window but more rows remain and no other trigger is pending (e.g. a steady
+// trickle of uploads). Standard mode already fans a KB's backlog across
+// concurrent claiming batches, so the short delay is normally just a light
+// debounce rather than a lock-release wait.
+//
+// How many triggers are re-armed follows followUpTriggerCount above: enough
+// to drain the backlog, capped by the free inflight slots, so the concurrent
+// batch count converges on ingest_max_inflight instead of idling at 1.
//
-// `delay` is the ProcessIn before the follow-up fires. Callers pass
+// Fan-out is deliberately skipped when rateLimited: widening concurrency
+// during a 429 storm only amplifies the backoff. Lite mode (no Redis) also
+// never fans out — liteLocks serializes batches per KB, so extra triggers
+// would only bounce off ErrWikiIngestConcurrent.
+//
+// `delay` is the ProcessIn before each follow-up fires. Callers pass
// wikiFollowUpDelay for the normal case and wikiRateLimitBackoff when the
// batch tripped an upstream rate limit — the released failed rows are
// eligible immediately, but nothing claims them until a trigger fires, so
// stretching the follow-up interval is what actually paces retries down
// during a 429 storm.
-func (s *wikiIngestService) scheduleFollowUp(ctx context.Context, payload WikiIngestPayload, delay time.Duration) bool {
+func (s *wikiIngestService) scheduleFollowUp(
+ ctx context.Context,
+ payload WikiIngestPayload,
+ delay time.Duration,
+ batchSize, maxInflight int,
+ rateLimited bool,
+) bool {
if s.pendingRepo == nil {
return false
}
@@ -48,21 +93,39 @@ func (s *wikiIngestService) scheduleFollowUp(ctx context.Context, payload WikiIn
return false
}
- logger.Infof(ctx, "wiki ingest: %d more documents pending for KB %s, scheduling follow-up in %s", count, payload.KnowledgeBaseID, delay)
+ // activeSlots stays -1 (unknown → single follow-up) in Lite mode and on
+ // a rate-limited exit; -1 also shows up in the log line as "unknown".
+ activeSlots := -1
+ if !rateLimited {
+ activeSlots = s.activeInflightSlots(ctx, payload.KnowledgeBaseID)
+ }
+ n := followUpTriggerCount(int(count), batchSize, activeSlots, maxInflight)
+
+ activeStr := fmt.Sprintf("%d", activeSlots)
+ if activeSlots < 0 {
+ activeStr = "unknown"
+ }
+ logger.Infof(ctx,
+ "wiki ingest: %d more documents pending for KB %s, scheduling %d follow-up(s) in %s (active slots %s/%d)",
+ count, payload.KnowledgeBaseID, n, delay, activeStr, maxInflight)
langfuse.InjectTracing(ctx, &payload)
payloadBytes, _ := json.Marshal(payload)
- t := asynq.NewTask(types.TypeWikiIngest, payloadBytes,
- asynq.Queue(types.QueueWiki),
- asynq.MaxRetry(wikiIngestMaxRetry),
- asynq.Timeout(60*time.Minute),
- asynq.ProcessIn(delay), // debounce (or rate-limit backoff) before draining the remainder
- )
- if _, err := s.task.Enqueue(t); err != nil {
- logger.Warnf(ctx, "wiki ingest: follow-up enqueue failed: %v", err)
- return false
+ scheduled := false
+ for i := 0; i < n; i++ {
+ t := asynq.NewTask(types.TypeWikiIngest, payloadBytes,
+ asynq.Queue(types.QueueWiki),
+ asynq.MaxRetry(wikiIngestMaxRetry),
+ asynq.Timeout(60*time.Minute),
+ asynq.ProcessIn(delay), // debounce (or rate-limit backoff) before draining the remainder
+ )
+ if _, err := s.task.Enqueue(t); err != nil {
+ logger.Warnf(ctx, "wiki ingest: follow-up enqueue %d/%d failed: %v", i+1, n, err)
+ continue
+ }
+ scheduled = true
}
- return true
+ return scheduled
}
// newWikiBatchContext builds the per-run lazy fetchers used by both the ingest
@@ -946,7 +1009,7 @@ func (s *wikiIngestService) ProcessWikiIngest(ctx context.Context, t *asynq.Task
logger.Warnf(ctx, "wiki ingest: KB %s hit upstream rate limiting, backing off follow-up to %s", payload.KnowledgeBaseID, followUpDelay)
}
followCtx, followCancel := wikiIngestCleanupContext(ctx)
- followUpScheduled = s.scheduleFollowUp(followCtx, payload, followUpDelay)
+ followUpScheduled = s.scheduleFollowUp(followCtx, payload, followUpDelay, batchSize, maxInflight, rateLimited)
followCancel()
return nil
}
diff --git a/internal/application/service/wiki_ingest_followup_test.go b/internal/application/service/wiki_ingest_followup_test.go
new file mode 100644
index 000000000..2536661e3
--- /dev/null
+++ b/internal/application/service/wiki_ingest_followup_test.go
@@ -0,0 +1,294 @@
+package service
+
+import (
+ "context"
+ "encoding/json"
+ "errors"
+ "sync"
+ "testing"
+ "time"
+
+ "github.com/alicebob/miniredis/v2"
+ "github.com/hibiken/asynq"
+ "github.com/redis/go-redis/v9"
+ "github.com/stretchr/testify/require"
+)
+
+// --- test doubles -----------------------------------------------------------
+
+// wikiPendingRepoForFollowUpTest reuses the cleanup-test repo and makes
+// PendingCount configurable, which is the only input scheduleFollowUp reads.
+type wikiPendingRepoForFollowUpTest struct {
+ wikiPendingRepoForCleanupTest
+ pending int64
+}
+
+func (r *wikiPendingRepoForFollowUpTest) PendingCount(
+ context.Context, string, string, string,
+) (int64, error) {
+ return r.pending, nil
+}
+
+// followUpEnqueuerRecorder captures every trigger scheduleFollowUp enqueues.
+// failFrom >= 0 makes enqueues from that index onward return failErr so the
+// partial-failure path can be exercised.
+type followUpEnqueuerRecorder struct {
+ mu sync.Mutex
+ tasks []*asynq.Task
+ failFrom int
+ failErr error
+}
+
+func (e *followUpEnqueuerRecorder) Enqueue(task *asynq.Task, _ ...asynq.Option) (*asynq.TaskInfo, error) {
+ e.mu.Lock()
+ defer e.mu.Unlock()
+ idx := len(e.tasks)
+ e.tasks = append(e.tasks, task)
+ if e.failFrom >= 0 && idx >= e.failFrom {
+ return nil, e.failErr
+ }
+ return &asynq.TaskInfo{}, nil
+}
+
+func (e *followUpEnqueuerRecorder) count() int {
+ e.mu.Lock()
+ defer e.mu.Unlock()
+ return len(e.tasks)
+}
+
+func (e *followUpEnqueuerRecorder) payloads(t *testing.T, kbID string) {
+ t.Helper()
+ e.mu.Lock()
+ defer e.mu.Unlock()
+ for _, task := range e.tasks {
+ var payload WikiIngestPayload
+ require.NoError(t, json.Unmarshal(task.Payload(), &payload))
+ require.Equal(t, kbID, payload.KnowledgeBaseID)
+ }
+}
+
+// --- pure fan-out computation ----------------------------------------------
+
+func TestFollowUpTriggerCount(t *testing.T) {
+ tests := []struct {
+ name string
+ pending, batchSize, active, max int
+ want int
+ }{
+ {"no pending enqueues nothing", 0, 5, 1, 4, 0},
+ {"defensive zero batch size", 10, 0, 1, 4, 0},
+ {"one batch or less stays at one", 5, 5, 1, 4, 1},
+ {"single row stays at one", 1, 5, 1, 4, 1},
+ {"just over one batch needs two", 6, 5, 1, 4, 2},
+ {"backlog need is capped by free slots", 1000, 5, 2, 4, 3},
+ {"all slots free counts own release", 1000, 5, 1, 4, 4},
+ {"full board degrades to one", 1000, 5, 4, 4, 1},
+ {"shrunk cap mid-flight clamps to one", 1000, 5, 7, 4, 1},
+ {"unknown slot count is conservative", 1000, 5, -1, 4, 1},
+ {"single-slot cap never fans out", 1000, 5, 0, 1, 1},
+ {"exact multiple of batch size", 15, 5, 1, 4, 3},
+ }
+ for _, tt := range tests {
+ t.Run(tt.name, func(t *testing.T) {
+ got := followUpTriggerCount(tt.pending, tt.batchSize, tt.active, tt.max)
+ require.Equal(t, tt.want, got)
+ })
+ }
+}
+
+// --- activeInflightSlots ----------------------------------------------------
+
+func TestActiveInflightSlotsPurgesExpiredAndCountsLive(t *testing.T) {
+ mr := miniredis.RunT(t)
+ rdb := redis.NewClient(&redis.Options{Addr: mr.Addr()})
+ t.Cleanup(func() { _ = rdb.Close() })
+
+ svc := &wikiIngestService{redisClient: rdb}
+ ctx := context.Background()
+ key := wikiInflightPrefix + "kb-1"
+
+ // Two slots that expired long ago plus one live slot.
+ require.NoError(t, rdb.ZAdd(ctx, key, redis.Z{
+ Score: float64(time.Now().Add(-time.Hour).UnixMilli()),
+ Member: "expired-1",
+ }).Err())
+ require.NoError(t, rdb.ZAdd(ctx, key, redis.Z{
+ Score: float64(time.Now().Add(-time.Minute).UnixMilli()),
+ Member: "expired-2",
+ }).Err())
+ require.NoError(t, rdb.ZAdd(ctx, key, redis.Z{
+ Score: float64(time.Now().Add(wikiInflightTTL).UnixMilli()),
+ Member: "live-1",
+ }).Err())
+
+ require.Equal(t, 1, svc.activeInflightSlots(ctx, "kb-1"))
+}
+
+func TestActiveInflightSlotsUnknownOnRedisError(t *testing.T) {
+ mr := miniredis.RunT(t)
+ rdb := redis.NewClient(&redis.Options{Addr: mr.Addr()})
+ require.NoError(t, rdb.Close()) // every Eval now fails
+
+ svc := &wikiIngestService{redisClient: rdb}
+ require.Equal(t, -1, svc.activeInflightSlots(context.Background(), "kb-1"))
+}
+
+// --- scheduleFollowUp -------------------------------------------------------
+
+func newFollowUpTestService(t *testing.T, pending int64) (*wikiIngestService, *followUpEnqueuerRecorder) {
+ t.Helper()
+ rec := &followUpEnqueuerRecorder{failFrom: -1}
+ svc := &wikiIngestService{
+ task: rec,
+ pendingRepo: &wikiPendingRepoForFollowUpTest{pending: pending},
+ }
+ return svc, rec
+}
+
+// reserveSlots claims n live slots the way n concurrent batches would, so the
+// fan-out computation sees a realistic board.
+func reserveSlots(t *testing.T, svc *wikiIngestService, kbID string, n, maxInflight int) {
+ t.Helper()
+ for i := 0; i < n; i++ {
+ release, granted := svc.reserveInflightSlot(context.Background(), kbID, maxInflight)
+ require.True(t, granted)
+ t.Cleanup(release)
+ }
+}
+
+func TestScheduleFollowUpFansOutToFreeSlots(t *testing.T) {
+ mr := miniredis.RunT(t)
+ rdb := redis.NewClient(&redis.Options{Addr: mr.Addr()})
+ t.Cleanup(func() { _ = rdb.Close() })
+
+ // 100 pending docs, batch size 10 → 10 batches needed. Two slots held
+ // (one of them modelling the caller's own, which releases on return),
+ // cap 4 → free = 4-2+1 = 3 follow-ups.
+ svc, rec := newFollowUpTestService(t, 100)
+ svc.redisClient = rdb
+ reserveSlots(t, svc, "kb-1", 2, 4)
+
+ got := svc.scheduleFollowUp(context.Background(),
+ WikiIngestPayload{KnowledgeBaseID: "kb-1"}, wikiFollowUpDelay, 10, 4, false)
+ require.True(t, got)
+ require.Equal(t, 3, rec.count())
+ rec.payloads(t, "kb-1")
+}
+
+func TestScheduleFollowUpBacklogSmallerThanBatchStaysSingle(t *testing.T) {
+ mr := miniredis.RunT(t)
+ rdb := redis.NewClient(&redis.Options{Addr: mr.Addr()})
+ t.Cleanup(func() { _ = rdb.Close() })
+
+ svc, rec := newFollowUpTestService(t, 7)
+ svc.redisClient = rdb
+ reserveSlots(t, svc, "kb-1", 1, 4)
+
+ got := svc.scheduleFollowUp(context.Background(),
+ WikiIngestPayload{KnowledgeBaseID: "kb-1"}, wikiFollowUpDelay, 10, 4, false)
+ require.True(t, got)
+ require.Equal(t, 1, rec.count())
+}
+
+func TestScheduleFollowUpRateLimitedStaysSingle(t *testing.T) {
+ mr := miniredis.RunT(t)
+ rdb := redis.NewClient(&redis.Options{Addr: mr.Addr()})
+ t.Cleanup(func() { _ = rdb.Close() })
+
+ // Even with the whole board free and a huge backlog, a rate-limited exit
+ // must not widen concurrency: widening during a 429 storm amplifies it.
+ svc, rec := newFollowUpTestService(t, 1000)
+ svc.redisClient = rdb
+
+ got := svc.scheduleFollowUp(context.Background(),
+ WikiIngestPayload{KnowledgeBaseID: "kb-1"}, wikiRateLimitBackoff, 10, 32, true)
+ require.True(t, got)
+ require.Equal(t, 1, rec.count())
+}
+
+func TestScheduleFollowUpLiteModeStaysSingle(t *testing.T) {
+ // No Redis client → liteLocks serializes batches per KB; extra triggers
+ // would only bounce off ErrWikiIngestConcurrent.
+ svc, rec := newFollowUpTestService(t, 1000)
+
+ got := svc.scheduleFollowUp(context.Background(),
+ WikiIngestPayload{KnowledgeBaseID: "kb-1"}, wikiFollowUpDelay, 10, 32, false)
+ require.True(t, got)
+ require.Equal(t, 1, rec.count())
+}
+
+func TestScheduleFollowUpNoPendingSchedulesNothing(t *testing.T) {
+ mr := miniredis.RunT(t)
+ rdb := redis.NewClient(&redis.Options{Addr: mr.Addr()})
+ t.Cleanup(func() { _ = rdb.Close() })
+
+ svc, rec := newFollowUpTestService(t, 0)
+ svc.redisClient = rdb
+
+ got := svc.scheduleFollowUp(context.Background(),
+ WikiIngestPayload{KnowledgeBaseID: "kb-1"}, wikiFollowUpDelay, 10, 4, false)
+ require.False(t, got)
+ require.Equal(t, 0, rec.count())
+}
+
+func TestScheduleFollowUpPartialEnqueueFailureStillReportsScheduled(t *testing.T) {
+ mr := miniredis.RunT(t)
+ rdb := redis.NewClient(&redis.Options{Addr: mr.Addr()})
+ t.Cleanup(func() { _ = rdb.Close() })
+
+ svc, rec := newFollowUpTestService(t, 100)
+ svc.redisClient = rdb
+ rec.failFrom = 1
+ rec.failErr = errors.New("enqueue failed")
+
+ // The caller holds its own slot (releases on return): free = 4-1+1 = 4.
+ // The first enqueue succeeds, the rest fail; "scheduled" must still be
+ // true because at least one trigger got out.
+ reserveSlots(t, svc, "kb-1", 1, 4)
+ got := svc.scheduleFollowUp(context.Background(),
+ WikiIngestPayload{KnowledgeBaseID: "kb-1"}, wikiFollowUpDelay, 10, 4, false)
+ require.True(t, got)
+ require.Equal(t, 4, rec.count())
+}
+
+// --- convergence ------------------------------------------------------------
+
+// TestFollowUpFanOutConvergesUnderCap simulates successive batch completions
+// fanning out on a board whose slots never free (worst case for over-issue)
+// and asserts the per-completion trigger count never exceeds the free slots,
+// i.e. the supply side alone cannot push concurrency past ingest_max_inflight.
+func TestFollowUpFanOutConvergesUnderCap(t *testing.T) {
+ const (
+ maxInflight = 8
+ batchSize = 10
+ )
+ var (
+ pending = 1000
+ activeSlots = 1 // the completing batch itself; releases on return
+ )
+ seen := map[int]bool{}
+ for round := 0; round < 6; round++ {
+ n := followUpTriggerCount(pending, batchSize, activeSlots, maxInflight)
+ require.Greater(t, n, 0)
+ require.LessOrEqual(t, n, maxInflight-activeSlots+1,
+ "round %d: supply must not exceed free slots", round)
+ if activeSlots == maxInflight {
+ require.Equal(t, 1, n,
+ "round %d: a full board must clamp supply to the follow-up chain", round)
+ }
+ seen[n] = true
+
+ // The n triggers fire: n-1 of them claim slots (one bounces off the
+ // cap into scheduleCappedRetry when the board is full), each claims a
+ // batch of batchSize rows.
+ newBatches := min(n, maxInflight-activeSlots)
+ pending -= newBatches * batchSize
+ activeSlots = maxInflight
+ if pending <= 0 {
+ pending = 1 // keep the loop meaningful; count stays capped below
+ }
+ }
+ // The first round must have fanned out beyond the follow-up chain —
+ // otherwise this would not be testing convergence at all.
+ require.Greater(t, len(seen), 1)
+}