Open-source LLM knowledge platform: turn raw documents into a queryable RAG, an autonomous reasoning agent, and a self-maintaining Wiki.
Go agentagenticaichatbotdsh-pluginembeddingsevaluationgenerative-aigolangknowledge-basellmmulti-tenantollamaopenaiquestion-answeringragrerankingsemantic-searchvector-searchwiki
fix(wiki): fan out follow-up triggers so backlogs drain at ingest_max_inflight
Duke4ba85e2edf71
diff --git a/internal/application/service/wiki_ingest.go b/internal/application/service/wiki_ingest.goindex 4df7d57f6..172033025 100644--- a/internal/application/service/wiki_ingest.go+++ b/internal/application/service/wiki_ingest.go@@ -1112,6 +1112,34 @@ func (s *wikiIngestService) reserveInflightSlot(ctx context.Context, kbID string}, true}+// wikiInflightCountScript purges expired slots exactly like the reserve+// script does, then returns the live count. Read-mostly companion to+// wikiInflightReserveScript so a fan-out decision sees the same view of the+// cap a reserver would.+const wikiInflightCountScript = `+redis.call('ZREMRANGEBYSCORE', KEYS[1], 0, tonumber(ARGV[1]))+return redis.call('ZCARD', KEYS[1])+`++// activeInflightSlots reports how many batches currently hold one of KB's+// in-flight slots (standard/Redis mode). Returns -1 when the count is+// unavailable (no Redis client, or a Redis error); callers must treat that as+// "unknown" and stay conservative instead of scaling on a stale number.+func (s *wikiIngestService) activeInflightSlots(ctx context.Context, kbID string) int {+ if s.redisClient == nil {+ return -1+ }+ res, err := s.redisClient.Eval(ctx, wikiInflightCountScript,+ []string{wikiInflightPrefix + kbID},+ time.Now().UnixMilli(),+ ).Int()+ if err != nil {+ logger.Warnf(ctx, "wiki ingest: inflight count failed for KB %s: %v", kbID, err)+ return -1+ }+ return res+}+// scheduleCappedRetry enqueues a single coalesced follow-up trigger after a// batch was turned away by the in-flight cap. asynq.TaskID collapses all// turned-away triggers for one KB into a single pending retry (no thunderingdiff --git a/internal/application/service/wiki_ingest_batch.go b/internal/application/service/wiki_ingest_batch.goindex c16c17582..796f7c0a0 100644--- a/internal/application/service/wiki_ingest_batch.go+++ b/internal/application/service/wiki_ingest_batch.go@@ -23,23 +23,68 @@ import ("golang.org/x/sync/errgroup")-// scheduleFollowUp enqueues another asynq trigger task if there are-// still pending ops in task_pending_ops for this KB. Returns true when-// a follow-up was scheduled.+// followUpTriggerCount decides how many follow-up triggers a completed batch+// should re-arm for its KB (#3461). Since Phase 3 the consumption side+// (disjoint claiming, per-slug locks, the inflight cap) fully supports+// concurrent batches, but nothing created more than one trigger for a KB with+// a standing backlog — so once uploads stopped, the backlog drained at+// 1 batch × batch-size per cycle regardless of ingest_max_inflight, leaving+// the wiki pool and the model server idle.//-// Post-Phase-3 this only backstops the case where a batch drained its-// claimed window but more rows remain and no other trigger is pending-// (e.g. steady trickle of uploads). Standard mode already fans a KB's-// backlog across concurrent claiming batches, so the short delay is-// normally just a light debounce rather than a lock-release wait.+// The count is the smaller of (a) how many batches the remaining rows still+// need and (b) how many slots could be free by the time the triggers fire:+// the caller's own slot releases right after it returns, hence the +1.+// activeSlots < 0 means "unknown" (no Redis / Redis error) and maxInflight+// <= 1 cannot host more than one batch anyway; both collapse to 1, matching+// the pre-fan-out behaviour. Over-issued triggers are harmless: the ones that+// cannot get a slot are coalesced by scheduleCappedRetry's fixed TaskID.+func followUpTriggerCount(pending, batchSize, activeSlots, maxInflight int) int {+ if pending <= 0 || batchSize <= 0 {+ return 0+ }+ batchesNeeded := (pending + batchSize - 1) / batchSize+ if activeSlots < 0 || maxInflight <= 1 {+ return 1+ }+ freeSlots := maxInflight - activeSlots + 1 // caller's slot releases on return+ if freeSlots < 1 {+ freeSlots = 1+ }+ return min(batchesNeeded, freeSlots)+}++// scheduleFollowUp enqueues asynq trigger tasks if there are still pending+// ops in task_pending_ops for this KB. Returns true when at least one+// follow-up was scheduled.+//+// Post-Phase-3 this backstops the case where a batch drained its claimed+// window but more rows remain and no other trigger is pending (e.g. a steady+// trickle of uploads). Standard mode already fans a KB's backlog across+// concurrent claiming batches, so the short delay is normally just a light+// debounce rather than a lock-release wait.+//+// How many triggers are re-armed follows followUpTriggerCount above: enough+// to drain the backlog, capped by the free inflight slots, so the concurrent+// batch count converges on ingest_max_inflight instead of idling at 1.//-// `delay` is the ProcessIn before the follow-up fires. Callers pass+// Fan-out is deliberately skipped when rateLimited: widening concurrency+// during a 429 storm only amplifies the backoff. Lite mode (no Redis) also+// never fans out — liteLocks serializes batches per KB, so extra triggers+// would only bounce off ErrWikiIngestConcurrent.+//+// `delay` is the ProcessIn before each follow-up fires. Callers pass// wikiFollowUpDelay for the normal case and wikiRateLimitBackoff when the// batch tripped an upstream rate limit — the released failed rows are// eligible immediately, but nothing claims them until a trigger fires, so// stretching the follow-up interval is what actually paces retries down// during a 429 storm.-func (s *wikiIngestService) scheduleFollowUp(ctx context.Context, payload WikiIngestPayload, delay time.Duration) bool {+func (s *wikiIngestService) scheduleFollowUp(+ ctx context.Context,+ payload WikiIngestPayload,+ delay time.Duration,+ batchSize, maxInflight int,+ rateLimited bool,+) bool {if s.pendingRepo == nil {return false}@@ -48,21 +93,39 @@ func (s *wikiIngestService) scheduleFollowUp(ctx context.Context, payload WikiInreturn false}- logger.Infof(ctx, "wiki ingest: %d more documents pending for KB %s, scheduling follow-up in %s", count, payload.KnowledgeBaseID, delay)+ // activeSlots stays -1 (unknown → single follow-up) in Lite mode and on+ // a rate-limited exit; -1 also shows up in the log line as "unknown".+ activeSlots := -1+ if !rateLimited {+ activeSlots = s.activeInflightSlots(ctx, payload.KnowledgeBaseID)+ }+ n := followUpTriggerCount(int(count), batchSize, activeSlots, maxInflight)++ activeStr := fmt.Sprintf("%d", activeSlots)+ if activeSlots < 0 {+ activeStr = "unknown"+ }+ logger.Infof(ctx,+ "wiki ingest: %d more documents pending for KB %s, scheduling %d follow-up(s) in %s (active slots %s/%d)",+ count, payload.KnowledgeBaseID, n, delay, activeStr, maxInflight)langfuse.InjectTracing(ctx, &payload)payloadBytes, _ := json.Marshal(payload)- t := asynq.NewTask(types.TypeWikiIngest, payloadBytes,- asynq.Queue(types.QueueWiki),- asynq.MaxRetry(wikiIngestMaxRetry),- asynq.Timeout(60*time.Minute),- asynq.ProcessIn(delay), // debounce (or rate-limit backoff) before draining the remainder- )- if _, err := s.task.Enqueue(t); err != nil {- logger.Warnf(ctx, "wiki ingest: follow-up enqueue failed: %v", err)- return false+ scheduled := false+ for i := 0; i < n; i++ {+ t := asynq.NewTask(types.TypeWikiIngest, payloadBytes,+ asynq.Queue(types.QueueWiki),+ asynq.MaxRetry(wikiIngestMaxRetry),+ asynq.Timeout(60*time.Minute),+ asynq.ProcessIn(delay), // debounce (or rate-limit backoff) before draining the remainder+ )+ if _, err := s.task.Enqueue(t); err != nil {+ logger.Warnf(ctx, "wiki ingest: follow-up enqueue %d/%d failed: %v", i+1, n, err)+ continue+ }+ scheduled = true}- return true+ return scheduled}// newWikiBatchContext builds the per-run lazy fetchers used by both the ingest@@ -946,7 +1009,7 @@ func (s *wikiIngestService) ProcessWikiIngest(ctx context.Context, t *asynq.Tasklogger.Warnf(ctx, "wiki ingest: KB %s hit upstream rate limiting, backing off follow-up to %s", payload.KnowledgeBaseID, followUpDelay)}followCtx, followCancel := wikiIngestCleanupContext(ctx)- followUpScheduled = s.scheduleFollowUp(followCtx, payload, followUpDelay)+ followUpScheduled = s.scheduleFollowUp(followCtx, payload, followUpDelay, batchSize, maxInflight, rateLimited)followCancel()return nil}diff --git a/internal/application/service/wiki_ingest_followup_test.go b/internal/application/service/wiki_ingest_followup_test.gonew file mode 100644index 000000000..2536661e3--- /dev/null+++ b/internal/application/service/wiki_ingest_followup_test.go@@ -0,0 +1,294 @@+package service++import (+ "context"+ "encoding/json"+ "errors"+ "sync"+ "testing"+ "time"++ "github.com/alicebob/miniredis/v2"+ "github.com/hibiken/asynq"+ "github.com/redis/go-redis/v9"+ "github.com/stretchr/testify/require"+)++// --- test doubles -----------------------------------------------------------++// wikiPendingRepoForFollowUpTest reuses the cleanup-test repo and makes+// PendingCount configurable, which is the only input scheduleFollowUp reads.+type wikiPendingRepoForFollowUpTest struct {+ wikiPendingRepoForCleanupTest+ pending int64+}++func (r *wikiPendingRepoForFollowUpTest) PendingCount(+ context.Context, string, string, string,+) (int64, error) {+ return r.pending, nil+}++// followUpEnqueuerRecorder captures every trigger scheduleFollowUp enqueues.+// failFrom >= 0 makes enqueues from that index onward return failErr so the+// partial-failure path can be exercised.+type followUpEnqueuerRecorder struct {+ mu sync.Mutex+ tasks []*asynq.Task+ failFrom int+ failErr error+}++func (e *followUpEnqueuerRecorder) Enqueue(task *asynq.Task, _ ...asynq.Option) (*asynq.TaskInfo, error) {+ e.mu.Lock()+ defer e.mu.Unlock()+ idx := len(e.tasks)+ e.tasks = append(e.tasks, task)+ if e.failFrom >= 0 && idx >= e.failFrom {+ return nil, e.failErr+ }+ return &asynq.TaskInfo{}, nil+}++func (e *followUpEnqueuerRecorder) count() int {+ e.mu.Lock()+ defer e.mu.Unlock()+ return len(e.tasks)+}++func (e *followUpEnqueuerRecorder) payloads(t *testing.T, kbID string) {+ t.Helper()+ e.mu.Lock()+ defer e.mu.Unlock()+ for _, task := range e.tasks {+ var payload WikiIngestPayload+ require.NoError(t, json.Unmarshal(task.Payload(), &payload))+ require.Equal(t, kbID, payload.KnowledgeBaseID)+ }+}++// --- pure fan-out computation ----------------------------------------------++func TestFollowUpTriggerCount(t *testing.T) {+ tests := []struct {+ name string+ pending, batchSize, active, max int+ want int+ }{+ {"no pending enqueues nothing", 0, 5, 1, 4, 0},+ {"defensive zero batch size", 10, 0, 1, 4, 0},+ {"one batch or less stays at one", 5, 5, 1, 4, 1},+ {"single row stays at one", 1, 5, 1, 4, 1},+ {"just over one batch needs two", 6, 5, 1, 4, 2},+ {"backlog need is capped by free slots", 1000, 5, 2, 4, 3},+ {"all slots free counts own release", 1000, 5, 1, 4, 4},+ {"full board degrades to one", 1000, 5, 4, 4, 1},+ {"shrunk cap mid-flight clamps to one", 1000, 5, 7, 4, 1},+ {"unknown slot count is conservative", 1000, 5, -1, 4, 1},+ {"single-slot cap never fans out", 1000, 5, 0, 1, 1},+ {"exact multiple of batch size", 15, 5, 1, 4, 3},+ }+ for _, tt := range tests {+ t.Run(tt.name, func(t *testing.T) {+ got := followUpTriggerCount(tt.pending, tt.batchSize, tt.active, tt.max)+ require.Equal(t, tt.want, got)+ })+ }+}++// --- activeInflightSlots ----------------------------------------------------++func TestActiveInflightSlotsPurgesExpiredAndCountsLive(t *testing.T) {+ mr := miniredis.RunT(t)+ rdb := redis.NewClient(&redis.Options{Addr: mr.Addr()})+ t.Cleanup(func() { _ = rdb.Close() })++ svc := &wikiIngestService{redisClient: rdb}+ ctx := context.Background()+ key := wikiInflightPrefix + "kb-1"++ // Two slots that expired long ago plus one live slot.+ require.NoError(t, rdb.ZAdd(ctx, key, redis.Z{+ Score: float64(time.Now().Add(-time.Hour).UnixMilli()),+ Member: "expired-1",+ }).Err())+ require.NoError(t, rdb.ZAdd(ctx, key, redis.Z{+ Score: float64(time.Now().Add(-time.Minute).UnixMilli()),+ Member: "expired-2",+ }).Err())+ require.NoError(t, rdb.ZAdd(ctx, key, redis.Z{+ Score: float64(time.Now().Add(wikiInflightTTL).UnixMilli()),+ Member: "live-1",+ }).Err())++ require.Equal(t, 1, svc.activeInflightSlots(ctx, "kb-1"))+}++func TestActiveInflightSlotsUnknownOnRedisError(t *testing.T) {+ mr := miniredis.RunT(t)+ rdb := redis.NewClient(&redis.Options{Addr: mr.Addr()})+ require.NoError(t, rdb.Close()) // every Eval now fails++ svc := &wikiIngestService{redisClient: rdb}+ require.Equal(t, -1, svc.activeInflightSlots(context.Background(), "kb-1"))+}++// --- scheduleFollowUp -------------------------------------------------------++func newFollowUpTestService(t *testing.T, pending int64) (*wikiIngestService, *followUpEnqueuerRecorder) {+ t.Helper()+ rec := &followUpEnqueuerRecorder{failFrom: -1}+ svc := &wikiIngestService{+ task: rec,+ pendingRepo: &wikiPendingRepoForFollowUpTest{pending: pending},+ }+ return svc, rec+}++// reserveSlots claims n live slots the way n concurrent batches would, so the+// fan-out computation sees a realistic board.+func reserveSlots(t *testing.T, svc *wikiIngestService, kbID string, n, maxInflight int) {+ t.Helper()+ for i := 0; i < n; i++ {+ release, granted := svc.reserveInflightSlot(context.Background(), kbID, maxInflight)+ require.True(t, granted)+ t.Cleanup(release)+ }+}++func TestScheduleFollowUpFansOutToFreeSlots(t *testing.T) {+ mr := miniredis.RunT(t)+ rdb := redis.NewClient(&redis.Options{Addr: mr.Addr()})+ t.Cleanup(func() { _ = rdb.Close() })++ // 100 pending docs, batch size 10 → 10 batches needed. Two slots held+ // (one of them modelling the caller's own, which releases on return),+ // cap 4 → free = 4-2+1 = 3 follow-ups.+ svc, rec := newFollowUpTestService(t, 100)+ svc.redisClient = rdb+ reserveSlots(t, svc, "kb-1", 2, 4)++ got := svc.scheduleFollowUp(context.Background(),+ WikiIngestPayload{KnowledgeBaseID: "kb-1"}, wikiFollowUpDelay, 10, 4, false)+ require.True(t, got)+ require.Equal(t, 3, rec.count())+ rec.payloads(t, "kb-1")+}++func TestScheduleFollowUpBacklogSmallerThanBatchStaysSingle(t *testing.T) {+ mr := miniredis.RunT(t)+ rdb := redis.NewClient(&redis.Options{Addr: mr.Addr()})+ t.Cleanup(func() { _ = rdb.Close() })++ svc, rec := newFollowUpTestService(t, 7)+ svc.redisClient = rdb+ reserveSlots(t, svc, "kb-1", 1, 4)++ got := svc.scheduleFollowUp(context.Background(),+ WikiIngestPayload{KnowledgeBaseID: "kb-1"}, wikiFollowUpDelay, 10, 4, false)+ require.True(t, got)+ require.Equal(t, 1, rec.count())+}++func TestScheduleFollowUpRateLimitedStaysSingle(t *testing.T) {+ mr := miniredis.RunT(t)+ rdb := redis.NewClient(&redis.Options{Addr: mr.Addr()})+ t.Cleanup(func() { _ = rdb.Close() })++ // Even with the whole board free and a huge backlog, a rate-limited exit+ // must not widen concurrency: widening during a 429 storm amplifies it.+ svc, rec := newFollowUpTestService(t, 1000)+ svc.redisClient = rdb++ got := svc.scheduleFollowUp(context.Background(),+ WikiIngestPayload{KnowledgeBaseID: "kb-1"}, wikiRateLimitBackoff, 10, 32, true)+ require.True(t, got)+ require.Equal(t, 1, rec.count())+}++func TestScheduleFollowUpLiteModeStaysSingle(t *testing.T) {+ // No Redis client → liteLocks serializes batches per KB; extra triggers+ // would only bounce off ErrWikiIngestConcurrent.+ svc, rec := newFollowUpTestService(t, 1000)++ got := svc.scheduleFollowUp(context.Background(),+ WikiIngestPayload{KnowledgeBaseID: "kb-1"}, wikiFollowUpDelay, 10, 32, false)+ require.True(t, got)+ require.Equal(t, 1, rec.count())+}++func TestScheduleFollowUpNoPendingSchedulesNothing(t *testing.T) {+ mr := miniredis.RunT(t)+ rdb := redis.NewClient(&redis.Options{Addr: mr.Addr()})+ t.Cleanup(func() { _ = rdb.Close() })++ svc, rec := newFollowUpTestService(t, 0)+ svc.redisClient = rdb++ got := svc.scheduleFollowUp(context.Background(),+ WikiIngestPayload{KnowledgeBaseID: "kb-1"}, wikiFollowUpDelay, 10, 4, false)+ require.False(t, got)+ require.Equal(t, 0, rec.count())+}++func TestScheduleFollowUpPartialEnqueueFailureStillReportsScheduled(t *testing.T) {+ mr := miniredis.RunT(t)+ rdb := redis.NewClient(&redis.Options{Addr: mr.Addr()})+ t.Cleanup(func() { _ = rdb.Close() })++ svc, rec := newFollowUpTestService(t, 100)+ svc.redisClient = rdb+ rec.failFrom = 1+ rec.failErr = errors.New("enqueue failed")++ // The caller holds its own slot (releases on return): free = 4-1+1 = 4.+ // The first enqueue succeeds, the rest fail; "scheduled" must still be+ // true because at least one trigger got out.+ reserveSlots(t, svc, "kb-1", 1, 4)+ got := svc.scheduleFollowUp(context.Background(),+ WikiIngestPayload{KnowledgeBaseID: "kb-1"}, wikiFollowUpDelay, 10, 4, false)+ require.True(t, got)+ require.Equal(t, 4, rec.count())+}++// --- convergence ------------------------------------------------------------++// TestFollowUpFanOutConvergesUnderCap simulates successive batch completions+// fanning out on a board whose slots never free (worst case for over-issue)+// and asserts the per-completion trigger count never exceeds the free slots,+// i.e. the supply side alone cannot push concurrency past ingest_max_inflight.+func TestFollowUpFanOutConvergesUnderCap(t *testing.T) {+ const (+ maxInflight = 8+ batchSize = 10+ )+ var (+ pending = 1000+ activeSlots = 1 // the completing batch itself; releases on return+ )+ seen := map[int]bool{}+ for round := 0; round < 6; round++ {+ n := followUpTriggerCount(pending, batchSize, activeSlots, maxInflight)+ require.Greater(t, n, 0)+ require.LessOrEqual(t, n, maxInflight-activeSlots+1,+ "round %d: supply must not exceed free slots", round)+ if activeSlots == maxInflight {+ require.Equal(t, 1, n,+ "round %d: a full board must clamp supply to the follow-up chain", round)+ }+ seen[n] = true++ // The n triggers fire: n-1 of them claim slots (one bounces off the+ // cap into scheduleCappedRetry when the board is full), each claims a+ // batch of batchSize rows.+ newBatches := min(n, maxInflight-activeSlots)+ pending -= newBatches * batchSize+ activeSlots = maxInflight+ if pending <= 0 {+ pending = 1 // keep the loop meaningful; count stays capped below+ }+ }+ // The first round must have fanned out beyond the follow-up chain —+ // otherwise this would not be testing convergence at all.+ require.Greater(t, len(seen), 1)+}