Skip to content

Open-source LLM knowledge platform: turn raw documents into a queryable RAG, an autonomous reasoning agent, and a self-maintaining Wiki.

fix(knowledge): stop two parse paths from stranding a row in processing (#3445)

lyingbugeaca7364cd77
diff --git a/internal/application/service/image_multimodal.go b/internal/application/service/image_multimodal.go
index 631425f03..bd8734404 100644
--- a/internal/application/service/image_multimodal.go
+++ b/internal/application/service/image_multimodal.go
@@ -659,13 +659,21 @@ func downloadImageFromURL(imageURL string) ([]byte, error) {
return secutils.DownloadBytes(imageURL)
}
+// multimodalPendingKey is the Redis key holding how many image tasks a
+// knowledge is still waiting on. The fan-out (enqueueImageMultimodalTasks)
+// seeds it and the fan-in (checkAndFinalizeAllImages) drains it, so the two
+// must agree on the name — hence one helper rather than two format strings.
+func multimodalPendingKey(knowledgeID string) string {
+ return fmt.Sprintf("multimodal:pending:%s", knowledgeID)
+}
+
func (s *ImageMultimodalService) checkAndFinalizeAllImages(ctx context.Context, payload types.ImageMultimodalPayload) {
if s.redisClient == nil {
s.enqueueKnowledgePostProcessTask(ctx, payload)
return
}
- redisKey := fmt.Sprintf("multimodal:pending:%s", payload.KnowledgeID)
+ redisKey := multimodalPendingKey(payload.KnowledgeID)
pendingCount, err := s.redisClient.Decr(ctx, redisKey).Result()
if err != nil && err != redis.Nil {
diff --git a/internal/application/service/knowledge_multimodal_slot_release_test.go b/internal/application/service/knowledge_multimodal_slot_release_test.go
new file mode 100644
index 000000000..2ace05217
--- /dev/null
+++ b/internal/application/service/knowledge_multimodal_slot_release_test.go
@@ -0,0 +1,254 @@
+package service
+
+import (
+ "context"
+ "errors"
+ "testing"
+
+ "github.com/Tencent/WeKnora/internal/infrastructure/docparser"
+ "github.com/Tencent/WeKnora/internal/types"
+ "github.com/alicebob/miniredis/v2"
+ "github.com/hibiken/asynq"
+ "github.com/redis/go-redis/v9"
+ "github.com/stretchr/testify/require"
+)
+
+// slotReleaseEnqueuer fails the image tasks whose index is in failAt, so a
+// test can reproduce a fan-out that only partially reaches the queue.
+//
+// onImageEnqueued lets a test simulate an image worker that finishes before
+// the fan-out loop returns — the ordering that decides whether the shortfall
+// release or the last sibling owns the finalize.
+type slotReleaseEnqueuer struct {
+ failAt map[int]bool
+ imageSeen int
+ imageEnqueued int
+ postProcess int
+ onImageEnqueued func()
+}
+
+func (e *slotReleaseEnqueuer) Enqueue(task *asynq.Task, _ ...asynq.Option) (*asynq.TaskInfo, error) {
+ switch task.Type() {
+ case types.TypeImageMultimodal:
+ idx := e.imageSeen
+ e.imageSeen++
+ if e.failAt[idx] {
+ return nil, errors.New("queue write refused")
+ }
+ e.imageEnqueued++
+ if e.onImageEnqueued != nil {
+ e.onImageEnqueued()
+ }
+ return &asynq.TaskInfo{ID: "image"}, nil
+ case types.TypeKnowledgePostProcess:
+ e.postProcess++
+ return &asynq.TaskInfo{ID: "post-process"}, nil
+ }
+ return &asynq.TaskInfo{ID: "other"}, nil
+}
+
+func newSlotReleaseRedis(t *testing.T) (*miniredis.Miniredis, *redis.Client) {
+ t.Helper()
+ mr, err := miniredis.Run()
+ require.NoError(t, err)
+ t.Cleanup(mr.Close)
+ rdb := redis.NewClient(&redis.Options{Addr: mr.Addr()})
+ t.Cleanup(func() { _ = rdb.Close() })
+ return mr, rdb
+}
+
+func slotReleaseFixture(images int) (
+ *types.Knowledge, *types.KnowledgeBase, []docparser.StoredImage, []types.ParsedChunk,
+) {
+ knowledge := &types.Knowledge{
+ ID: "knowledge-1",
+ TenantID: 1,
+ KnowledgeBaseID: "kb-1",
+ ParseStatus: types.ParseStatusProcessing,
+ }
+ kb := &types.KnowledgeBase{ID: "kb-1", TenantID: 1}
+ stored := make([]docparser.StoredImage, 0, images)
+ for i := 0; i < images; i++ {
+ stored = append(stored, docparser.StoredImage{
+ OriginalRef: "images/page.png",
+ ServingURL: "local://images/page-" + string(rune('a'+i)) + ".png",
+ })
+ }
+ return knowledge, kb, stored, []types.ParsedChunk{{ChunkID: "chunk-1", Content: "body"}}
+}
+
+// The happy path must leave the counter at exactly the number of enqueued
+// tasks and must NOT enqueue post-process — the last image to finish owns it.
+func TestEnqueueImageMultimodalTasksSeedsCounterForEveryImage(t *testing.T) {
+ mr, rdb := newSlotReleaseRedis(t)
+ enqueuer := &slotReleaseEnqueuer{}
+ svc := &knowledgeService{task: enqueuer, redisClient: rdb}
+ knowledge, kb, images, chunks := slotReleaseFixture(3)
+
+ svc.enqueueImageMultimodalTasks(context.Background(), knowledge, kb, images, chunks, nil)
+
+ require.Equal(t, 3, enqueuer.imageEnqueued)
+ require.Equal(t, 0, enqueuer.postProcess)
+ require.Equal(t, "3", mustGet(t, mr, multimodalPendingKey(knowledge.ID)))
+}
+
+// A slot whose task never reached the queue has no owner to drain it. Without
+// the release the counter would stall at 2 while only one task can ever
+// decrement it, so post-process would never run and the row would sit in
+// "processing" until the housekeeping sweep failed it an hour later.
+func TestEnqueueImageMultimodalTasksReleasesUnownedSlots(t *testing.T) {
+ mr, rdb := newSlotReleaseRedis(t)
+ enqueuer := &slotReleaseEnqueuer{failAt: map[int]bool{0: true}}
+ svc := &knowledgeService{task: enqueuer, redisClient: rdb}
+ knowledge, kb, images, chunks := slotReleaseFixture(2)
+
+ svc.enqueueImageMultimodalTasks(context.Background(), knowledge, kb, images, chunks, nil)
+
+ require.Equal(t, 1, enqueuer.imageEnqueued)
+ require.Equal(t, "1", mustGet(t, mr, multimodalPendingKey(knowledge.ID)),
+ "the un-enqueued slot must be released so the surviving task can reach zero")
+ require.Equal(t, 0, enqueuer.postProcess,
+ "one task is still outstanding, so it owns the finalize")
+}
+
+// Same shortfall, but the surviving image finishes before the fan-out returns.
+// The release is then what brings the counter to zero, so it has to drive
+// post-process itself rather than leaving the row for the sweep.
+func TestEnqueueImageMultimodalTasksReleaseFinalizesWhenSiblingsAlreadyDone(t *testing.T) {
+ mr, rdb := newSlotReleaseRedis(t)
+ knowledge, kb, images, chunks := slotReleaseFixture(2)
+ enqueuer := &slotReleaseEnqueuer{failAt: map[int]bool{0: true}}
+ enqueuer.onImageEnqueued = func() {
+ // Mimic a worker that completes immediately: drain its own slot.
+ require.NoError(t, rdb.Decr(context.Background(),
+ multimodalPendingKey(knowledge.ID)).Err())
+ }
+ svc := &knowledgeService{task: enqueuer, redisClient: rdb}
+
+ svc.enqueueImageMultimodalTasks(context.Background(), knowledge, kb, images, chunks, nil)
+
+ require.Equal(t, 1, enqueuer.postProcess, "the release owns the finalize here")
+ require.False(t, mr.Exists(multimodalPendingKey(knowledge.ID)),
+ "counter should be cleared once it reaches zero")
+}
+
+// Nothing reached the queue at all, so no image task will ever finalize the
+// row whatever the counter says. Post-process has to be driven directly.
+func TestEnqueueImageMultimodalTasksFinalizesWhenNoTaskEnqueued(t *testing.T) {
+ mr, rdb := newSlotReleaseRedis(t)
+ enqueuer := &slotReleaseEnqueuer{failAt: map[int]bool{0: true, 1: true}}
+ svc := &knowledgeService{task: enqueuer, redisClient: rdb}
+ knowledge, kb, images, chunks := slotReleaseFixture(2)
+
+ svc.enqueueImageMultimodalTasks(context.Background(), knowledge, kb, images, chunks, nil)
+
+ require.Equal(t, 0, enqueuer.imageEnqueued)
+ require.Equal(t, 1, enqueuer.postProcess)
+ require.False(t, mr.Exists(multimodalPendingKey(knowledge.ID)))
+}
+
+// Lite mode: no Redis at all. A partial fan-out must not crash, and the
+// surviving task still drives post-process through the nil-client path in
+// checkAndFinalizeAllImages — so the fan-out must stay quiet.
+func TestEnqueueImageMultimodalTasksWithoutRedisLeavesFinalizeToWorkers(t *testing.T) {
+ enqueuer := &slotReleaseEnqueuer{failAt: map[int]bool{0: true}}
+ svc := &knowledgeService{task: enqueuer}
+ knowledge, kb, images, chunks := slotReleaseFixture(2)
+
+ svc.enqueueImageMultimodalTasks(context.Background(), knowledge, kb, images, chunks, nil)
+
+ require.Equal(t, 1, enqueuer.imageEnqueued)
+ require.Equal(t, 0, enqueuer.postProcess)
+}
+
+func mustGet(t *testing.T, mr *miniredis.Miniredis, key string) string {
+ t.Helper()
+ value, err := mr.Get(key)
+ require.NoError(t, err)
+ return value
+}
+
+// failingDecrByHook fails the first `failures` DECRBY commands, so a test can
+// drive the release retry deterministically instead of racing miniredis.
+type failingDecrByHook struct {
+ failures int
+ calls int
+}
+
+func (h *failingDecrByHook) DialHook(next redis.DialHook) redis.DialHook { return next }
+
+func (h *failingDecrByHook) ProcessPipelineHook(
+ next redis.ProcessPipelineHook,
+) redis.ProcessPipelineHook {
+ return next
+}
+
+func (h *failingDecrByHook) ProcessHook(next redis.ProcessHook) redis.ProcessHook {
+ return func(ctx context.Context, cmd redis.Cmder) error {
+ if cmd.Name() == "decrby" {
+ h.calls++
+ if h.calls <= h.failures {
+ err := errors.New("redis unavailable")
+ cmd.SetErr(err)
+ return err
+ }
+ }
+ return next(ctx, cmd)
+ }
+}
+
+// A transient Redis error must not end the release: it is the only thing that
+// can drain an un-owned slot, so giving up on the first error would strand the
+// row in "processing" exactly as before this fix.
+func TestEnqueueImageMultimodalTasksRetriesSlotRelease(t *testing.T) {
+ mr, rdb := newSlotReleaseRedis(t)
+ hook := &failingDecrByHook{failures: multimodalSlotReleaseAttempts - 1}
+ rdb.AddHook(hook)
+ enqueuer := &slotReleaseEnqueuer{failAt: map[int]bool{0: true}}
+ svc := &knowledgeService{task: enqueuer, redisClient: rdb}
+ knowledge, kb, images, chunks := slotReleaseFixture(2)
+
+ svc.enqueueImageMultimodalTasks(context.Background(), knowledge, kb, images, chunks, nil)
+
+ require.Equal(t, multimodalSlotReleaseAttempts, hook.calls, "every attempt should be used")
+ require.Equal(t, "1", mustGet(t, mr, multimodalPendingKey(knowledge.ID)),
+ "the retry that succeeded must still release the un-owned slot")
+}
+
+// When Redis stays down the release cannot happen, so the row is left to the
+// housekeeping sweep. Assert the bounded retry rather than an unbounded loop,
+// and that nothing is finalized on a counter we failed to settle.
+func TestEnqueueImageMultimodalTasksGivesUpSlotReleaseAfterRetries(t *testing.T) {
+ mr, rdb := newSlotReleaseRedis(t)
+ hook := &failingDecrByHook{failures: multimodalSlotReleaseAttempts + 5}
+ rdb.AddHook(hook)
+ enqueuer := &slotReleaseEnqueuer{failAt: map[int]bool{0: true}}
+ svc := &knowledgeService{task: enqueuer, redisClient: rdb}
+ knowledge, kb, images, chunks := slotReleaseFixture(2)
+
+ svc.enqueueImageMultimodalTasks(context.Background(), knowledge, kb, images, chunks, nil)
+
+ require.Equal(t, multimodalSlotReleaseAttempts, hook.calls, "retries must stay bounded")
+ require.Equal(t, "2", mustGet(t, mr, multimodalPendingKey(knowledge.ID)))
+ require.Equal(t, 0, enqueuer.postProcess,
+ "must not finalize on a counter the release could not settle")
+}
+
+// The fan-out can be reached with an already-cancelled context (graceful
+// shutdown). The nothing-enqueued branch still has to clear the seeded key, or
+// it lingers for its full 24h TTL; asynq's Enqueue takes no context, so
+// post-process still goes out.
+func TestEnqueueImageMultimodalTasksClearsCounterOnCancelledContext(t *testing.T) {
+ mr, rdb := newSlotReleaseRedis(t)
+ enqueuer := &slotReleaseEnqueuer{failAt: map[int]bool{0: true, 1: true}}
+ svc := &knowledgeService{task: enqueuer, redisClient: rdb}
+ knowledge, kb, images, chunks := slotReleaseFixture(2)
+ ctx, cancel := context.WithCancel(context.Background())
+ cancel()
+
+ svc.enqueueImageMultimodalTasks(ctx, knowledge, kb, images, chunks, nil)
+
+ require.False(t, mr.Exists(multimodalPendingKey(knowledge.ID)),
+ "seeded counter must be cleared on a detached context")
+ require.Equal(t, 1, enqueuer.postProcess)
+}
diff --git a/internal/application/service/knowledge_process.go b/internal/application/service/knowledge_process.go
index 648593473..10ed2f4f7 100644
--- a/internal/application/service/knowledge_process.go
+++ b/internal/application/service/knowledge_process.go
@@ -251,6 +251,53 @@ func markKnowledgeProcessing(knowledge *types.Knowledge, now time.Time) {
knowledge.UpdatedAt = now
}
+// failKnowledgeOnEmbeddingModel records an unresolvable embedding model as this
+// attempt's terminal state.
+//
+// It re-reads the row instead of trusting the entry guard, because resolving a
+// model reads the database: a cancel or a ReplaceKnowledgeFile can land in the
+// window between that guard and this write. Both would be clobbered otherwise —
+// UpdateKnowledge is a full-row Save and file_path is not among the omitted
+// columns, so a stale in-memory row would roll file_path back over the
+// replacement and mark the replacement's live attempt failed.
+func (s *knowledgeService) failKnowledgeOnEmbeddingModel(
+ ctx context.Context, kb *types.KnowledgeBase, knowledge *types.Knowledge, cause error,
+) {
+ // A cancelled or expired context means the run was interrupted — the user
+ // cancelled (asynq CancelProcessing cancels the handler context), the
+ // worker was preempted, the process is shutting down. The model itself is
+ // not implicated, so recording "failed to get embedding model" would both
+ // mislabel the cause and overwrite the cancelled status the abort path
+ // just wrote. Leave the row to the abort path, or to the housekeeping
+ // sweep when nothing else claims it.
+ if ctx.Err() != nil || errors.Is(cause, context.Canceled) || errors.Is(cause, context.DeadlineExceeded) {
+ logger.Infof(ctx,
+ "Embedding model resolution interrupted for %s (%v); leaving parse status untouched",
+ knowledge.ID, cause)
+ return
+ }
+ if aborted, status := s.isKnowledgeAborted(ctx, knowledge.TenantID, knowledge.ID); aborted {
+ logger.Infof(ctx,
+ "Knowledge aborted (%s), not recording embedding model failure: %s", status, knowledge.ID)
+ return
+ }
+
+ knowledge.ParseStatus = types.ParseStatusFailed
+ knowledge.ErrorMessage = fmt.Sprintf("failed to get embedding model: %v", cause)
+ knowledge.UpdatedAt = time.Now()
+ if err := s.updateKnowledgeUnlessSourceReplaced(ctx, knowledge); err != nil {
+ logger.Errorf(ctx, "failed to persist embedding model failure for %s: %v", knowledge.ID, err)
+ }
+ s.beginStage(ctx, knowledge.ID, types.StageEmbedding, types.JSONMap{
+ "model_id": kb.EmbeddingModelID,
+ })
+ // The span's error_message is what the timeline renders, and its
+ // error_detail is withheld from non-admin responses — so carry the same
+ // text the document list shows rather than a fixed generic string.
+ s.failStage(ctx, knowledge.ID, types.StageEmbedding,
+ werrors.ErrCodeEmbeddingProviderFail, knowledge.ErrorMessage, cause)
+}
+
// buildSplitterConfigFromChunking normalizes effective chunking settings with
// the shared chunker defaults.
func buildSplitterConfigFromChunking(cc types.ChunkingConfig) chunker.SplitterConfig {
@@ -320,7 +367,19 @@ func (s *knowledgeService) processChunks(ctx context.Context,
var err error
embeddingModel, err = s.modelService.GetEmbeddingModel(ctx, kb.EmbeddingModelID)
if err != nil {
+ // Terminal for this attempt, and it has to be recorded as such.
+ // A KB that indexes vectors cannot proceed without an embedder,
+ // and processChunks reports nothing to its callers — returning
+ // silently leaves parse_status on "processing" with no task left
+ // to move it, so the row hangs until the housekeeping sweep fails
+ // it an hour later with a generic "stuck in processing" message.
+ // Failing here names the real cause instead: a model row that was
+ // deleted or deactivated, or a configuration the embedder factory
+ // refuses. (Credential and connectivity problems surface later, on
+ // the first BatchIndex round-trip — resolving a model only reads
+ // the row and constructs a client.)
logger.GetLogger(ctx).WithField("error", err).Errorf("processChunks get embedding model failed")
+ s.failKnowledgeOnEmbeddingModel(ctx, kb, knowledge, err)
return
}
} else {
@@ -715,27 +774,7 @@ func (s *knowledgeService) processChunks(ctx context.Context,
} else {
s.skipStage(ctx, knowledge.ID, types.StageMultimodal, "skipped")
// If there are no multimodal tasks, enqueue the post process task immediately
- lang := types.LanguageFromContextOrDefault(ctx)
- postProcessPayload := types.KnowledgePostProcessPayload{
- TenantID: knowledge.TenantID,
- KnowledgeID: knowledge.ID,
- KnowledgeBaseID: knowledge.KnowledgeBaseID,
- Language: lang,
- Attempt: attemptFromCtx(ctx),
- }
- langfuse.InjectTracing(ctx, &postProcessPayload)
- payloadBytes, err := json.Marshal(postProcessPayload)
- if err == nil {
- task := asynq.NewTask(types.TypeKnowledgePostProcess, payloadBytes,
- knowledgePostProcessTaskOptions()...)
- if _, err := s.task.Enqueue(task); err != nil {
- logger.Errorf(ctx, "Failed to enqueue knowledge post process task: %v", err)
- } else {
- logger.Infof(ctx, "Enqueued knowledge post process task for %s", knowledge.ID)
- }
- } else {
- logger.Errorf(ctx, "Failed to marshal knowledge post process payload: %v", err)
- }
+ s.enqueueKnowledgePostProcessTask(ctx, knowledge)
}
// Update tenant's storage usage
@@ -4068,13 +4107,23 @@ func (s *knowledgeService) enqueueImageMultimodalTasks(
}
attempt := attemptFromCtx(ctx)
- redisKey := fmt.Sprintf("multimodal:pending:%s", knowledge.ID)
+ lang := types.LanguageFromContextOrDefault(ctx)
+ redisKey := multimodalPendingKey(knowledge.ID)
+ // The counter has to be seeded BEFORE the first task is enqueued: an image
+ // task that finishes early would otherwise decrement a key we are about to
+ // overwrite, and its slot would be counted twice. The cost of seeding
+ // first is that a slot whose task never reaches the queue has no owner to
+ // drain it — releaseUnownedMultimodalSlots settles that after the fan-out.
+ counterSeeded := false
if s.redisClient != nil {
if err := s.redisClient.Set(ctx, redisKey, len(images), 24*time.Hour).Err(); err != nil {
logger.Warnf(ctx, "Failed to set multimodal pending count for %s: %v", knowledge.ID, err)
+ } else {
+ counterSeeded = true
}
}
+ enqueued := 0
for idx, img := range images {
// Match image to the ParsedChunk whose content contains the image URL.
// ChunkID was populated by processChunks with the real DB UUID.
@@ -4089,7 +4138,6 @@ func (s *knowledgeService) enqueueImageMultimodalTasks(
chunkID = chunks[0].ChunkID
}
- lang := types.LanguageFromContextOrDefault(ctx)
payload := types.ImageMultimodalPayload{
TenantID: knowledge.TenantID,
KnowledgeID: knowledge.ID,
@@ -4115,10 +4163,171 @@ func (s *knowledgeService) enqueueImageMultimodalTasks(
asynq.Queue(types.QueueMultimodal), asynq.MaxRetry(3), asynq.Timeout(30*time.Minute))
if _, err := s.task.Enqueue(task); err != nil {
logger.Warnf(ctx, "Failed to enqueue image multimodal task for %s: %v", img.ServingURL, err)
- } else {
- logger.Infof(ctx, "Enqueued image:multimodal task for %s", img.ServingURL)
+ continue
}
+ enqueued++
+ logger.Infof(ctx, "Enqueued image:multimodal task for %s", img.ServingURL)
+ }
+
+ s.releaseUnownedMultimodalSlots(
+ ctx, knowledge, redisKey, counterSeeded, len(images), enqueued,
+ )
+}
+
+// releaseUnownedMultimodalSlots settles the multimodal fan-in counter against
+// what actually reached the queue.
+//
+// Every enqueued image task drains exactly one slot when it exits terminally,
+// and the knowledge only leaves "processing" once the counter reaches zero. A
+// slot whose task was never enqueued (payload marshal failure, asynq refusing
+// the write) has no owner, so without this release the counter stalls above
+// zero, post-process is never triggered, and the row sits in "processing"
+// until the housekeeping sweep fails it an hour later — which reaches the user
+// as an unexplained parse failure. This mirrors the shortfall release
+// KnowledgePostProcess already performs for its own enrichment subtasks.
+func (s *knowledgeService) releaseUnownedMultimodalSlots(
+ ctx context.Context,
+ knowledge *types.Knowledge,
+ redisKey string,
+ counterSeeded bool,
+ planned, enqueued int,
+) {
+ if enqueued == 0 {
+ // No task exists to finalize this knowledge, whatever the counter
+ // says. Drive post-process directly so the row completes on the
+ // chunks that were already indexed instead of waiting to be swept.
+ logger.Warnf(ctx,
+ "[ImageMultimodal] No image task enqueued for %s (planned=%d); enqueueing post-process directly",
+ knowledge.ID, planned)
+ if counterSeeded {
+ // Detached like the shortfall path below: a cancelled parent
+ // context would otherwise leave the seeded key to expire on its
+ // own 24h TTL.
+ dctx, cancel := context.WithTimeout(
+ context.WithoutCancel(ctx), finalizeSubtaskDetachedTimeout)
+ s.redisClient.Del(dctx, redisKey)
+ cancel()
+ }
+ s.enqueueKnowledgePostProcessTask(ctx, knowledge)
+ return
+ }
+
+ shortfall := planned - enqueued
+ if shortfall <= 0 {
+ return
+ }
+ if !counterSeeded {
+ // Seeding failed, so the key is absent and the first image to finish
+ // already drives post-process through the missing-key fallback in
+ // checkAndFinalizeAllImages. Decrementing here would only create the
+ // key below zero and finalize before the siblings are done.
+ return
+ }
+
+ logger.Warnf(ctx,
+ "[ImageMultimodal] Releasing %d un-enqueued image slot(s) for %s (planned=%d enqueued=%d)",
+ shortfall, knowledge.ID, planned, enqueued)
+ // Detached ctx, for the same reason KnowledgePostProcess detaches its own
+ // releases: these slots have no other path to drain, so a parent context
+ // cancelled mid-fan-out (graceful shutdown) must not skip the release.
+ rctx, cancel := context.WithTimeout(context.WithoutCancel(ctx), finalizeSubtaskDetachedTimeout)
+ defer cancel()
+ pending, err := s.decrMultimodalSlots(rctx, redisKey, int64(shortfall))
+ if err != nil {
+ // Giving up here puts the row back in the state this whole fix exists
+ // to remove: a counter no task can drain, recovered only by the
+ // housekeeping sweep an hour later. Say so at error level so it is
+ // greppable, rather than hiding it in a warning.
+ logger.Errorf(ctx,
+ "Failed to release %d multimodal slot(s) for %s after %d attempts: %v; "+
+ "row will be left to the housekeeping sweep",
+ shortfall, knowledge.ID, multimodalSlotReleaseAttempts, err)
+ return
+ }
+ if pending <= 0 {
+ // Every enqueued sibling had already finished, so the release owns the
+ // finalize. Ordering is safe either way: the slots we just drained have
+ // no task behind them, so exactly one path reaches zero.
+ s.redisClient.Del(rctx, redisKey)
+ s.enqueueKnowledgePostProcessTask(ctx, knowledge)
+ }
+}
+
+// multimodalSlotReleaseAttempts bounds the retry on the release decrement.
+//
+// The release is the only thing that can ever drain an un-owned slot, so a
+// single transient Redis error must not be the end of it — that would strand
+// the row exactly as it did before this fix. Attempts stay small because they
+// share the detached context's budget, and the housekeeping sweep is still the
+// backstop when Redis is genuinely down.
+//
+// DECRBY is not idempotent, so a retry after a lost reply can over-decrement.
+// That is the safe direction to fail in: an over-decrement finalizes early
+// (post-process may miss some OCR / caption chunks, and repeat deliveries are
+// already handled by its non-processing-status branch), whereas not retrying
+// strands the document in "processing" for an hour.
+const multimodalSlotReleaseAttempts = 3
+
+// multimodalSlotReleaseBackoff is the unit of linear backoff between release
+// attempts, small enough that all attempts fit the detached 10s budget.
+const multimodalSlotReleaseBackoff = 100 * time.Millisecond
+
+// decrMultimodalSlots decrements the fan-in counter by `by`, retrying transient
+// Redis failures, and returns the resulting count.
+func (s *knowledgeService) decrMultimodalSlots(
+ ctx context.Context, redisKey string, by int64,
+) (int64, error) {
+ var lastErr error
+ for attempt := 1; attempt <= multimodalSlotReleaseAttempts; attempt++ {
+ pending, err := s.redisClient.DecrBy(ctx, redisKey, by).Result()
+ if err == nil {
+ return pending, nil
+ }
+ lastErr = err
+ if attempt == multimodalSlotReleaseAttempts {
+ break
+ }
+ logger.Warnf(ctx, "multimodal slot release attempt %d/%d failed for %s: %v",
+ attempt, multimodalSlotReleaseAttempts, redisKey, err)
+ select {
+ case <-time.After(time.Duration(attempt) * multimodalSlotReleaseBackoff):
+ case <-ctx.Done():
+ return 0, ctx.Err()
+ }
+ }
+ return 0, lastErr
+}
+
+// enqueueKnowledgePostProcessTask hands a knowledge to the post-process
+// orchestrator, which owns the enrichment fan-out and the promotion out of
+// "processing". Shared by the no-multimodal path and by the multimodal slot
+// release, so both build the same payload (attempt, language, tracing).
+func (s *knowledgeService) enqueueKnowledgePostProcessTask(
+ ctx context.Context, knowledge *types.Knowledge,
+) {
+ if s.task == nil {
+ return
+ }
+ payload := types.KnowledgePostProcessPayload{
+ TenantID: knowledge.TenantID,
+ KnowledgeID: knowledge.ID,
+ KnowledgeBaseID: knowledge.KnowledgeBaseID,
+ Language: types.LanguageFromContextOrDefault(ctx),
+ Attempt: attemptFromCtx(ctx),
+ }
+ langfuse.InjectTracing(ctx, &payload)
+ payloadBytes, err := json.Marshal(payload)
+ if err != nil {
+ logger.Errorf(ctx, "Failed to marshal knowledge post process payload: %v", err)
+ return
+ }
+ task := asynq.NewTask(types.TypeKnowledgePostProcess, payloadBytes,
+ knowledgePostProcessTaskOptions()...)
+ if _, err := s.task.Enqueue(task); err != nil {
+ logger.Errorf(ctx, "Failed to enqueue knowledge post process task: %v", err)
+ return
}
+ logger.Infof(ctx, "Enqueued knowledge post process task for %s", knowledge.ID)
}
// ProcessKnowledgeListReparse handles Asynq knowledge list reparse tasks.
diff --git a/internal/application/service/knowledge_process_embedding_failure_test.go b/internal/application/service/knowledge_process_embedding_failure_test.go
new file mode 100644
index 000000000..238ed02fa
--- /dev/null
+++ b/internal/application/service/knowledge_process_embedding_failure_test.go
@@ -0,0 +1,257 @@
+package service
+
+import (
+ "context"
+ "errors"
+ "fmt"
+ "strings"
+ "testing"
+
+ "github.com/Tencent/WeKnora/internal/models/embedding"
+ "github.com/Tencent/WeKnora/internal/types"
+ "github.com/Tencent/WeKnora/internal/types/interfaces"
+ "github.com/stretchr/testify/require"
+)
+
+type embedFailureKnowledgeRepo struct {
+ interfaces.KnowledgeRepository
+ knowledge *types.Knowledge
+ // stored is what the database starts returning once storedAfterReads
+ // reads have been served. The delay matters: processChunks already
+ // guards its entry, so a test that returns the cancelled / replaced row
+ // from the very first read never reaches the code under test. Delaying
+ // it models the real window — resolving an embedding model reads the
+ // database, so a cancel or a replacement can land while it runs.
+ stored *types.Knowledge
+ storedAfterReads int
+ reads int
+ updates []types.Knowledge
+}
+
+func (r *embedFailureKnowledgeRepo) GetKnowledgeByID(
+ context.Context, uint64, string,
+) (*types.Knowledge, error) {
+ r.reads++
+ if r.stored != nil && r.reads > r.storedAfterReads {
+ return r.stored, nil
+ }
+ return r.knowledge, nil
+}
+
+func (r *embedFailureKnowledgeRepo) UpdateKnowledge(_ context.Context, k *types.Knowledge) error {
+ r.updates = append(r.updates, *k)
+ return nil
+}
+
+type embedFailureModelService struct {
+ interfaces.ModelService
+ err error
+}
+
+func (s embedFailureModelService) GetEmbeddingModel(
+ context.Context, string,
+) (embedding.Embedder, error) {
+ return nil, s.err
+}
+
+// A KB that indexes vectors cannot proceed without an embedder, and
+// processChunks reports nothing to its callers — so this failure has to be
+// persisted here or the row stays "processing" with no task left to move it,
+// visible to the user only as an hour-long spinner followed by a generic
+// housekeeping failure.
+//
+// chunkRepo / graphEngine / retrieveEngine are deliberately left nil: any
+// attempt to keep processing past the missing embedder would panic and fail
+// this test, which is exactly the regression worth catching.
+func TestProcessChunksFailsKnowledgeWhenEmbeddingModelUnavailable(t *testing.T) {
+ modelErr := errors.New("embedding model not found")
+ knowledge := &types.Knowledge{
+ ID: "knowledge-1",
+ TenantID: 1,
+ KnowledgeBaseID: "kb-1",
+ ParseStatus: types.ParseStatusProcessing,
+ }
+ repo := &embedFailureKnowledgeRepo{knowledge: knowledge}
+ svc := &knowledgeService{
+ repo: repo,
+ modelService: embedFailureModelService{err: modelErr},
+ }
+ kb := &types.KnowledgeBase{
+ ID: "kb-1",
+ TenantID: 1,
+ EmbeddingModelID: "embedding-1",
+ IndexingStrategy: types.IndexingStrategy{VectorEnabled: true},
+ }
+
+ svc.processChunks(context.Background(), kb, knowledge,
+ []types.ParsedChunk{{Content: "body", Seq: 0, Start: 0, End: 4}})
+
+ require.Len(t, repo.updates, 1, "the failure must be persisted exactly once")
+ require.Equal(t, types.ParseStatusFailed, repo.updates[0].ParseStatus)
+ require.True(t,
+ strings.Contains(repo.updates[0].ErrorMessage, modelErr.Error()),
+ "error_message should name the underlying cause, got %q", repo.updates[0].ErrorMessage,
+ )
+ require.Equal(t, types.ParseStatusFailed, knowledge.ParseStatus)
+}
+
+// A cancelled context means the run was interrupted, not that the model is
+// unusable — the user cancelled (asynq CancelProcessing cancels the handler
+// context), the worker was preempted, or the process is shutting down.
+// Recording a model failure here would both mislabel the cause and overwrite
+// the cancelled status the abort path just wrote.
+func TestProcessChunksLeavesStatusAloneWhenEmbeddingModelCallIsCancelled(t *testing.T) {
+ for _, tc := range []struct {
+ name string
+ modelErr error
+ cancelCtx bool
+ }{
+ {name: "error is context.Canceled", modelErr: context.Canceled},
+ {name: "error wraps context.DeadlineExceeded", modelErr: fmt.Errorf("query: %w", context.DeadlineExceeded)},
+ {name: "context already cancelled", modelErr: errors.New("driver: bad connection"), cancelCtx: true},
+ } {
+ t.Run(tc.name, func(t *testing.T) {
+ knowledge := &types.Knowledge{
+ ID: "knowledge-1",
+ TenantID: 1,
+ KnowledgeBaseID: "kb-1",
+ ParseStatus: types.ParseStatusProcessing,
+ }
+ repo := &embedFailureKnowledgeRepo{knowledge: knowledge}
+ svc := &knowledgeService{
+ repo: repo,
+ modelService: embedFailureModelService{err: tc.modelErr},
+ }
+ kb := &types.KnowledgeBase{
+ ID: "kb-1",
+ TenantID: 1,
+ EmbeddingModelID: "embedding-1",
+ IndexingStrategy: types.IndexingStrategy{VectorEnabled: true},
+ }
+ ctx := context.Background()
+ if tc.cancelCtx {
+ cancelled, cancel := context.WithCancel(ctx)
+ cancel()
+ ctx = cancelled
+ }
+
+ svc.processChunks(ctx, kb, knowledge,
+ []types.ParsedChunk{{Content: "body", Seq: 0, Start: 0, End: 4}})
+
+ require.Empty(t, repo.updates, "an interrupted run must not be written as a failure")
+ require.Equal(t, types.ParseStatusProcessing, knowledge.ParseStatus)
+ })
+ }
+}
+
+// The row was cancelled after the entry guard ran. Resolving a model reads the
+// database, so that window is real — and the cancelled status must survive it.
+func TestProcessChunksDoesNotOverwriteCancelledRowOnEmbeddingFailure(t *testing.T) {
+ knowledge := &types.Knowledge{
+ ID: "knowledge-1",
+ TenantID: 1,
+ KnowledgeBaseID: "kb-1",
+ ParseStatus: types.ParseStatusProcessing,
+ }
+ repo := &embedFailureKnowledgeRepo{
+ knowledge: knowledge,
+ // The entry guard's read still sees a live row; the cancel lands
+ // while the embedding model is being resolved.
+ storedAfterReads: 1,
+ stored: &types.Knowledge{
+ ID: "knowledge-1",
+ TenantID: 1,
+ ParseStatus: types.ParseStatusCancelled,
+ },
+ }
+ svc := &knowledgeService{
+ repo: repo,
+ modelService: embedFailureModelService{err: errors.New("embedding model not found")},
+ }
+ kb := &types.KnowledgeBase{
+ ID: "kb-1",
+ TenantID: 1,
+ EmbeddingModelID: "embedding-1",
+ IndexingStrategy: types.IndexingStrategy{VectorEnabled: true},
+ }
+
+ svc.processChunks(context.Background(), kb, knowledge,
+ []types.ParsedChunk{{Content: "body", Seq: 0, Start: 0, End: 4}})
+
+ require.Greater(t, repo.reads, 1, "the guard must re-read after resolving the model")
+ require.Empty(t, repo.updates, "cancelled must not be overwritten with failed")
+}
+
+// ReplaceKnowledgeFile keeps the knowledge ID and swaps file_path, so a stale
+// worker must not persist its row: UpdateKnowledge is a full-row Save and
+// file_path is not omitted, so it would roll the path back over the
+// replacement and mark the replacement's live attempt failed.
+func TestProcessChunksSkipsEmbeddingFailureWriteWhenSourceReplaced(t *testing.T) {
+ knowledge := &types.Knowledge{
+ ID: "knowledge-1",
+ TenantID: 1,
+ KnowledgeBaseID: "kb-1",
+ ParseStatus: types.ParseStatusProcessing,
+ FilePath: "tenant/1/old.pdf",
+ }
+ repo := &embedFailureKnowledgeRepo{
+ knowledge: knowledge,
+ // Reads 1-2 are the entry guard (aborted + source-replaced) and still
+ // see this worker's own file; the replacement lands while the
+ // embedding model is being resolved.
+ storedAfterReads: 2,
+ stored: &types.Knowledge{
+ ID: "knowledge-1",
+ TenantID: 1,
+ ParseStatus: types.ParseStatusProcessing,
+ FilePath: "tenant/1/replacement.pdf",
+ },
+ }
+ svc := &knowledgeService{
+ repo: repo,
+ modelService: embedFailureModelService{err: errors.New("embedding model not found")},
+ }
+ kb := &types.KnowledgeBase{
+ ID: "kb-1",
+ TenantID: 1,
+ EmbeddingModelID: "embedding-1",
+ IndexingStrategy: types.IndexingStrategy{VectorEnabled: true},
+ }
+
+ svc.processChunks(context.Background(), kb, knowledge,
+ []types.ParsedChunk{{Content: "body", Seq: 0, Start: 0, End: 4}})
+
+ require.Greater(t, repo.reads, 2, "the guard must re-read after resolving the model")
+ require.Empty(t, repo.updates,
+ "a replaced source must not have this attempt's failure written back")
+}
+
+// A KB with vector and keyword indexing both off never resolves an embedder,
+// so the guard above must not fire for it.
+func TestProcessChunksSkipsEmbeddingModelWhenIndexingDisabled(t *testing.T) {
+ knowledge := &types.Knowledge{
+ ID: "knowledge-1",
+ TenantID: 1,
+ KnowledgeBaseID: "kb-1",
+ ParseStatus: types.ParseStatusProcessing,
+ }
+ chunkRepo := &parentChildChunkService{}
+ tenant := &types.Tenant{ID: 1}
+ ctx := context.WithValue(context.Background(), types.TenantInfoContextKey, tenant)
+ svc := &knowledgeService{
+ repo: &embedFailureKnowledgeRepo{knowledge: knowledge},
+ chunkRepo: chunkRepo,
+ modelService: embedFailureModelService{err: errors.New("must not be called")},
+ graphEngine: parentChildGraphRepo{},
+ tenantRepo: parentChildTenantRepo{},
+ task: parentChildTaskEnqueuer{},
+ }
+ kb := &types.KnowledgeBase{ID: "kb-1", TenantID: 1}
+ require.False(t, kb.NeedsEmbeddingModel())
+
+ svc.processChunks(ctx, kb, knowledge,
+ []types.ParsedChunk{{Content: "body", Seq: 0, Start: 0, End: 4}})
+
+ require.Len(t, chunkRepo.created, 1, "chunks are persisted even without vector indexing")
+ require.NotEqual(t, types.ParseStatusFailed, knowledge.ParseStatus)
+}