Open-source LLM knowledge platform: turn raw documents into a queryable RAG, an autonomous reasoning agent, and a self-maintaining Wiki.
Go agentagenticaichatbotdsh-pluginembeddingsevaluationgenerative-aigolangknowledge-basellmmulti-tenantollamaopenaiquestion-answeringragrerankingsemantic-searchvector-searchwiki
fix(knowledge): stop two parse paths from stranding a row in processing (#3445)
lyingbugeaca7364cd77
diff --git a/internal/application/service/image_multimodal.go b/internal/application/service/image_multimodal.goindex 631425f03..bd8734404 100644--- a/internal/application/service/image_multimodal.go+++ b/internal/application/service/image_multimodal.go@@ -659,13 +659,21 @@ func downloadImageFromURL(imageURL string) ([]byte, error) {return secutils.DownloadBytes(imageURL)}+// multimodalPendingKey is the Redis key holding how many image tasks a+// knowledge is still waiting on. The fan-out (enqueueImageMultimodalTasks)+// seeds it and the fan-in (checkAndFinalizeAllImages) drains it, so the two+// must agree on the name — hence one helper rather than two format strings.+func multimodalPendingKey(knowledgeID string) string {+ return fmt.Sprintf("multimodal:pending:%s", knowledgeID)+}+func (s *ImageMultimodalService) checkAndFinalizeAllImages(ctx context.Context, payload types.ImageMultimodalPayload) {if s.redisClient == nil {s.enqueueKnowledgePostProcessTask(ctx, payload)return}- redisKey := fmt.Sprintf("multimodal:pending:%s", payload.KnowledgeID)+ redisKey := multimodalPendingKey(payload.KnowledgeID)pendingCount, err := s.redisClient.Decr(ctx, redisKey).Result()if err != nil && err != redis.Nil {diff --git a/internal/application/service/knowledge_multimodal_slot_release_test.go b/internal/application/service/knowledge_multimodal_slot_release_test.gonew file mode 100644index 000000000..2ace05217--- /dev/null+++ b/internal/application/service/knowledge_multimodal_slot_release_test.go@@ -0,0 +1,254 @@+package service++import (+ "context"+ "errors"+ "testing"++ "github.com/Tencent/WeKnora/internal/infrastructure/docparser"+ "github.com/Tencent/WeKnora/internal/types"+ "github.com/alicebob/miniredis/v2"+ "github.com/hibiken/asynq"+ "github.com/redis/go-redis/v9"+ "github.com/stretchr/testify/require"+)++// slotReleaseEnqueuer fails the image tasks whose index is in failAt, so a+// test can reproduce a fan-out that only partially reaches the queue.+//+// onImageEnqueued lets a test simulate an image worker that finishes before+// the fan-out loop returns — the ordering that decides whether the shortfall+// release or the last sibling owns the finalize.+type slotReleaseEnqueuer struct {+ failAt map[int]bool+ imageSeen int+ imageEnqueued int+ postProcess int+ onImageEnqueued func()+}++func (e *slotReleaseEnqueuer) Enqueue(task *asynq.Task, _ ...asynq.Option) (*asynq.TaskInfo, error) {+ switch task.Type() {+ case types.TypeImageMultimodal:+ idx := e.imageSeen+ e.imageSeen+++ if e.failAt[idx] {+ return nil, errors.New("queue write refused")+ }+ e.imageEnqueued+++ if e.onImageEnqueued != nil {+ e.onImageEnqueued()+ }+ return &asynq.TaskInfo{ID: "image"}, nil+ case types.TypeKnowledgePostProcess:+ e.postProcess+++ return &asynq.TaskInfo{ID: "post-process"}, nil+ }+ return &asynq.TaskInfo{ID: "other"}, nil+}++func newSlotReleaseRedis(t *testing.T) (*miniredis.Miniredis, *redis.Client) {+ t.Helper()+ mr, err := miniredis.Run()+ require.NoError(t, err)+ t.Cleanup(mr.Close)+ rdb := redis.NewClient(&redis.Options{Addr: mr.Addr()})+ t.Cleanup(func() { _ = rdb.Close() })+ return mr, rdb+}++func slotReleaseFixture(images int) (+ *types.Knowledge, *types.KnowledgeBase, []docparser.StoredImage, []types.ParsedChunk,+) {+ knowledge := &types.Knowledge{+ ID: "knowledge-1",+ TenantID: 1,+ KnowledgeBaseID: "kb-1",+ ParseStatus: types.ParseStatusProcessing,+ }+ kb := &types.KnowledgeBase{ID: "kb-1", TenantID: 1}+ stored := make([]docparser.StoredImage, 0, images)+ for i := 0; i < images; i++ {+ stored = append(stored, docparser.StoredImage{+ OriginalRef: "images/page.png",+ ServingURL: "local://images/page-" + string(rune('a'+i)) + ".png",+ })+ }+ return knowledge, kb, stored, []types.ParsedChunk{{ChunkID: "chunk-1", Content: "body"}}+}++// The happy path must leave the counter at exactly the number of enqueued+// tasks and must NOT enqueue post-process — the last image to finish owns it.+func TestEnqueueImageMultimodalTasksSeedsCounterForEveryImage(t *testing.T) {+ mr, rdb := newSlotReleaseRedis(t)+ enqueuer := &slotReleaseEnqueuer{}+ svc := &knowledgeService{task: enqueuer, redisClient: rdb}+ knowledge, kb, images, chunks := slotReleaseFixture(3)++ svc.enqueueImageMultimodalTasks(context.Background(), knowledge, kb, images, chunks, nil)++ require.Equal(t, 3, enqueuer.imageEnqueued)+ require.Equal(t, 0, enqueuer.postProcess)+ require.Equal(t, "3", mustGet(t, mr, multimodalPendingKey(knowledge.ID)))+}++// A slot whose task never reached the queue has no owner to drain it. Without+// the release the counter would stall at 2 while only one task can ever+// decrement it, so post-process would never run and the row would sit in+// "processing" until the housekeeping sweep failed it an hour later.+func TestEnqueueImageMultimodalTasksReleasesUnownedSlots(t *testing.T) {+ mr, rdb := newSlotReleaseRedis(t)+ enqueuer := &slotReleaseEnqueuer{failAt: map[int]bool{0: true}}+ svc := &knowledgeService{task: enqueuer, redisClient: rdb}+ knowledge, kb, images, chunks := slotReleaseFixture(2)++ svc.enqueueImageMultimodalTasks(context.Background(), knowledge, kb, images, chunks, nil)++ require.Equal(t, 1, enqueuer.imageEnqueued)+ require.Equal(t, "1", mustGet(t, mr, multimodalPendingKey(knowledge.ID)),+ "the un-enqueued slot must be released so the surviving task can reach zero")+ require.Equal(t, 0, enqueuer.postProcess,+ "one task is still outstanding, so it owns the finalize")+}++// Same shortfall, but the surviving image finishes before the fan-out returns.+// The release is then what brings the counter to zero, so it has to drive+// post-process itself rather than leaving the row for the sweep.+func TestEnqueueImageMultimodalTasksReleaseFinalizesWhenSiblingsAlreadyDone(t *testing.T) {+ mr, rdb := newSlotReleaseRedis(t)+ knowledge, kb, images, chunks := slotReleaseFixture(2)+ enqueuer := &slotReleaseEnqueuer{failAt: map[int]bool{0: true}}+ enqueuer.onImageEnqueued = func() {+ // Mimic a worker that completes immediately: drain its own slot.+ require.NoError(t, rdb.Decr(context.Background(),+ multimodalPendingKey(knowledge.ID)).Err())+ }+ svc := &knowledgeService{task: enqueuer, redisClient: rdb}++ svc.enqueueImageMultimodalTasks(context.Background(), knowledge, kb, images, chunks, nil)++ require.Equal(t, 1, enqueuer.postProcess, "the release owns the finalize here")+ require.False(t, mr.Exists(multimodalPendingKey(knowledge.ID)),+ "counter should be cleared once it reaches zero")+}++// Nothing reached the queue at all, so no image task will ever finalize the+// row whatever the counter says. Post-process has to be driven directly.+func TestEnqueueImageMultimodalTasksFinalizesWhenNoTaskEnqueued(t *testing.T) {+ mr, rdb := newSlotReleaseRedis(t)+ enqueuer := &slotReleaseEnqueuer{failAt: map[int]bool{0: true, 1: true}}+ svc := &knowledgeService{task: enqueuer, redisClient: rdb}+ knowledge, kb, images, chunks := slotReleaseFixture(2)++ svc.enqueueImageMultimodalTasks(context.Background(), knowledge, kb, images, chunks, nil)++ require.Equal(t, 0, enqueuer.imageEnqueued)+ require.Equal(t, 1, enqueuer.postProcess)+ require.False(t, mr.Exists(multimodalPendingKey(knowledge.ID)))+}++// Lite mode: no Redis at all. A partial fan-out must not crash, and the+// surviving task still drives post-process through the nil-client path in+// checkAndFinalizeAllImages — so the fan-out must stay quiet.+func TestEnqueueImageMultimodalTasksWithoutRedisLeavesFinalizeToWorkers(t *testing.T) {+ enqueuer := &slotReleaseEnqueuer{failAt: map[int]bool{0: true}}+ svc := &knowledgeService{task: enqueuer}+ knowledge, kb, images, chunks := slotReleaseFixture(2)++ svc.enqueueImageMultimodalTasks(context.Background(), knowledge, kb, images, chunks, nil)++ require.Equal(t, 1, enqueuer.imageEnqueued)+ require.Equal(t, 0, enqueuer.postProcess)+}++func mustGet(t *testing.T, mr *miniredis.Miniredis, key string) string {+ t.Helper()+ value, err := mr.Get(key)+ require.NoError(t, err)+ return value+}++// failingDecrByHook fails the first `failures` DECRBY commands, so a test can+// drive the release retry deterministically instead of racing miniredis.+type failingDecrByHook struct {+ failures int+ calls int+}++func (h *failingDecrByHook) DialHook(next redis.DialHook) redis.DialHook { return next }++func (h *failingDecrByHook) ProcessPipelineHook(+ next redis.ProcessPipelineHook,+) redis.ProcessPipelineHook {+ return next+}++func (h *failingDecrByHook) ProcessHook(next redis.ProcessHook) redis.ProcessHook {+ return func(ctx context.Context, cmd redis.Cmder) error {+ if cmd.Name() == "decrby" {+ h.calls+++ if h.calls <= h.failures {+ err := errors.New("redis unavailable")+ cmd.SetErr(err)+ return err+ }+ }+ return next(ctx, cmd)+ }+}++// A transient Redis error must not end the release: it is the only thing that+// can drain an un-owned slot, so giving up on the first error would strand the+// row in "processing" exactly as before this fix.+func TestEnqueueImageMultimodalTasksRetriesSlotRelease(t *testing.T) {+ mr, rdb := newSlotReleaseRedis(t)+ hook := &failingDecrByHook{failures: multimodalSlotReleaseAttempts - 1}+ rdb.AddHook(hook)+ enqueuer := &slotReleaseEnqueuer{failAt: map[int]bool{0: true}}+ svc := &knowledgeService{task: enqueuer, redisClient: rdb}+ knowledge, kb, images, chunks := slotReleaseFixture(2)++ svc.enqueueImageMultimodalTasks(context.Background(), knowledge, kb, images, chunks, nil)++ require.Equal(t, multimodalSlotReleaseAttempts, hook.calls, "every attempt should be used")+ require.Equal(t, "1", mustGet(t, mr, multimodalPendingKey(knowledge.ID)),+ "the retry that succeeded must still release the un-owned slot")+}++// When Redis stays down the release cannot happen, so the row is left to the+// housekeeping sweep. Assert the bounded retry rather than an unbounded loop,+// and that nothing is finalized on a counter we failed to settle.+func TestEnqueueImageMultimodalTasksGivesUpSlotReleaseAfterRetries(t *testing.T) {+ mr, rdb := newSlotReleaseRedis(t)+ hook := &failingDecrByHook{failures: multimodalSlotReleaseAttempts + 5}+ rdb.AddHook(hook)+ enqueuer := &slotReleaseEnqueuer{failAt: map[int]bool{0: true}}+ svc := &knowledgeService{task: enqueuer, redisClient: rdb}+ knowledge, kb, images, chunks := slotReleaseFixture(2)++ svc.enqueueImageMultimodalTasks(context.Background(), knowledge, kb, images, chunks, nil)++ require.Equal(t, multimodalSlotReleaseAttempts, hook.calls, "retries must stay bounded")+ require.Equal(t, "2", mustGet(t, mr, multimodalPendingKey(knowledge.ID)))+ require.Equal(t, 0, enqueuer.postProcess,+ "must not finalize on a counter the release could not settle")+}++// The fan-out can be reached with an already-cancelled context (graceful+// shutdown). The nothing-enqueued branch still has to clear the seeded key, or+// it lingers for its full 24h TTL; asynq's Enqueue takes no context, so+// post-process still goes out.+func TestEnqueueImageMultimodalTasksClearsCounterOnCancelledContext(t *testing.T) {+ mr, rdb := newSlotReleaseRedis(t)+ enqueuer := &slotReleaseEnqueuer{failAt: map[int]bool{0: true, 1: true}}+ svc := &knowledgeService{task: enqueuer, redisClient: rdb}+ knowledge, kb, images, chunks := slotReleaseFixture(2)+ ctx, cancel := context.WithCancel(context.Background())+ cancel()++ svc.enqueueImageMultimodalTasks(ctx, knowledge, kb, images, chunks, nil)++ require.False(t, mr.Exists(multimodalPendingKey(knowledge.ID)),+ "seeded counter must be cleared on a detached context")+ require.Equal(t, 1, enqueuer.postProcess)+}diff --git a/internal/application/service/knowledge_process.go b/internal/application/service/knowledge_process.goindex 648593473..10ed2f4f7 100644--- a/internal/application/service/knowledge_process.go+++ b/internal/application/service/knowledge_process.go@@ -251,6 +251,53 @@ func markKnowledgeProcessing(knowledge *types.Knowledge, now time.Time) {knowledge.UpdatedAt = now}+// failKnowledgeOnEmbeddingModel records an unresolvable embedding model as this+// attempt's terminal state.+//+// It re-reads the row instead of trusting the entry guard, because resolving a+// model reads the database: a cancel or a ReplaceKnowledgeFile can land in the+// window between that guard and this write. Both would be clobbered otherwise —+// UpdateKnowledge is a full-row Save and file_path is not among the omitted+// columns, so a stale in-memory row would roll file_path back over the+// replacement and mark the replacement's live attempt failed.+func (s *knowledgeService) failKnowledgeOnEmbeddingModel(+ ctx context.Context, kb *types.KnowledgeBase, knowledge *types.Knowledge, cause error,+) {+ // A cancelled or expired context means the run was interrupted — the user+ // cancelled (asynq CancelProcessing cancels the handler context), the+ // worker was preempted, the process is shutting down. The model itself is+ // not implicated, so recording "failed to get embedding model" would both+ // mislabel the cause and overwrite the cancelled status the abort path+ // just wrote. Leave the row to the abort path, or to the housekeeping+ // sweep when nothing else claims it.+ if ctx.Err() != nil || errors.Is(cause, context.Canceled) || errors.Is(cause, context.DeadlineExceeded) {+ logger.Infof(ctx,+ "Embedding model resolution interrupted for %s (%v); leaving parse status untouched",+ knowledge.ID, cause)+ return+ }+ if aborted, status := s.isKnowledgeAborted(ctx, knowledge.TenantID, knowledge.ID); aborted {+ logger.Infof(ctx,+ "Knowledge aborted (%s), not recording embedding model failure: %s", status, knowledge.ID)+ return+ }++ knowledge.ParseStatus = types.ParseStatusFailed+ knowledge.ErrorMessage = fmt.Sprintf("failed to get embedding model: %v", cause)+ knowledge.UpdatedAt = time.Now()+ if err := s.updateKnowledgeUnlessSourceReplaced(ctx, knowledge); err != nil {+ logger.Errorf(ctx, "failed to persist embedding model failure for %s: %v", knowledge.ID, err)+ }+ s.beginStage(ctx, knowledge.ID, types.StageEmbedding, types.JSONMap{+ "model_id": kb.EmbeddingModelID,+ })+ // The span's error_message is what the timeline renders, and its+ // error_detail is withheld from non-admin responses — so carry the same+ // text the document list shows rather than a fixed generic string.+ s.failStage(ctx, knowledge.ID, types.StageEmbedding,+ werrors.ErrCodeEmbeddingProviderFail, knowledge.ErrorMessage, cause)+}+// buildSplitterConfigFromChunking normalizes effective chunking settings with// the shared chunker defaults.func buildSplitterConfigFromChunking(cc types.ChunkingConfig) chunker.SplitterConfig {@@ -320,7 +367,19 @@ func (s *knowledgeService) processChunks(ctx context.Context,var err errorembeddingModel, err = s.modelService.GetEmbeddingModel(ctx, kb.EmbeddingModelID)if err != nil {+ // Terminal for this attempt, and it has to be recorded as such.+ // A KB that indexes vectors cannot proceed without an embedder,+ // and processChunks reports nothing to its callers — returning+ // silently leaves parse_status on "processing" with no task left+ // to move it, so the row hangs until the housekeeping sweep fails+ // it an hour later with a generic "stuck in processing" message.+ // Failing here names the real cause instead: a model row that was+ // deleted or deactivated, or a configuration the embedder factory+ // refuses. (Credential and connectivity problems surface later, on+ // the first BatchIndex round-trip — resolving a model only reads+ // the row and constructs a client.)logger.GetLogger(ctx).WithField("error", err).Errorf("processChunks get embedding model failed")+ s.failKnowledgeOnEmbeddingModel(ctx, kb, knowledge, err)return}} else {@@ -715,27 +774,7 @@ func (s *knowledgeService) processChunks(ctx context.Context,} else {s.skipStage(ctx, knowledge.ID, types.StageMultimodal, "skipped")// If there are no multimodal tasks, enqueue the post process task immediately- lang := types.LanguageFromContextOrDefault(ctx)- postProcessPayload := types.KnowledgePostProcessPayload{- TenantID: knowledge.TenantID,- KnowledgeID: knowledge.ID,- KnowledgeBaseID: knowledge.KnowledgeBaseID,- Language: lang,- Attempt: attemptFromCtx(ctx),- }- langfuse.InjectTracing(ctx, &postProcessPayload)- payloadBytes, err := json.Marshal(postProcessPayload)- if err == nil {- task := asynq.NewTask(types.TypeKnowledgePostProcess, payloadBytes,- knowledgePostProcessTaskOptions()...)- if _, err := s.task.Enqueue(task); err != nil {- logger.Errorf(ctx, "Failed to enqueue knowledge post process task: %v", err)- } else {- logger.Infof(ctx, "Enqueued knowledge post process task for %s", knowledge.ID)- }- } else {- logger.Errorf(ctx, "Failed to marshal knowledge post process payload: %v", err)- }+ s.enqueueKnowledgePostProcessTask(ctx, knowledge)}// Update tenant's storage usage@@ -4068,13 +4107,23 @@ func (s *knowledgeService) enqueueImageMultimodalTasks(}attempt := attemptFromCtx(ctx)- redisKey := fmt.Sprintf("multimodal:pending:%s", knowledge.ID)+ lang := types.LanguageFromContextOrDefault(ctx)+ redisKey := multimodalPendingKey(knowledge.ID)+ // The counter has to be seeded BEFORE the first task is enqueued: an image+ // task that finishes early would otherwise decrement a key we are about to+ // overwrite, and its slot would be counted twice. The cost of seeding+ // first is that a slot whose task never reaches the queue has no owner to+ // drain it — releaseUnownedMultimodalSlots settles that after the fan-out.+ counterSeeded := falseif s.redisClient != nil {if err := s.redisClient.Set(ctx, redisKey, len(images), 24*time.Hour).Err(); err != nil {logger.Warnf(ctx, "Failed to set multimodal pending count for %s: %v", knowledge.ID, err)+ } else {+ counterSeeded = true}}+ enqueued := 0for idx, img := range images {// Match image to the ParsedChunk whose content contains the image URL.// ChunkID was populated by processChunks with the real DB UUID.@@ -4089,7 +4138,6 @@ func (s *knowledgeService) enqueueImageMultimodalTasks(chunkID = chunks[0].ChunkID}- lang := types.LanguageFromContextOrDefault(ctx)payload := types.ImageMultimodalPayload{TenantID: knowledge.TenantID,KnowledgeID: knowledge.ID,@@ -4115,10 +4163,171 @@ func (s *knowledgeService) enqueueImageMultimodalTasks(asynq.Queue(types.QueueMultimodal), asynq.MaxRetry(3), asynq.Timeout(30*time.Minute))if _, err := s.task.Enqueue(task); err != nil {logger.Warnf(ctx, "Failed to enqueue image multimodal task for %s: %v", img.ServingURL, err)- } else {- logger.Infof(ctx, "Enqueued image:multimodal task for %s", img.ServingURL)+ continue}+ enqueued+++ logger.Infof(ctx, "Enqueued image:multimodal task for %s", img.ServingURL)+ }++ s.releaseUnownedMultimodalSlots(+ ctx, knowledge, redisKey, counterSeeded, len(images), enqueued,+ )+}++// releaseUnownedMultimodalSlots settles the multimodal fan-in counter against+// what actually reached the queue.+//+// Every enqueued image task drains exactly one slot when it exits terminally,+// and the knowledge only leaves "processing" once the counter reaches zero. A+// slot whose task was never enqueued (payload marshal failure, asynq refusing+// the write) has no owner, so without this release the counter stalls above+// zero, post-process is never triggered, and the row sits in "processing"+// until the housekeeping sweep fails it an hour later — which reaches the user+// as an unexplained parse failure. This mirrors the shortfall release+// KnowledgePostProcess already performs for its own enrichment subtasks.+func (s *knowledgeService) releaseUnownedMultimodalSlots(+ ctx context.Context,+ knowledge *types.Knowledge,+ redisKey string,+ counterSeeded bool,+ planned, enqueued int,+) {+ if enqueued == 0 {+ // No task exists to finalize this knowledge, whatever the counter+ // says. Drive post-process directly so the row completes on the+ // chunks that were already indexed instead of waiting to be swept.+ logger.Warnf(ctx,+ "[ImageMultimodal] No image task enqueued for %s (planned=%d); enqueueing post-process directly",+ knowledge.ID, planned)+ if counterSeeded {+ // Detached like the shortfall path below: a cancelled parent+ // context would otherwise leave the seeded key to expire on its+ // own 24h TTL.+ dctx, cancel := context.WithTimeout(+ context.WithoutCancel(ctx), finalizeSubtaskDetachedTimeout)+ s.redisClient.Del(dctx, redisKey)+ cancel()+ }+ s.enqueueKnowledgePostProcessTask(ctx, knowledge)+ return+ }++ shortfall := planned - enqueued+ if shortfall <= 0 {+ return+ }+ if !counterSeeded {+ // Seeding failed, so the key is absent and the first image to finish+ // already drives post-process through the missing-key fallback in+ // checkAndFinalizeAllImages. Decrementing here would only create the+ // key below zero and finalize before the siblings are done.+ return+ }++ logger.Warnf(ctx,+ "[ImageMultimodal] Releasing %d un-enqueued image slot(s) for %s (planned=%d enqueued=%d)",+ shortfall, knowledge.ID, planned, enqueued)+ // Detached ctx, for the same reason KnowledgePostProcess detaches its own+ // releases: these slots have no other path to drain, so a parent context+ // cancelled mid-fan-out (graceful shutdown) must not skip the release.+ rctx, cancel := context.WithTimeout(context.WithoutCancel(ctx), finalizeSubtaskDetachedTimeout)+ defer cancel()+ pending, err := s.decrMultimodalSlots(rctx, redisKey, int64(shortfall))+ if err != nil {+ // Giving up here puts the row back in the state this whole fix exists+ // to remove: a counter no task can drain, recovered only by the+ // housekeeping sweep an hour later. Say so at error level so it is+ // greppable, rather than hiding it in a warning.+ logger.Errorf(ctx,+ "Failed to release %d multimodal slot(s) for %s after %d attempts: %v; "++ "row will be left to the housekeeping sweep",+ shortfall, knowledge.ID, multimodalSlotReleaseAttempts, err)+ return+ }+ if pending <= 0 {+ // Every enqueued sibling had already finished, so the release owns the+ // finalize. Ordering is safe either way: the slots we just drained have+ // no task behind them, so exactly one path reaches zero.+ s.redisClient.Del(rctx, redisKey)+ s.enqueueKnowledgePostProcessTask(ctx, knowledge)+ }+}++// multimodalSlotReleaseAttempts bounds the retry on the release decrement.+//+// The release is the only thing that can ever drain an un-owned slot, so a+// single transient Redis error must not be the end of it — that would strand+// the row exactly as it did before this fix. Attempts stay small because they+// share the detached context's budget, and the housekeeping sweep is still the+// backstop when Redis is genuinely down.+//+// DECRBY is not idempotent, so a retry after a lost reply can over-decrement.+// That is the safe direction to fail in: an over-decrement finalizes early+// (post-process may miss some OCR / caption chunks, and repeat deliveries are+// already handled by its non-processing-status branch), whereas not retrying+// strands the document in "processing" for an hour.+const multimodalSlotReleaseAttempts = 3++// multimodalSlotReleaseBackoff is the unit of linear backoff between release+// attempts, small enough that all attempts fit the detached 10s budget.+const multimodalSlotReleaseBackoff = 100 * time.Millisecond++// decrMultimodalSlots decrements the fan-in counter by `by`, retrying transient+// Redis failures, and returns the resulting count.+func (s *knowledgeService) decrMultimodalSlots(+ ctx context.Context, redisKey string, by int64,+) (int64, error) {+ var lastErr error+ for attempt := 1; attempt <= multimodalSlotReleaseAttempts; attempt++ {+ pending, err := s.redisClient.DecrBy(ctx, redisKey, by).Result()+ if err == nil {+ return pending, nil+ }+ lastErr = err+ if attempt == multimodalSlotReleaseAttempts {+ break+ }+ logger.Warnf(ctx, "multimodal slot release attempt %d/%d failed for %s: %v",+ attempt, multimodalSlotReleaseAttempts, redisKey, err)+ select {+ case <-time.After(time.Duration(attempt) * multimodalSlotReleaseBackoff):+ case <-ctx.Done():+ return 0, ctx.Err()+ }+ }+ return 0, lastErr+}++// enqueueKnowledgePostProcessTask hands a knowledge to the post-process+// orchestrator, which owns the enrichment fan-out and the promotion out of+// "processing". Shared by the no-multimodal path and by the multimodal slot+// release, so both build the same payload (attempt, language, tracing).+func (s *knowledgeService) enqueueKnowledgePostProcessTask(+ ctx context.Context, knowledge *types.Knowledge,+) {+ if s.task == nil {+ return+ }+ payload := types.KnowledgePostProcessPayload{+ TenantID: knowledge.TenantID,+ KnowledgeID: knowledge.ID,+ KnowledgeBaseID: knowledge.KnowledgeBaseID,+ Language: types.LanguageFromContextOrDefault(ctx),+ Attempt: attemptFromCtx(ctx),+ }+ langfuse.InjectTracing(ctx, &payload)+ payloadBytes, err := json.Marshal(payload)+ if err != nil {+ logger.Errorf(ctx, "Failed to marshal knowledge post process payload: %v", err)+ return+ }+ task := asynq.NewTask(types.TypeKnowledgePostProcess, payloadBytes,+ knowledgePostProcessTaskOptions()...)+ if _, err := s.task.Enqueue(task); err != nil {+ logger.Errorf(ctx, "Failed to enqueue knowledge post process task: %v", err)+ return}+ logger.Infof(ctx, "Enqueued knowledge post process task for %s", knowledge.ID)}// ProcessKnowledgeListReparse handles Asynq knowledge list reparse tasks.diff --git a/internal/application/service/knowledge_process_embedding_failure_test.go b/internal/application/service/knowledge_process_embedding_failure_test.gonew file mode 100644index 000000000..238ed02fa--- /dev/null+++ b/internal/application/service/knowledge_process_embedding_failure_test.go@@ -0,0 +1,257 @@+package service++import (+ "context"+ "errors"+ "fmt"+ "strings"+ "testing"++ "github.com/Tencent/WeKnora/internal/models/embedding"+ "github.com/Tencent/WeKnora/internal/types"+ "github.com/Tencent/WeKnora/internal/types/interfaces"+ "github.com/stretchr/testify/require"+)++type embedFailureKnowledgeRepo struct {+ interfaces.KnowledgeRepository+ knowledge *types.Knowledge+ // stored is what the database starts returning once storedAfterReads+ // reads have been served. The delay matters: processChunks already+ // guards its entry, so a test that returns the cancelled / replaced row+ // from the very first read never reaches the code under test. Delaying+ // it models the real window — resolving an embedding model reads the+ // database, so a cancel or a replacement can land while it runs.+ stored *types.Knowledge+ storedAfterReads int+ reads int+ updates []types.Knowledge+}++func (r *embedFailureKnowledgeRepo) GetKnowledgeByID(+ context.Context, uint64, string,+) (*types.Knowledge, error) {+ r.reads+++ if r.stored != nil && r.reads > r.storedAfterReads {+ return r.stored, nil+ }+ return r.knowledge, nil+}++func (r *embedFailureKnowledgeRepo) UpdateKnowledge(_ context.Context, k *types.Knowledge) error {+ r.updates = append(r.updates, *k)+ return nil+}++type embedFailureModelService struct {+ interfaces.ModelService+ err error+}++func (s embedFailureModelService) GetEmbeddingModel(+ context.Context, string,+) (embedding.Embedder, error) {+ return nil, s.err+}++// A KB that indexes vectors cannot proceed without an embedder, and+// processChunks reports nothing to its callers — so this failure has to be+// persisted here or the row stays "processing" with no task left to move it,+// visible to the user only as an hour-long spinner followed by a generic+// housekeeping failure.+//+// chunkRepo / graphEngine / retrieveEngine are deliberately left nil: any+// attempt to keep processing past the missing embedder would panic and fail+// this test, which is exactly the regression worth catching.+func TestProcessChunksFailsKnowledgeWhenEmbeddingModelUnavailable(t *testing.T) {+ modelErr := errors.New("embedding model not found")+ knowledge := &types.Knowledge{+ ID: "knowledge-1",+ TenantID: 1,+ KnowledgeBaseID: "kb-1",+ ParseStatus: types.ParseStatusProcessing,+ }+ repo := &embedFailureKnowledgeRepo{knowledge: knowledge}+ svc := &knowledgeService{+ repo: repo,+ modelService: embedFailureModelService{err: modelErr},+ }+ kb := &types.KnowledgeBase{+ ID: "kb-1",+ TenantID: 1,+ EmbeddingModelID: "embedding-1",+ IndexingStrategy: types.IndexingStrategy{VectorEnabled: true},+ }++ svc.processChunks(context.Background(), kb, knowledge,+ []types.ParsedChunk{{Content: "body", Seq: 0, Start: 0, End: 4}})++ require.Len(t, repo.updates, 1, "the failure must be persisted exactly once")+ require.Equal(t, types.ParseStatusFailed, repo.updates[0].ParseStatus)+ require.True(t,+ strings.Contains(repo.updates[0].ErrorMessage, modelErr.Error()),+ "error_message should name the underlying cause, got %q", repo.updates[0].ErrorMessage,+ )+ require.Equal(t, types.ParseStatusFailed, knowledge.ParseStatus)+}++// A cancelled context means the run was interrupted, not that the model is+// unusable — the user cancelled (asynq CancelProcessing cancels the handler+// context), the worker was preempted, or the process is shutting down.+// Recording a model failure here would both mislabel the cause and overwrite+// the cancelled status the abort path just wrote.+func TestProcessChunksLeavesStatusAloneWhenEmbeddingModelCallIsCancelled(t *testing.T) {+ for _, tc := range []struct {+ name string+ modelErr error+ cancelCtx bool+ }{+ {name: "error is context.Canceled", modelErr: context.Canceled},+ {name: "error wraps context.DeadlineExceeded", modelErr: fmt.Errorf("query: %w", context.DeadlineExceeded)},+ {name: "context already cancelled", modelErr: errors.New("driver: bad connection"), cancelCtx: true},+ } {+ t.Run(tc.name, func(t *testing.T) {+ knowledge := &types.Knowledge{+ ID: "knowledge-1",+ TenantID: 1,+ KnowledgeBaseID: "kb-1",+ ParseStatus: types.ParseStatusProcessing,+ }+ repo := &embedFailureKnowledgeRepo{knowledge: knowledge}+ svc := &knowledgeService{+ repo: repo,+ modelService: embedFailureModelService{err: tc.modelErr},+ }+ kb := &types.KnowledgeBase{+ ID: "kb-1",+ TenantID: 1,+ EmbeddingModelID: "embedding-1",+ IndexingStrategy: types.IndexingStrategy{VectorEnabled: true},+ }+ ctx := context.Background()+ if tc.cancelCtx {+ cancelled, cancel := context.WithCancel(ctx)+ cancel()+ ctx = cancelled+ }++ svc.processChunks(ctx, kb, knowledge,+ []types.ParsedChunk{{Content: "body", Seq: 0, Start: 0, End: 4}})++ require.Empty(t, repo.updates, "an interrupted run must not be written as a failure")+ require.Equal(t, types.ParseStatusProcessing, knowledge.ParseStatus)+ })+ }+}++// The row was cancelled after the entry guard ran. Resolving a model reads the+// database, so that window is real — and the cancelled status must survive it.+func TestProcessChunksDoesNotOverwriteCancelledRowOnEmbeddingFailure(t *testing.T) {+ knowledge := &types.Knowledge{+ ID: "knowledge-1",+ TenantID: 1,+ KnowledgeBaseID: "kb-1",+ ParseStatus: types.ParseStatusProcessing,+ }+ repo := &embedFailureKnowledgeRepo{+ knowledge: knowledge,+ // The entry guard's read still sees a live row; the cancel lands+ // while the embedding model is being resolved.+ storedAfterReads: 1,+ stored: &types.Knowledge{+ ID: "knowledge-1",+ TenantID: 1,+ ParseStatus: types.ParseStatusCancelled,+ },+ }+ svc := &knowledgeService{+ repo: repo,+ modelService: embedFailureModelService{err: errors.New("embedding model not found")},+ }+ kb := &types.KnowledgeBase{+ ID: "kb-1",+ TenantID: 1,+ EmbeddingModelID: "embedding-1",+ IndexingStrategy: types.IndexingStrategy{VectorEnabled: true},+ }++ svc.processChunks(context.Background(), kb, knowledge,+ []types.ParsedChunk{{Content: "body", Seq: 0, Start: 0, End: 4}})++ require.Greater(t, repo.reads, 1, "the guard must re-read after resolving the model")+ require.Empty(t, repo.updates, "cancelled must not be overwritten with failed")+}++// ReplaceKnowledgeFile keeps the knowledge ID and swaps file_path, so a stale+// worker must not persist its row: UpdateKnowledge is a full-row Save and+// file_path is not omitted, so it would roll the path back over the+// replacement and mark the replacement's live attempt failed.+func TestProcessChunksSkipsEmbeddingFailureWriteWhenSourceReplaced(t *testing.T) {+ knowledge := &types.Knowledge{+ ID: "knowledge-1",+ TenantID: 1,+ KnowledgeBaseID: "kb-1",+ ParseStatus: types.ParseStatusProcessing,+ FilePath: "tenant/1/old.pdf",+ }+ repo := &embedFailureKnowledgeRepo{+ knowledge: knowledge,+ // Reads 1-2 are the entry guard (aborted + source-replaced) and still+ // see this worker's own file; the replacement lands while the+ // embedding model is being resolved.+ storedAfterReads: 2,+ stored: &types.Knowledge{+ ID: "knowledge-1",+ TenantID: 1,+ ParseStatus: types.ParseStatusProcessing,+ FilePath: "tenant/1/replacement.pdf",+ },+ }+ svc := &knowledgeService{+ repo: repo,+ modelService: embedFailureModelService{err: errors.New("embedding model not found")},+ }+ kb := &types.KnowledgeBase{+ ID: "kb-1",+ TenantID: 1,+ EmbeddingModelID: "embedding-1",+ IndexingStrategy: types.IndexingStrategy{VectorEnabled: true},+ }++ svc.processChunks(context.Background(), kb, knowledge,+ []types.ParsedChunk{{Content: "body", Seq: 0, Start: 0, End: 4}})++ require.Greater(t, repo.reads, 2, "the guard must re-read after resolving the model")+ require.Empty(t, repo.updates,+ "a replaced source must not have this attempt's failure written back")+}++// A KB with vector and keyword indexing both off never resolves an embedder,+// so the guard above must not fire for it.+func TestProcessChunksSkipsEmbeddingModelWhenIndexingDisabled(t *testing.T) {+ knowledge := &types.Knowledge{+ ID: "knowledge-1",+ TenantID: 1,+ KnowledgeBaseID: "kb-1",+ ParseStatus: types.ParseStatusProcessing,+ }+ chunkRepo := &parentChildChunkService{}+ tenant := &types.Tenant{ID: 1}+ ctx := context.WithValue(context.Background(), types.TenantInfoContextKey, tenant)+ svc := &knowledgeService{+ repo: &embedFailureKnowledgeRepo{knowledge: knowledge},+ chunkRepo: chunkRepo,+ modelService: embedFailureModelService{err: errors.New("must not be called")},+ graphEngine: parentChildGraphRepo{},+ tenantRepo: parentChildTenantRepo{},+ task: parentChildTaskEnqueuer{},+ }+ kb := &types.KnowledgeBase{ID: "kb-1", TenantID: 1}+ require.False(t, kb.NeedsEmbeddingModel())++ svc.processChunks(ctx, kb, knowledge,+ []types.ParsedChunk{{Content: "body", Seq: 0, Start: 0, End: 4}})++ require.Len(t, chunkRepo.created, 1, "chunks are persisted even without vector indexing")+ require.NotEqual(t, types.ParseStatusFailed, knowledge.ParseStatus)+}