Skip to content

Open-source LLM knowledge platform: turn raw documents into a queryable RAG, an autonomous reasoning agent, and a self-maintaining Wiki.

fix(knowledge): stop pipeline errors from stranding rows in processing

wizardchend951cb78d132
diff --git a/internal/application/service/knowledge.go b/internal/application/service/knowledge.go
index 2e4ef0252..aa51000f9 100644
--- a/internal/application/service/knowledge.go
+++ b/internal/application/service/knowledge.go
@@ -397,25 +397,32 @@ func (s *knowledgeService) isKnowledgeDeleting(ctx context.Context, tenantID uin
return knowledge.ParseStatus == types.ParseStatusDeleting
}
+// abortStatusInterrupted is the pseudo-status isKnowledgeAborted reports
+// when the worker's own context is done: bail out, but clean up nothing.
+const abortStatusInterrupted = "interrupted"
+
// isKnowledgeAborted returns (true, status) when the knowledge has been
// marked as deleting OR cancelled so async pipeline workers should bail
// out. Status is returned so callers can branch on cleanup behavior:
// deleting → existing cleanup of partial chunks/index applies;
// cancelled → keep partially written data per user expectation.
//
-// When the row is missing or unreadable we conservatively return
-// (true, ParseStatusDeleting): the existing deleting branch already
-// handles cleanup-or-no-op semantics safely.
+// Only a row that is really gone reads as deleting. A transient read error
+// must not: callers would wipe a live document's chunks and index and leave
+// it stuck in processing, so the check reports "not aborted" and lets the
+// pipeline's next write surface a real outage.
func (s *knowledgeService) isKnowledgeAborted(
ctx context.Context, tenantID uint64, knowledgeID string,
) (bool, string) {
knowledge, err := s.repo.GetKnowledgeByID(ctx, tenantID, knowledgeID)
- if err != nil {
- logger.Warnf(ctx, "Failed to check knowledge abort status (assuming deleted): %v", err)
- return true, types.ParseStatusDeleting
- }
- if knowledge == nil {
+ switch {
+ case err == nil && knowledge == nil, errors.Is(err, repository.ErrKnowledgeNotFound):
return true, types.ParseStatusDeleting
+ case err != nil && ctx.Err() != nil:
+ return true, abortStatusInterrupted
+ case err != nil:
+ logger.Warnf(ctx, "Failed to check knowledge abort status for %s (continuing): %v", knowledgeID, err)
+ return false, ""
}
switch knowledge.ParseStatus {
case types.ParseStatusDeleting, types.ParseStatusCancelled:
diff --git a/internal/application/service/knowledge_post_process.go b/internal/application/service/knowledge_post_process.go
index af8eae867..3321c079b 100644
--- a/internal/application/service/knowledge_post_process.go
+++ b/internal/application/service/knowledge_post_process.go
@@ -328,11 +328,12 @@ func (s *KnowledgePostProcessService) Handle(ctx context.Context, task *asynq.Ta
promoted, err = s.knowledgeRepo.SetFinalizing(ctx, payload.KnowledgeID, expectedSubtasks)
}
if err != nil {
+ // Acking here would read as "no longer processing" below and
+ // strand the row; retry so the handoff (or dead-letter) happens.
logger.Warnf(ctx, "[KnowledgePostProcess] SetFinalizing failed for %s: %v",
payload.KnowledgeID, err)
- if willSpawnWiki {
- return fmt.Errorf("seed finalizing with wiki pending op: %w", err)
- }
+ s.tracker().FailSpan(ctx, postSpan, "FINALIZING_HANDOFF_FAILED", err.Error(), err)
+ return fmt.Errorf("enter finalizing: %w", err)
}
if promoted {
enteredFinalizing = true
diff --git a/internal/application/service/knowledge_post_process_wiki_enqueue_test.go b/internal/application/service/knowledge_post_process_wiki_enqueue_test.go
index 26a6696b4..582e3370e 100644
--- a/internal/application/service/knowledge_post_process_wiki_enqueue_test.go
+++ b/internal/application/service/knowledge_post_process_wiki_enqueue_test.go
@@ -17,6 +17,8 @@ type wikiEnqueueFailureKnowledgeRepo struct {
interfaces.KnowledgeRepository
knowledge *types.Knowledge
expectedSubtasks int
+ setFinalizingErr error
+ finalized []string
}
func (r *wikiEnqueueFailureKnowledgeRepo) GetKnowledgeByIDOnly(
@@ -31,15 +33,19 @@ func (r *wikiEnqueueFailureKnowledgeRepo) SetFinalizing(
_ string,
expectedSubtasks int,
) (bool, error) {
+ if r.setFinalizingErr != nil {
+ return false, r.setFinalizingErr
+ }
r.expectedSubtasks = expectedSubtasks
r.knowledge.ParseStatus = types.ParseStatusFinalizing
return true, nil
}
func (r *wikiEnqueueFailureKnowledgeRepo) FinalizeSubtask(
- context.Context,
- string,
+ _ context.Context,
+ knowledgeID string,
) (int, bool, error) {
+ r.finalized = append(r.finalized, knowledgeID)
return 0, false, nil
}
diff --git a/internal/application/service/knowledge_process.go b/internal/application/service/knowledge_process.go
index d79fabc36..45fc88ff8 100644
--- a/internal/application/service/knowledge_process.go
+++ b/internal/application/service/knowledge_process.go
@@ -263,6 +263,15 @@ func markKnowledgeProcessing(knowledge *types.Knowledge, now time.Time) {
// replacement and mark the replacement's live attempt failed.
func (s *knowledgeService) failKnowledgeOnEmbeddingModel(
ctx context.Context, kb *types.KnowledgeBase, knowledge *types.Knowledge, cause error,
+) {
+ s.failKnowledgeAtEmbedding(ctx, kb, knowledge, werrors.ErrCodeEmbeddingProviderFail,
+ "failed to get embedding model", cause)
+}
+
+// failKnowledgeAtEmbedding records cause as this attempt's terminal state at
+// the embedding stage; see failKnowledgeOnEmbeddingModel for the guards.
+func (s *knowledgeService) failKnowledgeAtEmbedding(
+ ctx context.Context, kb *types.KnowledgeBase, knowledge *types.Knowledge, code, what string, cause error,
) {
// A cancelled or expired context means the run was interrupted — the user
// cancelled (asynq CancelProcessing cancels the handler context), the
@@ -273,21 +282,21 @@ func (s *knowledgeService) failKnowledgeOnEmbeddingModel(
// sweep when nothing else claims it.
if ctx.Err() != nil || errors.Is(cause, context.Canceled) || errors.Is(cause, context.DeadlineExceeded) {
logger.Infof(ctx,
- "Embedding model resolution interrupted for %s (%v); leaving parse status untouched",
- knowledge.ID, cause)
+ "%s interrupted for %s (%v); leaving parse status untouched",
+ what, knowledge.ID, cause)
return
}
if aborted, status := s.isKnowledgeAborted(ctx, knowledge.TenantID, knowledge.ID); aborted {
logger.Infof(ctx,
- "Knowledge aborted (%s), not recording embedding model failure: %s", status, knowledge.ID)
+ "Knowledge aborted (%s), not recording %q: %s", status, what, knowledge.ID)
return
}
knowledge.ParseStatus = types.ParseStatusFailed
- knowledge.ErrorMessage = fmt.Sprintf("failed to get embedding model: %v", cause)
+ knowledge.ErrorMessage = fmt.Sprintf("%s: %v", what, cause)
knowledge.UpdatedAt = time.Now()
if err := s.updateKnowledgeUnlessSourceReplaced(ctx, knowledge); err != nil {
- logger.Errorf(ctx, "failed to persist embedding model failure for %s: %v", knowledge.ID, err)
+ logger.Errorf(ctx, "failed to persist %q for %s: %v", what, knowledge.ID, err)
}
s.beginStage(ctx, knowledge.ID, types.StageEmbedding, types.JSONMap{
"model_id": kb.EmbeddingModelID,
@@ -295,8 +304,7 @@ func (s *knowledgeService) failKnowledgeOnEmbeddingModel(
// The span's error_message is what the timeline renders, and its
// error_detail is withheld from non-admin responses — so carry the same
// text the document list shows rather than a fixed generic string.
- s.failStage(ctx, knowledge.ID, types.StageEmbedding,
- werrors.ErrCodeEmbeddingProviderFail, knowledge.ErrorMessage, cause)
+ s.failStage(ctx, knowledge.ID, types.StageEmbedding, code, knowledge.ErrorMessage, cause)
}
// buildSplitterConfigFromChunking normalizes effective chunking settings with
@@ -400,7 +408,14 @@ func (s *knowledgeService) processChunks(ctx context.Context,
tenantInfo := ctx.Value(types.TenantInfoContextKey).(*types.Tenant)
retrieveEngine, err := retriever.CreateRetrieveEngineForKB(
ctx, s.retrieveEngine, s.ownership, tenantInfo.ID, kb.VectorStoreID)
- if err == nil && embeddingModel != nil {
+ if err != nil && embeddingModel != nil {
+ // Indexing below dereferences the engine; a nil one would panic.
+ logger.Errorf(ctx, "processChunks resolve vector store for KB %s failed: %v", kb.ID, err)
+ s.failKnowledgeAtEmbedding(ctx, kb, knowledge, werrors.ErrCodeVectorStoreWriteFailed,
+ "failed to resolve vector store", err)
+ return
+ }
+ if embeddingModel != nil {
if err := retrieveEngine.DeleteByKnowledgeIDList(ctx, []string{knowledge.ID}, embeddingModel.GetDimensions(), knowledge.Type); err != nil {
logger.Warnf(ctx, "Failed to delete existing index data (may not exist): %v", err)
// 不返回错误,继续处理(可能没有旧数据)
diff --git a/internal/application/service/knowledge_stuck_processing_test.go b/internal/application/service/knowledge_stuck_processing_test.go
new file mode 100644
index 000000000..0928a66d3
--- /dev/null
+++ b/internal/application/service/knowledge_stuck_processing_test.go
@@ -0,0 +1,207 @@
+package service
+
+import (
+ "context"
+ "encoding/json"
+ "errors"
+ "testing"
+ "time"
+
+ apprepo "github.com/Tencent/WeKnora/internal/application/repository"
+ "github.com/Tencent/WeKnora/internal/models/chat"
+ "github.com/Tencent/WeKnora/internal/types"
+ "github.com/Tencent/WeKnora/internal/types/interfaces"
+ "github.com/hibiken/asynq"
+ "github.com/stretchr/testify/assert"
+ "github.com/stretchr/testify/require"
+)
+
+// A failed processing→finalizing handoff must be retried, not acked: acking
+// read as "no longer processing" and left the row stuck in processing.
+func TestPostProcessRetriesWhenFinalizingHandoffFails(t *testing.T) {
+ const knowledgeID = "knowledge-handoff-fails"
+ queue := &wikiEnqueueFailureTaskQueue{}
+ svc, repo := newWikiEnqueueTestService(knowledgeID, &wikiEnqueueFailurePendingRepo{}, queue)
+ svc.kbService.(*wikiEnqueueFailureKBService).kb.IndexingStrategy.WikiEnabled = false
+ repo.setFinalizingErr = errors.New("postgres unavailable")
+
+ err := svc.Handle(context.Background(), newWikiEnqueuePostProcessTask(t, knowledgeID))
+
+ require.ErrorIs(t, err, repo.setFinalizingErr)
+ assert.Equal(t, types.ParseStatusProcessing, repo.knowledge.ParseStatus)
+ assert.Empty(t, queue.taskTypes, "no enrichment may fan out before the handoff lands")
+}
+
+type wikiUnavailablePendingRepo struct {
+ interfaces.TaskPendingOpsRepository
+ drainKeys []string
+ drainErr error
+ drained []string
+}
+
+func (r *wikiUnavailablePendingRepo) DrainUnclaimed(
+ _ context.Context, taskType, scope, scopeID, op string, _ time.Time,
+) ([]string, error) {
+ r.drained = append(r.drained, taskType+"|"+scope+"|"+scopeID+"|"+op)
+ return r.drainKeys, r.drainErr
+}
+
+type wikiUnavailableModelService struct {
+ interfaces.ModelService
+ err error
+}
+
+func (s *wikiUnavailableModelService) GetChatModel(context.Context, string) (chat.Chat, error) {
+ return nil, s.err
+}
+
+// A wiki that cannot run (disabled, no model, model deleted) fails the same
+// way on every retry, so its queued ingest ops must be released instead of
+// holding their documents in "finalizing" forever.
+func TestWikiIngestReleasesDocumentsWhenWikiUnavailable(t *testing.T) {
+ payload, err := json.Marshal(WikiIngestPayload{TenantID: 7, KnowledgeBaseID: "kb-1"})
+ require.NoError(t, err)
+ enabled := types.IndexingStrategy{WikiEnabled: true}
+ tests := []struct {
+ name string
+ kb *types.KnowledgeBase
+ modelErr error
+ }{
+ {name: "wiki disabled", kb: &types.KnowledgeBase{ID: "kb-1"}},
+ {name: "no synthesis model", kb: &types.KnowledgeBase{ID: "kb-1", IndexingStrategy: enabled}},
+ {
+ name: "synthesis model deleted",
+ kb: &types.KnowledgeBase{
+ ID: "kb-1", IndexingStrategy: enabled,
+ WikiConfig: &types.WikiConfig{SynthesisModelID: "gone"},
+ },
+ modelErr: ErrModelNotFound,
+ },
+ }
+ for _, test := range tests {
+ t.Run(test.name, func(t *testing.T) {
+ pending := &wikiUnavailablePendingRepo{drainKeys: []string{"k-1", "k-2"}}
+ knowledgeRepo := &wikiEnqueueFailureKnowledgeRepo{}
+ svc := &wikiIngestService{
+ kbService: &wikiGuardKBService{kb: test.kb},
+ modelService: &wikiUnavailableModelService{err: test.modelErr},
+ pendingRepo: pending,
+ knowledgeRepo: knowledgeRepo,
+ }
+
+ err := svc.ProcessWikiIngest(context.Background(), asynq.NewTask(types.TypeWikiIngest, payload))
+
+ require.NoError(t, err)
+ assert.Equal(t, []string{wikiTaskType + "|" + wikiTaskScope + "|kb-1|" + WikiOpIngest}, pending.drained)
+ assert.Equal(t, []string{"k-1", "k-2"}, knowledgeRepo.finalized)
+ })
+ }
+}
+
+// A transient model lookup failure keeps the ops and retries as before.
+func TestWikiIngestKeepsOpsOnTransientModelError(t *testing.T) {
+ payload, err := json.Marshal(WikiIngestPayload{TenantID: 7, KnowledgeBaseID: "kb-1"})
+ require.NoError(t, err)
+ pending := &wikiUnavailablePendingRepo{}
+ transient := errors.New("connection reset")
+ svc := &wikiIngestService{
+ kbService: &wikiGuardKBService{kb: &types.KnowledgeBase{
+ ID: "kb-1", IndexingStrategy: types.IndexingStrategy{WikiEnabled: true},
+ WikiConfig: &types.WikiConfig{SynthesisModelID: "m-1"},
+ }},
+ modelService: &wikiUnavailableModelService{err: transient},
+ pendingRepo: pending,
+ }
+
+ err = svc.ProcessWikiIngest(context.Background(), asynq.NewTask(types.TypeWikiIngest, payload))
+
+ require.ErrorIs(t, err, transient)
+ assert.Empty(t, pending.drained)
+}
+
+type abortCheckRepo struct {
+ interfaces.KnowledgeRepository
+ knowledge *types.Knowledge
+ err error
+}
+
+func (r *abortCheckRepo) GetKnowledgeByID(context.Context, uint64, string) (*types.Knowledge, error) {
+ return r.knowledge, r.err
+}
+
+// Only a row that is really gone may read as deleting; callers wipe chunks
+// and index on that status.
+func TestIsKnowledgeAbortedDistinguishesMissingFromUnreadable(t *testing.T) {
+ cancelled, cancel := context.WithCancel(context.Background())
+ cancel()
+ tests := []struct {
+ name string
+ ctx context.Context
+ repo *abortCheckRepo
+ wantAborted bool
+ wantStatus string
+ }{
+ {
+ name: "missing", ctx: context.Background(),
+ repo: &abortCheckRepo{err: apprepo.ErrKnowledgeNotFound},
+ wantAborted: true, wantStatus: types.ParseStatusDeleting,
+ },
+ {
+ name: "transient read error", ctx: context.Background(),
+ repo: &abortCheckRepo{err: errors.New("connection reset")},
+ },
+ {
+ name: "worker context done", ctx: cancelled,
+ repo: &abortCheckRepo{err: context.Canceled},
+ wantAborted: true, wantStatus: abortStatusInterrupted,
+ },
+ {
+ name: "cancelled", ctx: context.Background(),
+ repo: &abortCheckRepo{knowledge: &types.Knowledge{ParseStatus: types.ParseStatusCancelled}},
+ wantAborted: true, wantStatus: types.ParseStatusCancelled,
+ },
+ {
+ name: "processing", ctx: context.Background(),
+ repo: &abortCheckRepo{knowledge: &types.Knowledge{ParseStatus: types.ParseStatusProcessing}},
+ wantStatus: types.ParseStatusProcessing,
+ },
+ }
+ for _, test := range tests {
+ t.Run(test.name, func(t *testing.T) {
+ svc := &knowledgeService{repo: test.repo}
+ aborted, status := svc.isKnowledgeAborted(test.ctx, 1, "k-1")
+ assert.Equal(t, test.wantAborted, aborted)
+ assert.Equal(t, test.wantStatus, status)
+ })
+ }
+}
+
+// Rows held only by a durable wiki op get their KB's trigger re-armed, once
+// per KB and at most once per threshold.
+func TestHousekeepingRearmsWikiTriggerForDurablyHeldRows(t *testing.T) {
+ db := setupHousekeepingDB(t)
+ queue := &wikiGuardTaskQueue{}
+ svc := newHousekeepingSvcForTest(db)
+ svc.task = queue
+ stale := time.Now().Add(-3 * time.Hour)
+ for _, id := range []string{"k-1", "k-2"} {
+ require.NoError(t, db.Exec(
+ `INSERT INTO knowledges (id, tenant_id, knowledge_base_id, parse_status, updated_at)
+ VALUES (?, 7, 'kb-1', ?, ?)`, id, types.ParseStatusFinalizing, stale,
+ ).Error)
+ insertWikiPendingOp(t, db, "kb-1", id)
+ }
+
+ svc.runSweep(context.Background())
+ svc.runSweep(context.Background())
+
+ require.Len(t, queue.tasks, 1)
+ assert.Equal(t, types.TypeWikiIngest, queue.tasks[0].Type())
+ var payload WikiIngestPayload
+ require.NoError(t, json.Unmarshal(queue.tasks[0].Payload(), &payload))
+ assert.Equal(t, uint64(7), payload.TenantID)
+ assert.Equal(t, "kb-1", payload.KnowledgeBaseID)
+ var status string
+ require.NoError(t, db.Raw(`SELECT parse_status FROM knowledges WHERE id = 'k-1'`).Scan(&status).Error)
+ assert.Equal(t, types.ParseStatusFinalizing, status)
+}
diff --git a/internal/middleware/asynqdl/recover.go b/internal/middleware/asynqdl/recover.go
new file mode 100644
index 000000000..6cfb61424
--- /dev/null
+++ b/internal/middleware/asynqdl/recover.go
@@ -0,0 +1,36 @@
+package asynqdl
+
+import (
+ "context"
+ "fmt"
+ "runtime/debug"
+
+ "github.com/Tencent/WeKnora/internal/logger"
+ "github.com/hibiken/asynq"
+)
+
+// RecoverMiddleware turns a handler panic into an ordinary task error.
+// asynq recovers panics itself, but only in its processor, above every
+// middleware — so a panic skips the dead-letter callback and a document on
+// its last attempt stays "processing" until housekeeping. Install it right
+// after MiddlewareWithCallback so that callback sees the error.
+func RecoverMiddleware() asynq.MiddlewareFunc {
+ return func(next asynq.Handler) asynq.Handler {
+ return asynq.HandlerFunc(func(ctx context.Context, t *asynq.Task) error {
+ return CallRecovered(ctx, t, next.ProcessTask)
+ })
+ }
+}
+
+// CallRecovered runs handler, converting a panic into an error.
+func CallRecovered(
+ ctx context.Context, t *asynq.Task, handler func(context.Context, *asynq.Task) error,
+) (err error) {
+ defer func() {
+ if r := recover(); r != nil {
+ logger.Errorf(ctx, "task %s panicked: %v\n%s", t.Type(), r, debug.Stack())
+ err = fmt.Errorf("task %s panicked: %v", t.Type(), r)
+ }
+ }()
+ return handler(ctx, t)
+}
diff --git a/internal/middleware/asynqdl/recover_test.go b/internal/middleware/asynqdl/recover_test.go
new file mode 100644
index 000000000..8044f2696
--- /dev/null
+++ b/internal/middleware/asynqdl/recover_test.go
@@ -0,0 +1,29 @@
+package asynqdl
+
+import (
+ "context"
+ "testing"
+
+ "github.com/hibiken/asynq"
+ "github.com/stretchr/testify/assert"
+ "github.com/stretchr/testify/require"
+)
+
+// A panic below RecoverMiddleware reaches the dead-letter callback as an
+// ordinary error instead of unwinding past it.
+func TestRecoverMiddleware_PanicReachesDeadLetterCallback(t *testing.T) {
+ repo := &fakeRepo{}
+ var callbackErr error
+ cb := func(_ context.Context, _ *asynq.Task, err error) { callbackErr = err }
+ panicking := asynq.HandlerFunc(func(context.Context, *asynq.Task) error {
+ panic("nil retrieve engine")
+ })
+ h := MiddlewareWithCallback(repo, cb)(RecoverMiddleware()(panicking))
+
+ err := h.ProcessTask(context.Background(), asynq.NewTask("document:process", []byte(`{}`)))
+
+ require.Error(t, err)
+ assert.Contains(t, err.Error(), "document:process panicked")
+ assert.Equal(t, err, callbackErr)
+ assert.Equal(t, 1, repo.rowCount())
+}
diff --git a/internal/router/sync_task.go b/internal/router/sync_task.go
index 8c8762f3e..28e5fab1a 100644
--- a/internal/router/sync_task.go
+++ b/internal/router/sync_task.go
@@ -7,6 +7,7 @@ import (
"time"
"github.com/Tencent/WeKnora/internal/logger"
+ "github.com/Tencent/WeKnora/internal/middleware/asynqdl"
"github.com/Tencent/WeKnora/internal/types"
"github.com/Tencent/WeKnora/internal/types/interfaces"
"github.com/google/uuid"
@@ -100,7 +101,8 @@ func (e *SyncTaskExecutor) Enqueue(task *asynq.Task, opts ...asynq.Option) (*asy
}
attemptCtx := types.WithTaskRetryMetadata(ctx, attempt, maxRetry)
- lastErr = handler(attemptCtx, task)
+ // An unrecovered panic here would take down the whole process.
+ lastErr = asynqdl.CallRecovered(attemptCtx, task, handler)
if lastErr == nil {
logger.Infof(ctx, "[SyncTask] Task completed type=%s id=%s elapsed=%v",
task.Type(), taskID, time.Since(start))
diff --git a/internal/router/task.go b/internal/router/task.go
index fe0e240fe..4e3c8a279 100644
--- a/internal/router/task.go
+++ b/internal/router/task.go
@@ -245,6 +245,7 @@ func RunAsynqServer(params AsynqTaskParams) *asynq.ServeMux {
// UI signal users actually see.
knowledgeFailer := newDeadLetterKnowledgeFailer(params.KnowledgeService, params.SpanTracker)
mux.Use(asynqdl.MiddlewareWithCallback(params.DeadLetterRepo, knowledgeFailer))
+ mux.Use(asynqdl.RecoverMiddleware())
// Mark every asynq worker execution as a background task so the chat
// concurrency governor throttles ingestion/enrichment LLM traffic while