Open-source LLM knowledge platform: turn raw documents into a queryable RAG, an autonomous reasoning agent, and a self-maintaining Wiki.
Go agentagenticaichatbotdsh-pluginembeddingsevaluationgenerative-aigolangknowledge-basellmmulti-tenantollamaopenaiquestion-answeringragrerankingsemantic-searchvector-searchwiki
fix(knowledge): stop pipeline errors from stranding rows in processing
wizardchend951cb78d132
diff --git a/internal/application/service/knowledge.go b/internal/application/service/knowledge.goindex 2e4ef0252..aa51000f9 100644--- a/internal/application/service/knowledge.go+++ b/internal/application/service/knowledge.go@@ -397,25 +397,32 @@ func (s *knowledgeService) isKnowledgeDeleting(ctx context.Context, tenantID uinreturn knowledge.ParseStatus == types.ParseStatusDeleting}+// abortStatusInterrupted is the pseudo-status isKnowledgeAborted reports+// when the worker's own context is done: bail out, but clean up nothing.+const abortStatusInterrupted = "interrupted"+// isKnowledgeAborted returns (true, status) when the knowledge has been// marked as deleting OR cancelled so async pipeline workers should bail// out. Status is returned so callers can branch on cleanup behavior:// deleting → existing cleanup of partial chunks/index applies;// cancelled → keep partially written data per user expectation.//-// When the row is missing or unreadable we conservatively return-// (true, ParseStatusDeleting): the existing deleting branch already-// handles cleanup-or-no-op semantics safely.+// Only a row that is really gone reads as deleting. A transient read error+// must not: callers would wipe a live document's chunks and index and leave+// it stuck in processing, so the check reports "not aborted" and lets the+// pipeline's next write surface a real outage.func (s *knowledgeService) isKnowledgeAborted(ctx context.Context, tenantID uint64, knowledgeID string,) (bool, string) {knowledge, err := s.repo.GetKnowledgeByID(ctx, tenantID, knowledgeID)- if err != nil {- logger.Warnf(ctx, "Failed to check knowledge abort status (assuming deleted): %v", err)- return true, types.ParseStatusDeleting- }- if knowledge == nil {+ switch {+ case err == nil && knowledge == nil, errors.Is(err, repository.ErrKnowledgeNotFound):return true, types.ParseStatusDeleting+ case err != nil && ctx.Err() != nil:+ return true, abortStatusInterrupted+ case err != nil:+ logger.Warnf(ctx, "Failed to check knowledge abort status for %s (continuing): %v", knowledgeID, err)+ return false, ""}switch knowledge.ParseStatus {case types.ParseStatusDeleting, types.ParseStatusCancelled:diff --git a/internal/application/service/knowledge_post_process.go b/internal/application/service/knowledge_post_process.goindex af8eae867..3321c079b 100644--- a/internal/application/service/knowledge_post_process.go+++ b/internal/application/service/knowledge_post_process.go@@ -328,11 +328,12 @@ func (s *KnowledgePostProcessService) Handle(ctx context.Context, task *asynq.Tapromoted, err = s.knowledgeRepo.SetFinalizing(ctx, payload.KnowledgeID, expectedSubtasks)}if err != nil {+ // Acking here would read as "no longer processing" below and+ // strand the row; retry so the handoff (or dead-letter) happens.logger.Warnf(ctx, "[KnowledgePostProcess] SetFinalizing failed for %s: %v",payload.KnowledgeID, err)- if willSpawnWiki {- return fmt.Errorf("seed finalizing with wiki pending op: %w", err)- }+ s.tracker().FailSpan(ctx, postSpan, "FINALIZING_HANDOFF_FAILED", err.Error(), err)+ return fmt.Errorf("enter finalizing: %w", err)}if promoted {enteredFinalizing = truediff --git a/internal/application/service/knowledge_post_process_wiki_enqueue_test.go b/internal/application/service/knowledge_post_process_wiki_enqueue_test.goindex 26a6696b4..582e3370e 100644--- a/internal/application/service/knowledge_post_process_wiki_enqueue_test.go+++ b/internal/application/service/knowledge_post_process_wiki_enqueue_test.go@@ -17,6 +17,8 @@ type wikiEnqueueFailureKnowledgeRepo struct {interfaces.KnowledgeRepositoryknowledge *types.KnowledgeexpectedSubtasks int+ setFinalizingErr error+ finalized []string}func (r *wikiEnqueueFailureKnowledgeRepo) GetKnowledgeByIDOnly(@@ -31,15 +33,19 @@ func (r *wikiEnqueueFailureKnowledgeRepo) SetFinalizing(_ string,expectedSubtasks int,) (bool, error) {+ if r.setFinalizingErr != nil {+ return false, r.setFinalizingErr+ }r.expectedSubtasks = expectedSubtasksr.knowledge.ParseStatus = types.ParseStatusFinalizingreturn true, nil}func (r *wikiEnqueueFailureKnowledgeRepo) FinalizeSubtask(- context.Context,- string,+ _ context.Context,+ knowledgeID string,) (int, bool, error) {+ r.finalized = append(r.finalized, knowledgeID)return 0, false, nil}diff --git a/internal/application/service/knowledge_process.go b/internal/application/service/knowledge_process.goindex d79fabc36..45fc88ff8 100644--- a/internal/application/service/knowledge_process.go+++ b/internal/application/service/knowledge_process.go@@ -263,6 +263,15 @@ func markKnowledgeProcessing(knowledge *types.Knowledge, now time.Time) {// replacement and mark the replacement's live attempt failed.func (s *knowledgeService) failKnowledgeOnEmbeddingModel(ctx context.Context, kb *types.KnowledgeBase, knowledge *types.Knowledge, cause error,+) {+ s.failKnowledgeAtEmbedding(ctx, kb, knowledge, werrors.ErrCodeEmbeddingProviderFail,+ "failed to get embedding model", cause)+}++// failKnowledgeAtEmbedding records cause as this attempt's terminal state at+// the embedding stage; see failKnowledgeOnEmbeddingModel for the guards.+func (s *knowledgeService) failKnowledgeAtEmbedding(+ ctx context.Context, kb *types.KnowledgeBase, knowledge *types.Knowledge, code, what string, cause error,) {// A cancelled or expired context means the run was interrupted — the user// cancelled (asynq CancelProcessing cancels the handler context), the@@ -273,21 +282,21 @@ func (s *knowledgeService) failKnowledgeOnEmbeddingModel(// sweep when nothing else claims it.if ctx.Err() != nil || errors.Is(cause, context.Canceled) || errors.Is(cause, context.DeadlineExceeded) {logger.Infof(ctx,- "Embedding model resolution interrupted for %s (%v); leaving parse status untouched",- knowledge.ID, cause)+ "%s interrupted for %s (%v); leaving parse status untouched",+ what, knowledge.ID, cause)return}if aborted, status := s.isKnowledgeAborted(ctx, knowledge.TenantID, knowledge.ID); aborted {logger.Infof(ctx,- "Knowledge aborted (%s), not recording embedding model failure: %s", status, knowledge.ID)+ "Knowledge aborted (%s), not recording %q: %s", status, what, knowledge.ID)return}knowledge.ParseStatus = types.ParseStatusFailed- knowledge.ErrorMessage = fmt.Sprintf("failed to get embedding model: %v", cause)+ knowledge.ErrorMessage = fmt.Sprintf("%s: %v", what, cause)knowledge.UpdatedAt = time.Now()if err := s.updateKnowledgeUnlessSourceReplaced(ctx, knowledge); err != nil {- logger.Errorf(ctx, "failed to persist embedding model failure for %s: %v", knowledge.ID, err)+ logger.Errorf(ctx, "failed to persist %q for %s: %v", what, knowledge.ID, err)}s.beginStage(ctx, knowledge.ID, types.StageEmbedding, types.JSONMap{"model_id": kb.EmbeddingModelID,@@ -295,8 +304,7 @@ func (s *knowledgeService) failKnowledgeOnEmbeddingModel(// The span's error_message is what the timeline renders, and its// error_detail is withheld from non-admin responses — so carry the same// text the document list shows rather than a fixed generic string.- s.failStage(ctx, knowledge.ID, types.StageEmbedding,- werrors.ErrCodeEmbeddingProviderFail, knowledge.ErrorMessage, cause)+ s.failStage(ctx, knowledge.ID, types.StageEmbedding, code, knowledge.ErrorMessage, cause)}// buildSplitterConfigFromChunking normalizes effective chunking settings with@@ -400,7 +408,14 @@ func (s *knowledgeService) processChunks(ctx context.Context,tenantInfo := ctx.Value(types.TenantInfoContextKey).(*types.Tenant)retrieveEngine, err := retriever.CreateRetrieveEngineForKB(ctx, s.retrieveEngine, s.ownership, tenantInfo.ID, kb.VectorStoreID)- if err == nil && embeddingModel != nil {+ if err != nil && embeddingModel != nil {+ // Indexing below dereferences the engine; a nil one would panic.+ logger.Errorf(ctx, "processChunks resolve vector store for KB %s failed: %v", kb.ID, err)+ s.failKnowledgeAtEmbedding(ctx, kb, knowledge, werrors.ErrCodeVectorStoreWriteFailed,+ "failed to resolve vector store", err)+ return+ }+ if embeddingModel != nil {if err := retrieveEngine.DeleteByKnowledgeIDList(ctx, []string{knowledge.ID}, embeddingModel.GetDimensions(), knowledge.Type); err != nil {logger.Warnf(ctx, "Failed to delete existing index data (may not exist): %v", err)// 不返回错误,继续处理(可能没有旧数据)diff --git a/internal/application/service/knowledge_stuck_processing_test.go b/internal/application/service/knowledge_stuck_processing_test.gonew file mode 100644index 000000000..0928a66d3--- /dev/null+++ b/internal/application/service/knowledge_stuck_processing_test.go@@ -0,0 +1,207 @@+package service++import (+ "context"+ "encoding/json"+ "errors"+ "testing"+ "time"++ apprepo "github.com/Tencent/WeKnora/internal/application/repository"+ "github.com/Tencent/WeKnora/internal/models/chat"+ "github.com/Tencent/WeKnora/internal/types"+ "github.com/Tencent/WeKnora/internal/types/interfaces"+ "github.com/hibiken/asynq"+ "github.com/stretchr/testify/assert"+ "github.com/stretchr/testify/require"+)++// A failed processing→finalizing handoff must be retried, not acked: acking+// read as "no longer processing" and left the row stuck in processing.+func TestPostProcessRetriesWhenFinalizingHandoffFails(t *testing.T) {+ const knowledgeID = "knowledge-handoff-fails"+ queue := &wikiEnqueueFailureTaskQueue{}+ svc, repo := newWikiEnqueueTestService(knowledgeID, &wikiEnqueueFailurePendingRepo{}, queue)+ svc.kbService.(*wikiEnqueueFailureKBService).kb.IndexingStrategy.WikiEnabled = false+ repo.setFinalizingErr = errors.New("postgres unavailable")++ err := svc.Handle(context.Background(), newWikiEnqueuePostProcessTask(t, knowledgeID))++ require.ErrorIs(t, err, repo.setFinalizingErr)+ assert.Equal(t, types.ParseStatusProcessing, repo.knowledge.ParseStatus)+ assert.Empty(t, queue.taskTypes, "no enrichment may fan out before the handoff lands")+}++type wikiUnavailablePendingRepo struct {+ interfaces.TaskPendingOpsRepository+ drainKeys []string+ drainErr error+ drained []string+}++func (r *wikiUnavailablePendingRepo) DrainUnclaimed(+ _ context.Context, taskType, scope, scopeID, op string, _ time.Time,+) ([]string, error) {+ r.drained = append(r.drained, taskType+"|"+scope+"|"+scopeID+"|"+op)+ return r.drainKeys, r.drainErr+}++type wikiUnavailableModelService struct {+ interfaces.ModelService+ err error+}++func (s *wikiUnavailableModelService) GetChatModel(context.Context, string) (chat.Chat, error) {+ return nil, s.err+}++// A wiki that cannot run (disabled, no model, model deleted) fails the same+// way on every retry, so its queued ingest ops must be released instead of+// holding their documents in "finalizing" forever.+func TestWikiIngestReleasesDocumentsWhenWikiUnavailable(t *testing.T) {+ payload, err := json.Marshal(WikiIngestPayload{TenantID: 7, KnowledgeBaseID: "kb-1"})+ require.NoError(t, err)+ enabled := types.IndexingStrategy{WikiEnabled: true}+ tests := []struct {+ name string+ kb *types.KnowledgeBase+ modelErr error+ }{+ {name: "wiki disabled", kb: &types.KnowledgeBase{ID: "kb-1"}},+ {name: "no synthesis model", kb: &types.KnowledgeBase{ID: "kb-1", IndexingStrategy: enabled}},+ {+ name: "synthesis model deleted",+ kb: &types.KnowledgeBase{+ ID: "kb-1", IndexingStrategy: enabled,+ WikiConfig: &types.WikiConfig{SynthesisModelID: "gone"},+ },+ modelErr: ErrModelNotFound,+ },+ }+ for _, test := range tests {+ t.Run(test.name, func(t *testing.T) {+ pending := &wikiUnavailablePendingRepo{drainKeys: []string{"k-1", "k-2"}}+ knowledgeRepo := &wikiEnqueueFailureKnowledgeRepo{}+ svc := &wikiIngestService{+ kbService: &wikiGuardKBService{kb: test.kb},+ modelService: &wikiUnavailableModelService{err: test.modelErr},+ pendingRepo: pending,+ knowledgeRepo: knowledgeRepo,+ }++ err := svc.ProcessWikiIngest(context.Background(), asynq.NewTask(types.TypeWikiIngest, payload))++ require.NoError(t, err)+ assert.Equal(t, []string{wikiTaskType + "|" + wikiTaskScope + "|kb-1|" + WikiOpIngest}, pending.drained)+ assert.Equal(t, []string{"k-1", "k-2"}, knowledgeRepo.finalized)+ })+ }+}++// A transient model lookup failure keeps the ops and retries as before.+func TestWikiIngestKeepsOpsOnTransientModelError(t *testing.T) {+ payload, err := json.Marshal(WikiIngestPayload{TenantID: 7, KnowledgeBaseID: "kb-1"})+ require.NoError(t, err)+ pending := &wikiUnavailablePendingRepo{}+ transient := errors.New("connection reset")+ svc := &wikiIngestService{+ kbService: &wikiGuardKBService{kb: &types.KnowledgeBase{+ ID: "kb-1", IndexingStrategy: types.IndexingStrategy{WikiEnabled: true},+ WikiConfig: &types.WikiConfig{SynthesisModelID: "m-1"},+ }},+ modelService: &wikiUnavailableModelService{err: transient},+ pendingRepo: pending,+ }++ err = svc.ProcessWikiIngest(context.Background(), asynq.NewTask(types.TypeWikiIngest, payload))++ require.ErrorIs(t, err, transient)+ assert.Empty(t, pending.drained)+}++type abortCheckRepo struct {+ interfaces.KnowledgeRepository+ knowledge *types.Knowledge+ err error+}++func (r *abortCheckRepo) GetKnowledgeByID(context.Context, uint64, string) (*types.Knowledge, error) {+ return r.knowledge, r.err+}++// Only a row that is really gone may read as deleting; callers wipe chunks+// and index on that status.+func TestIsKnowledgeAbortedDistinguishesMissingFromUnreadable(t *testing.T) {+ cancelled, cancel := context.WithCancel(context.Background())+ cancel()+ tests := []struct {+ name string+ ctx context.Context+ repo *abortCheckRepo+ wantAborted bool+ wantStatus string+ }{+ {+ name: "missing", ctx: context.Background(),+ repo: &abortCheckRepo{err: apprepo.ErrKnowledgeNotFound},+ wantAborted: true, wantStatus: types.ParseStatusDeleting,+ },+ {+ name: "transient read error", ctx: context.Background(),+ repo: &abortCheckRepo{err: errors.New("connection reset")},+ },+ {+ name: "worker context done", ctx: cancelled,+ repo: &abortCheckRepo{err: context.Canceled},+ wantAborted: true, wantStatus: abortStatusInterrupted,+ },+ {+ name: "cancelled", ctx: context.Background(),+ repo: &abortCheckRepo{knowledge: &types.Knowledge{ParseStatus: types.ParseStatusCancelled}},+ wantAborted: true, wantStatus: types.ParseStatusCancelled,+ },+ {+ name: "processing", ctx: context.Background(),+ repo: &abortCheckRepo{knowledge: &types.Knowledge{ParseStatus: types.ParseStatusProcessing}},+ wantStatus: types.ParseStatusProcessing,+ },+ }+ for _, test := range tests {+ t.Run(test.name, func(t *testing.T) {+ svc := &knowledgeService{repo: test.repo}+ aborted, status := svc.isKnowledgeAborted(test.ctx, 1, "k-1")+ assert.Equal(t, test.wantAborted, aborted)+ assert.Equal(t, test.wantStatus, status)+ })+ }+}++// Rows held only by a durable wiki op get their KB's trigger re-armed, once+// per KB and at most once per threshold.+func TestHousekeepingRearmsWikiTriggerForDurablyHeldRows(t *testing.T) {+ db := setupHousekeepingDB(t)+ queue := &wikiGuardTaskQueue{}+ svc := newHousekeepingSvcForTest(db)+ svc.task = queue+ stale := time.Now().Add(-3 * time.Hour)+ for _, id := range []string{"k-1", "k-2"} {+ require.NoError(t, db.Exec(+ `INSERT INTO knowledges (id, tenant_id, knowledge_base_id, parse_status, updated_at)+ VALUES (?, 7, 'kb-1', ?, ?)`, id, types.ParseStatusFinalizing, stale,+ ).Error)+ insertWikiPendingOp(t, db, "kb-1", id)+ }++ svc.runSweep(context.Background())+ svc.runSweep(context.Background())++ require.Len(t, queue.tasks, 1)+ assert.Equal(t, types.TypeWikiIngest, queue.tasks[0].Type())+ var payload WikiIngestPayload+ require.NoError(t, json.Unmarshal(queue.tasks[0].Payload(), &payload))+ assert.Equal(t, uint64(7), payload.TenantID)+ assert.Equal(t, "kb-1", payload.KnowledgeBaseID)+ var status string+ require.NoError(t, db.Raw(`SELECT parse_status FROM knowledges WHERE id = 'k-1'`).Scan(&status).Error)+ assert.Equal(t, types.ParseStatusFinalizing, status)+}diff --git a/internal/middleware/asynqdl/recover.go b/internal/middleware/asynqdl/recover.gonew file mode 100644index 000000000..6cfb61424--- /dev/null+++ b/internal/middleware/asynqdl/recover.go@@ -0,0 +1,36 @@+package asynqdl++import (+ "context"+ "fmt"+ "runtime/debug"++ "github.com/Tencent/WeKnora/internal/logger"+ "github.com/hibiken/asynq"+)++// RecoverMiddleware turns a handler panic into an ordinary task error.+// asynq recovers panics itself, but only in its processor, above every+// middleware — so a panic skips the dead-letter callback and a document on+// its last attempt stays "processing" until housekeeping. Install it right+// after MiddlewareWithCallback so that callback sees the error.+func RecoverMiddleware() asynq.MiddlewareFunc {+ return func(next asynq.Handler) asynq.Handler {+ return asynq.HandlerFunc(func(ctx context.Context, t *asynq.Task) error {+ return CallRecovered(ctx, t, next.ProcessTask)+ })+ }+}++// CallRecovered runs handler, converting a panic into an error.+func CallRecovered(+ ctx context.Context, t *asynq.Task, handler func(context.Context, *asynq.Task) error,+) (err error) {+ defer func() {+ if r := recover(); r != nil {+ logger.Errorf(ctx, "task %s panicked: %v\n%s", t.Type(), r, debug.Stack())+ err = fmt.Errorf("task %s panicked: %v", t.Type(), r)+ }+ }()+ return handler(ctx, t)+}diff --git a/internal/middleware/asynqdl/recover_test.go b/internal/middleware/asynqdl/recover_test.gonew file mode 100644index 000000000..8044f2696--- /dev/null+++ b/internal/middleware/asynqdl/recover_test.go@@ -0,0 +1,29 @@+package asynqdl++import (+ "context"+ "testing"++ "github.com/hibiken/asynq"+ "github.com/stretchr/testify/assert"+ "github.com/stretchr/testify/require"+)++// A panic below RecoverMiddleware reaches the dead-letter callback as an+// ordinary error instead of unwinding past it.+func TestRecoverMiddleware_PanicReachesDeadLetterCallback(t *testing.T) {+ repo := &fakeRepo{}+ var callbackErr error+ cb := func(_ context.Context, _ *asynq.Task, err error) { callbackErr = err }+ panicking := asynq.HandlerFunc(func(context.Context, *asynq.Task) error {+ panic("nil retrieve engine")+ })+ h := MiddlewareWithCallback(repo, cb)(RecoverMiddleware()(panicking))++ err := h.ProcessTask(context.Background(), asynq.NewTask("document:process", []byte(`{}`)))++ require.Error(t, err)+ assert.Contains(t, err.Error(), "document:process panicked")+ assert.Equal(t, err, callbackErr)+ assert.Equal(t, 1, repo.rowCount())+}diff --git a/internal/router/sync_task.go b/internal/router/sync_task.goindex 8c8762f3e..28e5fab1a 100644--- a/internal/router/sync_task.go+++ b/internal/router/sync_task.go@@ -7,6 +7,7 @@ import ("time""github.com/Tencent/WeKnora/internal/logger"+ "github.com/Tencent/WeKnora/internal/middleware/asynqdl""github.com/Tencent/WeKnora/internal/types""github.com/Tencent/WeKnora/internal/types/interfaces""github.com/google/uuid"@@ -100,7 +101,8 @@ func (e *SyncTaskExecutor) Enqueue(task *asynq.Task, opts ...asynq.Option) (*asy}attemptCtx := types.WithTaskRetryMetadata(ctx, attempt, maxRetry)- lastErr = handler(attemptCtx, task)+ // An unrecovered panic here would take down the whole process.+ lastErr = asynqdl.CallRecovered(attemptCtx, task, handler)if lastErr == nil {logger.Infof(ctx, "[SyncTask] Task completed type=%s id=%s elapsed=%v",task.Type(), taskID, time.Since(start))diff --git a/internal/router/task.go b/internal/router/task.goindex fe0e240fe..4e3c8a279 100644--- a/internal/router/task.go+++ b/internal/router/task.go@@ -245,6 +245,7 @@ func RunAsynqServer(params AsynqTaskParams) *asynq.ServeMux {// UI signal users actually see.knowledgeFailer := newDeadLetterKnowledgeFailer(params.KnowledgeService, params.SpanTracker)mux.Use(asynqdl.MiddlewareWithCallback(params.DeadLetterRepo, knowledgeFailer))+ mux.Use(asynqdl.RecoverMiddleware())// Mark every asynq worker execution as a background task so the chat// concurrency governor throttles ingestion/enrichment LLM traffic while