Open-source LLM knowledge platform: turn raw documents into a queryable RAG, an autonomous reasoning agent, and a self-maintaining Wiki.
Go agentagenticaichatbotdsh-pluginembeddingsevaluationgenerative-aigolangknowledge-basellmmulti-tenantollamaopenaiquestion-answeringragrerankingsemantic-searchvector-searchwiki
fix(wiki): preserve pending work when source lookups fail (#3715)
xxxxx4d7e663128e7
diff --git a/internal/application/service/wiki_ingest.go b/internal/application/service/wiki_ingest.goindex 172033025..ef523b00d 100644--- a/internal/application/service/wiki_ingest.go+++ b/internal/application/service/wiki_ingest.go@@ -14,6 +14,7 @@ import ("unicode/utf8""github.com/Tencent/WeKnora/internal/agent"+ apprepo "github.com/Tencent/WeKnora/internal/application/repository""github.com/Tencent/WeKnora/internal/logger""github.com/Tencent/WeKnora/internal/models/chat""github.com/Tencent/WeKnora/internal/searchutil"@@ -531,7 +532,6 @@ func EnqueueWikiIngest(kbID, knowledgeID string,) (bool, error) {pendingOp, err := newWikiIngestPendingOp(ctx, tenantID, kbID, knowledgeID)-// Persist the pending op. A re-ingest of the same knowledge id while// a previous op is still queued simply appends another row; the// peekPendingList consumer collapses by dedup_key (== knowledge_id),@@ -3052,45 +3052,58 @@ func isTransientLLMError(ctx context.Context, err error) bool {// to the DB. A nil result from GetKnowledgeByIDOnly also counts as gone: the// repo layer uses GORM First() which filters soft-deleted rows, so a// soft-deleted knowledge surfaces as "not found" here — exactly what we want.-func (s *wikiIngestService) isKnowledgeGone(ctx context.Context, kbID, knowledgeID string) bool {+// Other lookup errors leave liveness unknown and must reach the caller so it+// can retry instead of discarding the document's pending work.+func (s *wikiIngestService) isKnowledgeGone(ctx context.Context, kbID, knowledgeID string) (bool, error) {if knowledgeID == "" {- return true+ return true, nil}if s.redisClient != nil {if exists, err := s.redisClient.Exists(ctx, WikiDeletedTombstoneKey(kbID, knowledgeID)).Result(); err == nil && exists > 0 {- return true+ return true, nil}}kn, err := s.knowledgeSvc.GetKnowledgeByIDOnly(ctx, knowledgeID)- if err != nil || kn == nil {- return true+ if errors.Is(err, apprepo.ErrKnowledgeNotFound) {+ return true, nil+ }+ if err != nil {+ return false, fmt.Errorf("wiki ingest: check knowledge %s: %w", knowledgeID, err)+ }+ if kn == nil {+ return true, nil}switch kn.ParseStatus {case types.ParseStatusDeleting, types.ParseStatusCancelled:- return true+ return true, nil}- return false+ return false, nil}// filterLiveUpdates drops additions/summaries whose source knowledge has been// deleted since the Map phase finished. Retract updates are preserved so// pages still get cleaned up. Caches per-knowledge results to avoid DB// hammering when a single reduce slug carries many updates for the same doc.-func (s *wikiIngestService) filterLiveUpdates(ctx context.Context, kbID string, updates []SlugUpdate) []SlugUpdate {+func (s *wikiIngestService) filterLiveUpdates(+ ctx context.Context, kbID string, updates []SlugUpdate,+) ([]SlugUpdate, error) {if len(updates) == 0 {- return updates+ return updates, nil}goneCache := make(map[string]bool)- isGone := func(kid string) bool {+ isGone := func(kid string) (bool, error) {if kid == "" {- return false+ return false, nil}if v, ok := goneCache[kid]; ok {- return v+ return v, nil+ }+ v, err := s.isKnowledgeGone(ctx, kbID, kid)+ if err != nil {+ return false, err}- v := s.isKnowledgeGone(ctx, kbID, kid)goneCache[kid] = v- return v+ return v, nil}filtered := make([]SlugUpdate, 0, len(updates))dropped := 0@@ -3099,7 +3112,11 @@ func (s *wikiIngestService) filterLiveUpdates(ctx context.Context, kbID string,case "retract", "retractStale":filtered = append(filtered, u)default:- if isGone(u.KnowledgeID) {+ gone, err := isGone(u.KnowledgeID)+ if err != nil {+ return nil, err+ }+ if gone {dropped++continue}@@ -3109,7 +3126,7 @@ func (s *wikiIngestService) filterLiveUpdates(ctx context.Context, kbID string,if dropped > 0 {logger.Infof(ctx, "wiki ingest: reduce dropped %d updates for deleted knowledge(s)", dropped)}- return filtered+ return filtered, nil}// reconstructContent rebuilds document text from chunks.diff --git a/internal/application/service/wiki_ingest_batch.go b/internal/application/service/wiki_ingest_batch.goindex e9c743971..06988a826 100644--- a/internal/application/service/wiki_ingest_batch.go+++ b/internal/application/service/wiki_ingest_batch.go@@ -1332,7 +1332,14 @@ func (s *wikiIngestService) mapOneDocument(// was in flight, we must NOT proceed to LLM extraction — doing so would// create wiki pages whose source_refs point at a ghost knowledge ID,// permanently unreachable via wiki_read_source_doc.- if s.isKnowledgeGone(ctx, payload.KnowledgeBaseID, knowledgeID) {+ gone, err := s.isKnowledgeGone(ctx, payload.KnowledgeBaseID, knowledgeID)+ if err != nil {+ failCtx, cancel := wikiIngestCleanupContext(ctx)+ defer cancel()+ s.tracker().FailSpan(failCtx, wikiSpan, "KNOWLEDGE_LOOKUP_FAILED", err.Error(), err)+ return nil, nil, err+ }+ if gone {logger.Infof(ctx, "wiki ingest: knowledge %s has been deleted, skip map", knowledgeID)s.tracker().SkipSpan(ctx, wikiSpan, "knowledge_deleted")return nil, nil, nil@@ -1868,7 +1875,10 @@ func (s *wikiIngestService) reduceSlugUpdates(// knowledge no longer exists so we don't resurrect a ghost source_ref.// Retract updates are kept — they actively remove refs, which is what we// want when the doc is gone.- updates = s.filterLiveUpdates(ctx, kbID, updates)+ updates, err = s.filterLiveUpdates(ctx, kbID, updates)+ if err != nil {+ return false, "", false, err+ }if len(updates) == 0 {return false, "", false, nil}diff --git a/internal/application/service/wiki_ingest_cancellation_stress_test.go b/internal/application/service/wiki_ingest_cancellation_stress_test.gonew file mode 100644index 000000000..fed603312--- /dev/null+++ b/internal/application/service/wiki_ingest_cancellation_stress_test.go@@ -0,0 +1,68 @@+package service++import (+ "context"+ "fmt"+ "testing"++ "github.com/Tencent/WeKnora/internal/types"+ "github.com/alicebob/miniredis/v2"+ "github.com/hibiken/asynq"+ "github.com/redis/go-redis/v9"+ "github.com/stretchr/testify/assert"+ "github.com/stretchr/testify/require"+)++// Each worker repeatedly interrupts a full batch after claiming it, then+// recovers the same rows. Redis is shared; each KB has a separate in-memory+// SQLite database because this tests handler recovery, not PostgreSQL locking.+func TestWikiInterruptedLookupRecoveryStress(t *testing.T) {+ const workers, rounds, batchSize = 16, 10, 5+ mr := miniredis.RunT(t)+ rdb := redis.NewClient(&redis.Options{Addr: mr.Addr()})+ t.Cleanup(func() { _ = rdb.Close() })+ for worker := range workers {+ t.Run(fmt.Sprintf("kb-%d", worker), func(t *testing.T) {+ t.Parallel()+ kbID := fmt.Sprintf("kb-%d", worker)+ svc, db := newWikiLookupRecoveryService(t, kbID, rdb)+ lookup := svc.knowledgeSvc.(*wikiKnowledgeLookupStub)+ for round := range rounds {+ var task *asynq.Task+ for doc := range batchSize {+ id := fmt.Sprintf("k-%d-%d-%d", worker, round, doc)+ task = seedWikiLookupTask(t, svc, db, kbID, id)+ }+ ctx, cancel := context.WithCancel(context.Background())+ lookup.lookup = func(ctx context.Context, id string) (*types.Knowledge, error) {+ cancel()+ return svc.knowledgeRepo.GetKnowledgeByIDOnly(ctx, id)+ }+ err := svc.ProcessWikiIngest(ctx, task)+ cancel()+ require.NoError(t, err)+ var ops []types.TaskPendingOp+ require.NoError(t, db.Find(&ops).Error)+ require.Len(t, ops, batchSize, "every interrupted op must remain retryable")+ for _, op := range ops {+ require.Nil(t, op.ClaimedAt)+ require.Equal(t, 1, op.FailCount)+ }+ require.Zero(t, rdb.ZCard(context.Background(), wikiInflightPrefix+kbID).Val())++ lookup.lookup = svc.knowledgeRepo.GetKnowledgeByIDOnly+ require.NoError(t, svc.ProcessWikiIngest(context.Background(), task))+ count, err := svc.pendingRepo.PendingCount(context.Background(), wikiTaskType, wikiTaskScope, kbID)+ require.NoError(t, err)+ require.Zero(t, count, "healthy follow-up must drain the preserved work")+ require.Zero(t, rdb.ZCard(context.Background(), wikiInflightPrefix+kbID).Val())+ }+ var completed int64+ require.NoError(t, db.Model(&types.Knowledge{}).+ Where("parse_status = ? AND pending_subtasks_count = 0", types.ParseStatusCompleted).+ Count(&completed).Error)+ assert.EqualValues(t, rounds*batchSize, completed)+ assert.Len(t, svc.task.(*wikiGuardTaskQueue).tasks, rounds)+ })+ }+}diff --git a/internal/application/service/wiki_ingest_cancellation_test.go b/internal/application/service/wiki_ingest_cancellation_test.gonew file mode 100644index 000000000..aa764eb76--- /dev/null+++ b/internal/application/service/wiki_ingest_cancellation_test.go@@ -0,0 +1,320 @@+package service++import (+ "context"+ "encoding/json"+ "errors"+ "fmt"+ "testing"+ "time"++ apprepo "github.com/Tencent/WeKnora/internal/application/repository"+ "github.com/Tencent/WeKnora/internal/models/chat"+ "github.com/Tencent/WeKnora/internal/types"+ "github.com/Tencent/WeKnora/internal/types/interfaces"+ "github.com/alicebob/miniredis/v2"+ "github.com/hibiken/asynq"+ "github.com/redis/go-redis/v9"+ "github.com/stretchr/testify/assert"+ "github.com/stretchr/testify/require"+ "gorm.io/driver/sqlite"+ "gorm.io/gorm"+)++type wikiKnowledgeLookupStub struct {+ interfaces.KnowledgeService+ lookup func(context.Context, string) (*types.Knowledge, error)+}++func (s *wikiKnowledgeLookupStub) GetKnowledgeByIDOnly(ctx context.Context, id string) (*types.Knowledge, error) {+ return s.lookup(ctx, id)+}++// A canceled or failed lookup does not establish that the source was deleted.+// Map must return an error so the batch keeps the durable op for retry.+func TestWikiMapPreservesLookupFailures(t *testing.T) {+ for _, lookupErr := range []error{context.Canceled, context.DeadlineExceeded, errors.New("database unavailable")} {+ t.Run(lookupErr.Error(), func(t *testing.T) {+ svc := &wikiIngestService{knowledgeSvc: &wikiKnowledgeLookupStub{+ lookup: func(context.Context, string) (*types.Knowledge, error) { return nil, lookupErr },+ }}+ result, updates, err := svc.mapOneDocument(context.Background(), nil,+ WikiIngestPayload{TenantID: 7, KnowledgeBaseID: "kb-1"},+ WikiPendingOp{KnowledgeID: "k-1", Op: WikiOpIngest}, nil)+ require.ErrorIs(t, err, lookupErr)+ assert.Nil(t, result)+ assert.Empty(t, updates)+ })+ }+}++// Reduce must not report a successful no-op when it cannot read a source:+// the batch otherwise deletes that document's durable op without its pages.+func TestWikiReducePreservesLookupFailures(t *testing.T) {+ for _, lookupErr := range []error{context.Canceled, context.DeadlineExceeded, errors.New("database unavailable")} {+ t.Run(lookupErr.Error(), func(t *testing.T) {+ svc := &wikiIngestService{knowledgeSvc: &wikiKnowledgeLookupStub{+ lookup: func(context.Context, string) (*types.Knowledge, error) { return nil, lookupErr },+ }}+ changed, _, _, err := svc.reduceSlugUpdates(context.Background(), nil, "kb-1", "entity/example",+ []SlugUpdate{{Slug: "entity/example", Type: "entity", KnowledgeID: "k-1"}}, 7, nil, nil)+ require.ErrorIs(t, err, lookupErr)+ assert.False(t, changed)+ })+ }+}++func TestWikiLiveUpdatesKeepDeletionAndRetractionSemantics(t *testing.T) {+ lookupCalls := make(map[string]int)+ svc := &wikiIngestService{knowledgeSvc: &wikiKnowledgeLookupStub{+ lookup: func(_ context.Context, id string) (*types.Knowledge, error) {+ lookupCalls[id]+++ switch id {+ case "missing":+ return nil, fmt.Errorf("lookup: %w", apprepo.ErrKnowledgeNotFound)+ case "nil":+ return nil, nil+ case "deleting", "cancelled":+ return &types.Knowledge{ParseStatus: id}, nil+ default:+ return &types.Knowledge{ID: id, ParseStatus: types.ParseStatusFinalizing}, nil+ }+ },+ }}+ updates := []SlugUpdate{+ {Type: "entity", KnowledgeID: "live"},+ {Type: "summary", KnowledgeID: "live"},+ {Type: "entity", KnowledgeID: "missing"},+ {Type: "summary", KnowledgeID: "nil"},+ {Type: "entity", KnowledgeID: "deleting"},+ {Type: "entity", KnowledgeID: "cancelled"},+ {Type: "retract", KnowledgeID: "missing"},+ {Type: "retractStale", KnowledgeID: "missing"},+ {Type: "entity"},+ }+ got, err := svc.filterLiveUpdates(context.Background(), "kb-1", updates)+ require.NoError(t, err)+ assert.Equal(t, []SlugUpdate{updates[0], updates[1], updates[6], updates[7], updates[8]}, got)+ assert.Equal(t, 1, lookupCalls["live"], "shared source lookup is cached within the slug")+ assert.Equal(t, 1, lookupCalls["missing"])+}++func TestWikiLiveUpdatesRejectPartialLookupResults(t *testing.T) {+ lookupErr := errors.New("database unavailable")+ svc := &wikiIngestService{knowledgeSvc: &wikiKnowledgeLookupStub{+ lookup: func(_ context.Context, id string) (*types.Knowledge, error) {+ if id == "unreadable" {+ return nil, lookupErr+ }+ return &types.Knowledge{ID: id}, nil+ },+ }}+ got, err := svc.filterLiveUpdates(context.Background(), "kb-1", []SlugUpdate{+ {Type: "retract", KnowledgeID: "deleted"},+ {Type: "entity", KnowledgeID: "live"},+ {Type: "entity", KnowledgeID: "unreadable"},+ })+ require.ErrorIs(t, err, lookupErr)+ assert.Nil(t, got, "a partial slug must not be applied as a successful complete update")+}++func TestWikiKnowledgeTombstoneSkipsLookup(t *testing.T) {+ mr := miniredis.RunT(t)+ rdb := redis.NewClient(&redis.Options{Addr: mr.Addr()})+ t.Cleanup(func() { _ = rdb.Close() })+ ctx := context.Background()+ require.NoError(t, rdb.Set(ctx, WikiDeletedTombstoneKey("kb-1", "k-1"), "1", time.Minute).Err())+ svc := &wikiIngestService{redisClient: rdb, knowledgeSvc: &wikiKnowledgeLookupStub{+ lookup: func(context.Context, string) (*types.Knowledge, error) {+ t.Error("tombstoned knowledge should not require a database lookup")+ return nil, errors.New("database unavailable")+ },+ }}+ result, updates, err := svc.mapOneDocument(ctx, nil, WikiIngestPayload{KnowledgeBaseID: "kb-1"},+ WikiPendingOp{KnowledgeID: "k-1"}, nil)+ require.NoError(t, err)+ assert.Nil(t, result)+ assert.Empty(t, updates)+}++type wikiLookupKBService struct {+ interfaces.KnowledgeBaseService+ kb *types.KnowledgeBase+}++func (s *wikiLookupKBService) GetKnowledgeBaseByIDOnly(context.Context, string) (*types.KnowledgeBase, error) {+ return s.kb, nil+}++func (s *wikiLookupKBService) GetKnowledgeBaseByID(context.Context, string) (*types.KnowledgeBase, error) {+ return s.kb, nil+}++type wikiLookupModelService struct{ interfaces.ModelService }++func (wikiLookupModelService) GetChatModel(context.Context, string) (chat.Chat, error) {+ // The recovery fixture has no chunks, so the real pipeline skips the LLM.+ return nil, nil+}++type wikiLookupEmptyChunks struct{ interfaces.ChunkRepository }++func (wikiLookupEmptyChunks) ListChunksByKnowledgeID(context.Context, uint64, string) ([]*types.Chunk, error) {+ return nil, nil+}++func newWikiLookupRecoveryService(t *testing.T, kbID string, rdb *redis.Client) (*wikiIngestService, *gorm.DB) {+ t.Helper()+ db, err := gorm.Open(sqlite.Open(":memory:"), &gorm.Config{})+ require.NoError(t, err)+ sqlDB, err := db.DB()+ require.NoError(t, err)+ sqlDB.SetMaxOpenConns(1)+ t.Cleanup(func() { _ = sqlDB.Close() })+ require.NoError(t, db.Exec(knowledgeTestDDL).Error)+ require.NoError(t, db.Exec("ALTER TABLE knowledges ADD COLUMN processed_at DATETIME").Error)+ require.NoError(t, db.AutoMigrate(&types.TaskPendingOp{}))+ require.NoError(t, db.Exec("CREATE TABLE tenants (id INTEGER PRIMARY KEY, deleted_at DATETIME)").Error)+ require.NoError(t, db.Exec("INSERT INTO tenants (id) VALUES (7)").Error)+ repo := apprepo.NewKnowledgeRepository(db)+ svc := &wikiIngestService{+ kbService: &wikiLookupKBService{kb: &types.KnowledgeBase{+ ID: kbID, SummaryModelID: "unused", IndexingStrategy: types.IndexingStrategy{WikiEnabled: true},+ }},+ modelService: wikiLookupModelService{}, chunkRepo: wikiLookupEmptyChunks{},+ knowledgeRepo: repo, knowledgeSvc: &wikiKnowledgeLookupStub{lookup: repo.GetKnowledgeByIDOnly},+ pendingRepo: apprepo.NewTaskPendingOpsRepository(db), task: &wikiGuardTaskQueue{}, redisClient: rdb,+ }+ return svc, db+}++func seedWikiLookupTask(t *testing.T, svc *wikiIngestService, db *gorm.DB, kbID, knowledgeID string) *asynq.Task {+ t.Helper()+ require.NoError(t, db.Exec(`INSERT INTO knowledges+ (id, tenant_id, knowledge_base_id, parse_status, pending_subtasks_count) VALUES (?, 7, ?, ?, 1)`,+ knowledgeID, kbID, types.ParseStatusFinalizing).Error)+ op, err := newWikiIngestPendingOp(context.Background(), 7, kbID, knowledgeID)+ require.NoError(t, err)+ require.NoError(t, svc.pendingRepo.Enqueue(context.Background(), op))+ payload, err := json.Marshal(WikiIngestPayload{TenantID: 7, KnowledgeBaseID: kbID})+ require.NoError(t, err)+ return asynq.NewTask(types.TypeWikiIngest, payload)+}++// Exercise the real batch handler, durable repository and finalizing counter.+// Cancellation happens after the op is claimed, at the source-document read.+func TestWikiIngestRetainsInterruptedLookupAndRecovers(t *testing.T) {+ for _, mode := range []string{"lite", "redis"} {+ t.Run(mode, func(t *testing.T) {+ var rdb *redis.Client+ if mode == "redis" {+ mr := miniredis.RunT(t)+ rdb = redis.NewClient(&redis.Options{Addr: mr.Addr()})+ t.Cleanup(func() { _ = rdb.Close() })+ }+ svc, db := newWikiLookupRecoveryService(t, "kb-1", rdb)+ task := seedWikiLookupTask(t, svc, db, "kb-1", "k-1")+ ctx, cancel := context.WithCancel(context.Background())+ defer cancel()+ lookup := svc.knowledgeSvc.(*wikiKnowledgeLookupStub)+ lookup.lookup = func(ctx context.Context, id string) (*types.Knowledge, error) {+ cancel()+ return svc.knowledgeRepo.GetKnowledgeByIDOnly(ctx, id)+ }+ require.NoError(t, svc.ProcessWikiIngest(ctx, task))+ var pending types.TaskPendingOp+ require.NoError(t, db.First(&pending).Error, "interrupted source must stay queued")+ assert.Equal(t, 1, pending.FailCount)+ assert.Nil(t, pending.ClaimedAt, "next batch must be able to reclaim immediately")+ kn, err := svc.knowledgeRepo.GetKnowledgeByIDOnly(context.Background(), "k-1")+ require.NoError(t, err)+ assert.Equal(t, types.ParseStatusFinalizing, kn.ParseStatus)+ assert.Equal(t, 1, kn.PendingSubtasksCount)+ require.Len(t, svc.task.(*wikiGuardTaskQueue).tasks, 1, "retry must be scheduled")+ if rdb != nil {+ require.Zero(t, rdb.ZCard(context.Background(), wikiInflightPrefix+"kb-1").Val())+ }++ lookup.lookup = svc.knowledgeRepo.GetKnowledgeByIDOnly+ require.NoError(t, svc.ProcessWikiIngest(context.Background(), task))+ count, err := svc.pendingRepo.PendingCount(context.Background(), wikiTaskType, wikiTaskScope, "kb-1")+ require.NoError(t, err)+ assert.Zero(t, count)+ kn, err = svc.knowledgeRepo.GetKnowledgeByIDOnly(context.Background(), "k-1")+ require.NoError(t, err)+ assert.Equal(t, types.ParseStatusCompleted, kn.ParseStatus)+ assert.Zero(t, kn.PendingSubtasksCount)+ })+ }+}++func TestWikiMapClosesSpanAfterLookupCancellation(t *testing.T) {+ tracker, db := setupSpanTrackerTest(t)+ sqlDB, err := db.DB()+ require.NoError(t, err)+ t.Cleanup(func() { _ = sqlDB.Close() })+ ctx, cancel := context.WithCancel(context.Background())+ defer cancel()+ _, attempt, err := tracker.OpenAttempt(ctx, "k-1", "")+ require.NoError(t, err)+ require.NotNil(t, tracker.BeginStage(ctx, "k-1", attempt, types.StagePostProcess, nil))+ svc := &wikiIngestService{spanTracker: tracker, knowledgeSvc: &wikiKnowledgeLookupStub{+ lookup: func(ctx context.Context, _ string) (*types.Knowledge, error) {+ cancel()+ return nil, ctx.Err()+ },+ }}+ _, _, err = svc.mapOneDocument(ctx, nil, WikiIngestPayload{KnowledgeBaseID: "kb-1"},+ WikiPendingOp{KnowledgeID: "k-1"}, nil)+ require.ErrorIs(t, err, context.Canceled)+ var span types.KnowledgeProcessingSpan+ require.NoError(t, db.Where("name = ?", "postprocess.wiki").First(&span).Error)+ assert.Equal(t, types.SpanStatusFailed, span.Status, "interrupted map must not remain running in the trace")+ assert.NotNil(t, span.FinishedAt)+}++func TestWikiLookupFailureDoesNotBlockHealthySibling(t *testing.T) {+ svc, db := newWikiLookupRecoveryService(t, "kb-1", nil)+ seedWikiLookupTask(t, svc, db, "kb-1", "unreadable")+ task := seedWikiLookupTask(t, svc, db, "kb-1", "healthy")+ lookup := svc.knowledgeSvc.(*wikiKnowledgeLookupStub)+ lookup.lookup = func(ctx context.Context, id string) (*types.Knowledge, error) {+ if id == "unreadable" {+ return nil, errors.New("temporary read failure")+ }+ return svc.knowledgeRepo.GetKnowledgeByIDOnly(ctx, id)+ }+ require.NoError(t, svc.ProcessWikiIngest(context.Background(), task))+ var ops []types.TaskPendingOp+ require.NoError(t, db.Find(&ops).Error)+ require.Len(t, ops, 1)+ assert.Equal(t, "unreadable", ops[0].DedupKey)+ healthy, err := svc.knowledgeRepo.GetKnowledgeByIDOnly(context.Background(), "healthy")+ require.NoError(t, err)+ assert.Equal(t, types.ParseStatusCompleted, healthy.ParseStatus)+}++func TestWikiPersistentLookupFailureExhaustsRetryBudget(t *testing.T) {+ svc, db := newWikiLookupRecoveryService(t, "kb-1", nil)+ require.NoError(t, db.AutoMigrate(&types.TaskDeadLetter{}))+ svc.deadLetterRepo = apprepo.NewTaskDeadLetterRepository(db)+ task := seedWikiLookupTask(t, svc, db, "kb-1", "unreadable")+ svc.knowledgeSvc.(*wikiKnowledgeLookupStub).lookup = func(context.Context, string) (*types.Knowledge, error) {+ return nil, errors.New("persistent read failure")+ }+ for range wikiMaxFailRetries + 1 {+ require.NoError(t, svc.ProcessWikiIngest(context.Background(), task))+ }+ count, err := svc.pendingRepo.PendingCount(context.Background(), wikiTaskType, wikiTaskScope, "kb-1")+ require.NoError(t, err)+ assert.Zero(t, count, "persistent failure must not retry forever")+ var letters []types.TaskDeadLetter+ require.NoError(t, db.Find(&letters).Error)+ require.Len(t, letters, 1)+ assert.Equal(t, "unreadable", letters[0].RelatedID)+ assert.Equal(t, wikiMaxFailRetries+1, letters[0].FailCount)+ kn, err := svc.knowledgeRepo.GetKnowledgeByIDOnly(context.Background(), "unreadable")+ require.NoError(t, err)+ assert.Zero(t, kn.PendingSubtasksCount, "exhausted work must release the finalizing slot")+ assert.NotEqual(t, types.ParseStatusFinalizing, kn.ParseStatus)+}diff --git a/internal/application/service/wiki_ingest_http_cancellation_test.go b/internal/application/service/wiki_ingest_http_cancellation_test.gonew file mode 100644index 000000000..0dbe4ae83--- /dev/null+++ b/internal/application/service/wiki_ingest_http_cancellation_test.go@@ -0,0 +1,95 @@+package service++import (+ "context"+ "io"+ "net/http"+ "net/http/httptest"+ "sync/atomic"+ "testing"+ "time"++ "github.com/Tencent/WeKnora/internal/models/api"+ "github.com/Tencent/WeKnora/internal/models/api/openaicompletions"+ "github.com/Tencent/WeKnora/internal/types"+ "github.com/stretchr/testify/require"+)++// Exercise the production HTTP client through Wiki's request coalescing layer.+// A model that never sends headers or stops in the middle of a response must+// not prevent cancellation or a subsequent independent request from completing.+func TestWikiHTTPModelCancellationAndRecovery(t *testing.T) {+ withSSRFWhitelist(t, "127.0.0.1")+ for _, partialBody := range []bool{false, true} {+ name := "before_headers"+ if partialBody {+ name = "during_body"+ }+ t.Run(name, func(t *testing.T) {+ started, stopped := make(chan struct{}), make(chan struct{})+ shutdown := make(chan struct{})+ var calls atomic.Int32+ server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {+ _, _ = io.Copy(io.Discard, r.Body)+ if calls.Add(1) == 1 {+ if partialBody {+ w.Header().Set("Content-Type", "application/json")+ _, _ = io.WriteString(w, `{"choices":[`)+ w.(http.Flusher).Flush()+ }+ close(started)+ select {+ case <-r.Context().Done():+ case <-shutdown:+ }+ close(stopped)+ return+ }+ w.Header().Set("Content-Type", "application/json")+ _, _ = io.WriteString(w, `{"choices":[{"message":{"content":"recovered"},"finish_reason":"stop"}]}`)+ }))+ t.Cleanup(func() {+ close(shutdown)+ server.Close()+ })+ model := openaicompletions.New(openaicompletions.Config{+ Endpoint: api.Endpoint{+ BaseURL: server.URL, Model: "fixture", ModelID: "fixture", Client: server.Client(),+ },+ Settings: api.DefaultOpenAICompletions(),+ })+ svc := &wikiIngestService{}+ parent := context.WithValue(context.Background(), types.TenantIDContextKey, uint64(7))+ ctx, cancel := context.WithCancel(parent)+ defer cancel()+ result := make(chan error, 1)+ go func() {+ _, err := svc.generateWithTemplate(ctx, model, "stalled request", nil)+ result <- err+ }()+ select {+ case <-started:+ case <-time.After(3 * time.Second):+ t.Fatal("model request did not reach the HTTP server")+ }+ cancel()+ select {+ case err := <-result:+ require.ErrorIs(t, err, context.Canceled)+ case <-time.After(3 * time.Second):+ t.Fatal("Wiki model call did not exit on cancellation")+ }+ select {+ case <-stopped:+ case <-time.After(3 * time.Second):+ t.Fatal("the underlying HTTP request remained active after cancellation")+ }+ recoveryCtx, recoveryCancel := context.WithTimeout(parent, 3*time.Second)+ defer recoveryCancel()+ content, err := svc.generateWithTemplate(recoveryCtx, model, "independent recovery request", nil)+ require.NoError(t, err)+ require.Equal(t, "recovered", content)+ require.EqualValues(t, 2, calls.Load())+ })+ }+}