Skip to content

Open-source LLM knowledge platform: turn raw documents into a queryable RAG, an autonomous reasoning agent, and a self-maintaining Wiki.

fix(wiki): preserve pending work when source lookups fail (#3715)

xxxxx4d7e663128e7
diff --git a/internal/application/service/wiki_ingest.go b/internal/application/service/wiki_ingest.go
index 172033025..ef523b00d 100644
--- a/internal/application/service/wiki_ingest.go
+++ b/internal/application/service/wiki_ingest.go
@@ -14,6 +14,7 @@ import (
"unicode/utf8"
"github.com/Tencent/WeKnora/internal/agent"
+ apprepo "github.com/Tencent/WeKnora/internal/application/repository"
"github.com/Tencent/WeKnora/internal/logger"
"github.com/Tencent/WeKnora/internal/models/chat"
"github.com/Tencent/WeKnora/internal/searchutil"
@@ -531,7 +532,6 @@ func EnqueueWikiIngest(
kbID, knowledgeID string,
) (bool, error) {
pendingOp, err := newWikiIngestPendingOp(ctx, tenantID, kbID, knowledgeID)
-
// Persist the pending op. A re-ingest of the same knowledge id while
// a previous op is still queued simply appends another row; the
// peekPendingList consumer collapses by dedup_key (== knowledge_id),
@@ -3052,45 +3052,58 @@ func isTransientLLMError(ctx context.Context, err error) bool {
// to the DB. A nil result from GetKnowledgeByIDOnly also counts as gone: the
// repo layer uses GORM First() which filters soft-deleted rows, so a
// soft-deleted knowledge surfaces as "not found" here — exactly what we want.
-func (s *wikiIngestService) isKnowledgeGone(ctx context.Context, kbID, knowledgeID string) bool {
+// Other lookup errors leave liveness unknown and must reach the caller so it
+// can retry instead of discarding the document's pending work.
+func (s *wikiIngestService) isKnowledgeGone(ctx context.Context, kbID, knowledgeID string) (bool, error) {
if knowledgeID == "" {
- return true
+ return true, nil
}
if s.redisClient != nil {
if exists, err := s.redisClient.Exists(ctx, WikiDeletedTombstoneKey(kbID, knowledgeID)).Result(); err == nil && exists > 0 {
- return true
+ return true, nil
}
}
kn, err := s.knowledgeSvc.GetKnowledgeByIDOnly(ctx, knowledgeID)
- if err != nil || kn == nil {
- return true
+ if errors.Is(err, apprepo.ErrKnowledgeNotFound) {
+ return true, nil
+ }
+ if err != nil {
+ return false, fmt.Errorf("wiki ingest: check knowledge %s: %w", knowledgeID, err)
+ }
+ if kn == nil {
+ return true, nil
}
switch kn.ParseStatus {
case types.ParseStatusDeleting, types.ParseStatusCancelled:
- return true
+ return true, nil
}
- return false
+ return false, nil
}
// filterLiveUpdates drops additions/summaries whose source knowledge has been
// deleted since the Map phase finished. Retract updates are preserved so
// pages still get cleaned up. Caches per-knowledge results to avoid DB
// hammering when a single reduce slug carries many updates for the same doc.
-func (s *wikiIngestService) filterLiveUpdates(ctx context.Context, kbID string, updates []SlugUpdate) []SlugUpdate {
+func (s *wikiIngestService) filterLiveUpdates(
+ ctx context.Context, kbID string, updates []SlugUpdate,
+) ([]SlugUpdate, error) {
if len(updates) == 0 {
- return updates
+ return updates, nil
}
goneCache := make(map[string]bool)
- isGone := func(kid string) bool {
+ isGone := func(kid string) (bool, error) {
if kid == "" {
- return false
+ return false, nil
}
if v, ok := goneCache[kid]; ok {
- return v
+ return v, nil
+ }
+ v, err := s.isKnowledgeGone(ctx, kbID, kid)
+ if err != nil {
+ return false, err
}
- v := s.isKnowledgeGone(ctx, kbID, kid)
goneCache[kid] = v
- return v
+ return v, nil
}
filtered := make([]SlugUpdate, 0, len(updates))
dropped := 0
@@ -3099,7 +3112,11 @@ func (s *wikiIngestService) filterLiveUpdates(ctx context.Context, kbID string,
case "retract", "retractStale":
filtered = append(filtered, u)
default:
- if isGone(u.KnowledgeID) {
+ gone, err := isGone(u.KnowledgeID)
+ if err != nil {
+ return nil, err
+ }
+ if gone {
dropped++
continue
}
@@ -3109,7 +3126,7 @@ func (s *wikiIngestService) filterLiveUpdates(ctx context.Context, kbID string,
if dropped > 0 {
logger.Infof(ctx, "wiki ingest: reduce dropped %d updates for deleted knowledge(s)", dropped)
}
- return filtered
+ return filtered, nil
}
// reconstructContent rebuilds document text from chunks.
diff --git a/internal/application/service/wiki_ingest_batch.go b/internal/application/service/wiki_ingest_batch.go
index e9c743971..06988a826 100644
--- a/internal/application/service/wiki_ingest_batch.go
+++ b/internal/application/service/wiki_ingest_batch.go
@@ -1332,7 +1332,14 @@ func (s *wikiIngestService) mapOneDocument(
// was in flight, we must NOT proceed to LLM extraction — doing so would
// create wiki pages whose source_refs point at a ghost knowledge ID,
// permanently unreachable via wiki_read_source_doc.
- if s.isKnowledgeGone(ctx, payload.KnowledgeBaseID, knowledgeID) {
+ gone, err := s.isKnowledgeGone(ctx, payload.KnowledgeBaseID, knowledgeID)
+ if err != nil {
+ failCtx, cancel := wikiIngestCleanupContext(ctx)
+ defer cancel()
+ s.tracker().FailSpan(failCtx, wikiSpan, "KNOWLEDGE_LOOKUP_FAILED", err.Error(), err)
+ return nil, nil, err
+ }
+ if gone {
logger.Infof(ctx, "wiki ingest: knowledge %s has been deleted, skip map", knowledgeID)
s.tracker().SkipSpan(ctx, wikiSpan, "knowledge_deleted")
return nil, nil, nil
@@ -1868,7 +1875,10 @@ func (s *wikiIngestService) reduceSlugUpdates(
// knowledge no longer exists so we don't resurrect a ghost source_ref.
// Retract updates are kept — they actively remove refs, which is what we
// want when the doc is gone.
- updates = s.filterLiveUpdates(ctx, kbID, updates)
+ updates, err = s.filterLiveUpdates(ctx, kbID, updates)
+ if err != nil {
+ return false, "", false, err
+ }
if len(updates) == 0 {
return false, "", false, nil
}
diff --git a/internal/application/service/wiki_ingest_cancellation_stress_test.go b/internal/application/service/wiki_ingest_cancellation_stress_test.go
new file mode 100644
index 000000000..fed603312
--- /dev/null
+++ b/internal/application/service/wiki_ingest_cancellation_stress_test.go
@@ -0,0 +1,68 @@
+package service
+
+import (
+ "context"
+ "fmt"
+ "testing"
+
+ "github.com/Tencent/WeKnora/internal/types"
+ "github.com/alicebob/miniredis/v2"
+ "github.com/hibiken/asynq"
+ "github.com/redis/go-redis/v9"
+ "github.com/stretchr/testify/assert"
+ "github.com/stretchr/testify/require"
+)
+
+// Each worker repeatedly interrupts a full batch after claiming it, then
+// recovers the same rows. Redis is shared; each KB has a separate in-memory
+// SQLite database because this tests handler recovery, not PostgreSQL locking.
+func TestWikiInterruptedLookupRecoveryStress(t *testing.T) {
+ const workers, rounds, batchSize = 16, 10, 5
+ mr := miniredis.RunT(t)
+ rdb := redis.NewClient(&redis.Options{Addr: mr.Addr()})
+ t.Cleanup(func() { _ = rdb.Close() })
+ for worker := range workers {
+ t.Run(fmt.Sprintf("kb-%d", worker), func(t *testing.T) {
+ t.Parallel()
+ kbID := fmt.Sprintf("kb-%d", worker)
+ svc, db := newWikiLookupRecoveryService(t, kbID, rdb)
+ lookup := svc.knowledgeSvc.(*wikiKnowledgeLookupStub)
+ for round := range rounds {
+ var task *asynq.Task
+ for doc := range batchSize {
+ id := fmt.Sprintf("k-%d-%d-%d", worker, round, doc)
+ task = seedWikiLookupTask(t, svc, db, kbID, id)
+ }
+ ctx, cancel := context.WithCancel(context.Background())
+ lookup.lookup = func(ctx context.Context, id string) (*types.Knowledge, error) {
+ cancel()
+ return svc.knowledgeRepo.GetKnowledgeByIDOnly(ctx, id)
+ }
+ err := svc.ProcessWikiIngest(ctx, task)
+ cancel()
+ require.NoError(t, err)
+ var ops []types.TaskPendingOp
+ require.NoError(t, db.Find(&ops).Error)
+ require.Len(t, ops, batchSize, "every interrupted op must remain retryable")
+ for _, op := range ops {
+ require.Nil(t, op.ClaimedAt)
+ require.Equal(t, 1, op.FailCount)
+ }
+ require.Zero(t, rdb.ZCard(context.Background(), wikiInflightPrefix+kbID).Val())
+
+ lookup.lookup = svc.knowledgeRepo.GetKnowledgeByIDOnly
+ require.NoError(t, svc.ProcessWikiIngest(context.Background(), task))
+ count, err := svc.pendingRepo.PendingCount(context.Background(), wikiTaskType, wikiTaskScope, kbID)
+ require.NoError(t, err)
+ require.Zero(t, count, "healthy follow-up must drain the preserved work")
+ require.Zero(t, rdb.ZCard(context.Background(), wikiInflightPrefix+kbID).Val())
+ }
+ var completed int64
+ require.NoError(t, db.Model(&types.Knowledge{}).
+ Where("parse_status = ? AND pending_subtasks_count = 0", types.ParseStatusCompleted).
+ Count(&completed).Error)
+ assert.EqualValues(t, rounds*batchSize, completed)
+ assert.Len(t, svc.task.(*wikiGuardTaskQueue).tasks, rounds)
+ })
+ }
+}
diff --git a/internal/application/service/wiki_ingest_cancellation_test.go b/internal/application/service/wiki_ingest_cancellation_test.go
new file mode 100644
index 000000000..aa764eb76
--- /dev/null
+++ b/internal/application/service/wiki_ingest_cancellation_test.go
@@ -0,0 +1,320 @@
+package service
+
+import (
+ "context"
+ "encoding/json"
+ "errors"
+ "fmt"
+ "testing"
+ "time"
+
+ apprepo "github.com/Tencent/WeKnora/internal/application/repository"
+ "github.com/Tencent/WeKnora/internal/models/chat"
+ "github.com/Tencent/WeKnora/internal/types"
+ "github.com/Tencent/WeKnora/internal/types/interfaces"
+ "github.com/alicebob/miniredis/v2"
+ "github.com/hibiken/asynq"
+ "github.com/redis/go-redis/v9"
+ "github.com/stretchr/testify/assert"
+ "github.com/stretchr/testify/require"
+ "gorm.io/driver/sqlite"
+ "gorm.io/gorm"
+)
+
+type wikiKnowledgeLookupStub struct {
+ interfaces.KnowledgeService
+ lookup func(context.Context, string) (*types.Knowledge, error)
+}
+
+func (s *wikiKnowledgeLookupStub) GetKnowledgeByIDOnly(ctx context.Context, id string) (*types.Knowledge, error) {
+ return s.lookup(ctx, id)
+}
+
+// A canceled or failed lookup does not establish that the source was deleted.
+// Map must return an error so the batch keeps the durable op for retry.
+func TestWikiMapPreservesLookupFailures(t *testing.T) {
+ for _, lookupErr := range []error{context.Canceled, context.DeadlineExceeded, errors.New("database unavailable")} {
+ t.Run(lookupErr.Error(), func(t *testing.T) {
+ svc := &wikiIngestService{knowledgeSvc: &wikiKnowledgeLookupStub{
+ lookup: func(context.Context, string) (*types.Knowledge, error) { return nil, lookupErr },
+ }}
+ result, updates, err := svc.mapOneDocument(context.Background(), nil,
+ WikiIngestPayload{TenantID: 7, KnowledgeBaseID: "kb-1"},
+ WikiPendingOp{KnowledgeID: "k-1", Op: WikiOpIngest}, nil)
+ require.ErrorIs(t, err, lookupErr)
+ assert.Nil(t, result)
+ assert.Empty(t, updates)
+ })
+ }
+}
+
+// Reduce must not report a successful no-op when it cannot read a source:
+// the batch otherwise deletes that document's durable op without its pages.
+func TestWikiReducePreservesLookupFailures(t *testing.T) {
+ for _, lookupErr := range []error{context.Canceled, context.DeadlineExceeded, errors.New("database unavailable")} {
+ t.Run(lookupErr.Error(), func(t *testing.T) {
+ svc := &wikiIngestService{knowledgeSvc: &wikiKnowledgeLookupStub{
+ lookup: func(context.Context, string) (*types.Knowledge, error) { return nil, lookupErr },
+ }}
+ changed, _, _, err := svc.reduceSlugUpdates(context.Background(), nil, "kb-1", "entity/example",
+ []SlugUpdate{{Slug: "entity/example", Type: "entity", KnowledgeID: "k-1"}}, 7, nil, nil)
+ require.ErrorIs(t, err, lookupErr)
+ assert.False(t, changed)
+ })
+ }
+}
+
+func TestWikiLiveUpdatesKeepDeletionAndRetractionSemantics(t *testing.T) {
+ lookupCalls := make(map[string]int)
+ svc := &wikiIngestService{knowledgeSvc: &wikiKnowledgeLookupStub{
+ lookup: func(_ context.Context, id string) (*types.Knowledge, error) {
+ lookupCalls[id]++
+ switch id {
+ case "missing":
+ return nil, fmt.Errorf("lookup: %w", apprepo.ErrKnowledgeNotFound)
+ case "nil":
+ return nil, nil
+ case "deleting", "cancelled":
+ return &types.Knowledge{ParseStatus: id}, nil
+ default:
+ return &types.Knowledge{ID: id, ParseStatus: types.ParseStatusFinalizing}, nil
+ }
+ },
+ }}
+ updates := []SlugUpdate{
+ {Type: "entity", KnowledgeID: "live"},
+ {Type: "summary", KnowledgeID: "live"},
+ {Type: "entity", KnowledgeID: "missing"},
+ {Type: "summary", KnowledgeID: "nil"},
+ {Type: "entity", KnowledgeID: "deleting"},
+ {Type: "entity", KnowledgeID: "cancelled"},
+ {Type: "retract", KnowledgeID: "missing"},
+ {Type: "retractStale", KnowledgeID: "missing"},
+ {Type: "entity"},
+ }
+ got, err := svc.filterLiveUpdates(context.Background(), "kb-1", updates)
+ require.NoError(t, err)
+ assert.Equal(t, []SlugUpdate{updates[0], updates[1], updates[6], updates[7], updates[8]}, got)
+ assert.Equal(t, 1, lookupCalls["live"], "shared source lookup is cached within the slug")
+ assert.Equal(t, 1, lookupCalls["missing"])
+}
+
+func TestWikiLiveUpdatesRejectPartialLookupResults(t *testing.T) {
+ lookupErr := errors.New("database unavailable")
+ svc := &wikiIngestService{knowledgeSvc: &wikiKnowledgeLookupStub{
+ lookup: func(_ context.Context, id string) (*types.Knowledge, error) {
+ if id == "unreadable" {
+ return nil, lookupErr
+ }
+ return &types.Knowledge{ID: id}, nil
+ },
+ }}
+ got, err := svc.filterLiveUpdates(context.Background(), "kb-1", []SlugUpdate{
+ {Type: "retract", KnowledgeID: "deleted"},
+ {Type: "entity", KnowledgeID: "live"},
+ {Type: "entity", KnowledgeID: "unreadable"},
+ })
+ require.ErrorIs(t, err, lookupErr)
+ assert.Nil(t, got, "a partial slug must not be applied as a successful complete update")
+}
+
+func TestWikiKnowledgeTombstoneSkipsLookup(t *testing.T) {
+ mr := miniredis.RunT(t)
+ rdb := redis.NewClient(&redis.Options{Addr: mr.Addr()})
+ t.Cleanup(func() { _ = rdb.Close() })
+ ctx := context.Background()
+ require.NoError(t, rdb.Set(ctx, WikiDeletedTombstoneKey("kb-1", "k-1"), "1", time.Minute).Err())
+ svc := &wikiIngestService{redisClient: rdb, knowledgeSvc: &wikiKnowledgeLookupStub{
+ lookup: func(context.Context, string) (*types.Knowledge, error) {
+ t.Error("tombstoned knowledge should not require a database lookup")
+ return nil, errors.New("database unavailable")
+ },
+ }}
+ result, updates, err := svc.mapOneDocument(ctx, nil, WikiIngestPayload{KnowledgeBaseID: "kb-1"},
+ WikiPendingOp{KnowledgeID: "k-1"}, nil)
+ require.NoError(t, err)
+ assert.Nil(t, result)
+ assert.Empty(t, updates)
+}
+
+type wikiLookupKBService struct {
+ interfaces.KnowledgeBaseService
+ kb *types.KnowledgeBase
+}
+
+func (s *wikiLookupKBService) GetKnowledgeBaseByIDOnly(context.Context, string) (*types.KnowledgeBase, error) {
+ return s.kb, nil
+}
+
+func (s *wikiLookupKBService) GetKnowledgeBaseByID(context.Context, string) (*types.KnowledgeBase, error) {
+ return s.kb, nil
+}
+
+type wikiLookupModelService struct{ interfaces.ModelService }
+
+func (wikiLookupModelService) GetChatModel(context.Context, string) (chat.Chat, error) {
+ // The recovery fixture has no chunks, so the real pipeline skips the LLM.
+ return nil, nil
+}
+
+type wikiLookupEmptyChunks struct{ interfaces.ChunkRepository }
+
+func (wikiLookupEmptyChunks) ListChunksByKnowledgeID(context.Context, uint64, string) ([]*types.Chunk, error) {
+ return nil, nil
+}
+
+func newWikiLookupRecoveryService(t *testing.T, kbID string, rdb *redis.Client) (*wikiIngestService, *gorm.DB) {
+ t.Helper()
+ db, err := gorm.Open(sqlite.Open(":memory:"), &gorm.Config{})
+ require.NoError(t, err)
+ sqlDB, err := db.DB()
+ require.NoError(t, err)
+ sqlDB.SetMaxOpenConns(1)
+ t.Cleanup(func() { _ = sqlDB.Close() })
+ require.NoError(t, db.Exec(knowledgeTestDDL).Error)
+ require.NoError(t, db.Exec("ALTER TABLE knowledges ADD COLUMN processed_at DATETIME").Error)
+ require.NoError(t, db.AutoMigrate(&types.TaskPendingOp{}))
+ require.NoError(t, db.Exec("CREATE TABLE tenants (id INTEGER PRIMARY KEY, deleted_at DATETIME)").Error)
+ require.NoError(t, db.Exec("INSERT INTO tenants (id) VALUES (7)").Error)
+ repo := apprepo.NewKnowledgeRepository(db)
+ svc := &wikiIngestService{
+ kbService: &wikiLookupKBService{kb: &types.KnowledgeBase{
+ ID: kbID, SummaryModelID: "unused", IndexingStrategy: types.IndexingStrategy{WikiEnabled: true},
+ }},
+ modelService: wikiLookupModelService{}, chunkRepo: wikiLookupEmptyChunks{},
+ knowledgeRepo: repo, knowledgeSvc: &wikiKnowledgeLookupStub{lookup: repo.GetKnowledgeByIDOnly},
+ pendingRepo: apprepo.NewTaskPendingOpsRepository(db), task: &wikiGuardTaskQueue{}, redisClient: rdb,
+ }
+ return svc, db
+}
+
+func seedWikiLookupTask(t *testing.T, svc *wikiIngestService, db *gorm.DB, kbID, knowledgeID string) *asynq.Task {
+ t.Helper()
+ require.NoError(t, db.Exec(`INSERT INTO knowledges
+ (id, tenant_id, knowledge_base_id, parse_status, pending_subtasks_count) VALUES (?, 7, ?, ?, 1)`,
+ knowledgeID, kbID, types.ParseStatusFinalizing).Error)
+ op, err := newWikiIngestPendingOp(context.Background(), 7, kbID, knowledgeID)
+ require.NoError(t, err)
+ require.NoError(t, svc.pendingRepo.Enqueue(context.Background(), op))
+ payload, err := json.Marshal(WikiIngestPayload{TenantID: 7, KnowledgeBaseID: kbID})
+ require.NoError(t, err)
+ return asynq.NewTask(types.TypeWikiIngest, payload)
+}
+
+// Exercise the real batch handler, durable repository and finalizing counter.
+// Cancellation happens after the op is claimed, at the source-document read.
+func TestWikiIngestRetainsInterruptedLookupAndRecovers(t *testing.T) {
+ for _, mode := range []string{"lite", "redis"} {
+ t.Run(mode, func(t *testing.T) {
+ var rdb *redis.Client
+ if mode == "redis" {
+ mr := miniredis.RunT(t)
+ rdb = redis.NewClient(&redis.Options{Addr: mr.Addr()})
+ t.Cleanup(func() { _ = rdb.Close() })
+ }
+ svc, db := newWikiLookupRecoveryService(t, "kb-1", rdb)
+ task := seedWikiLookupTask(t, svc, db, "kb-1", "k-1")
+ ctx, cancel := context.WithCancel(context.Background())
+ defer cancel()
+ lookup := svc.knowledgeSvc.(*wikiKnowledgeLookupStub)
+ lookup.lookup = func(ctx context.Context, id string) (*types.Knowledge, error) {
+ cancel()
+ return svc.knowledgeRepo.GetKnowledgeByIDOnly(ctx, id)
+ }
+ require.NoError(t, svc.ProcessWikiIngest(ctx, task))
+ var pending types.TaskPendingOp
+ require.NoError(t, db.First(&pending).Error, "interrupted source must stay queued")
+ assert.Equal(t, 1, pending.FailCount)
+ assert.Nil(t, pending.ClaimedAt, "next batch must be able to reclaim immediately")
+ kn, err := svc.knowledgeRepo.GetKnowledgeByIDOnly(context.Background(), "k-1")
+ require.NoError(t, err)
+ assert.Equal(t, types.ParseStatusFinalizing, kn.ParseStatus)
+ assert.Equal(t, 1, kn.PendingSubtasksCount)
+ require.Len(t, svc.task.(*wikiGuardTaskQueue).tasks, 1, "retry must be scheduled")
+ if rdb != nil {
+ require.Zero(t, rdb.ZCard(context.Background(), wikiInflightPrefix+"kb-1").Val())
+ }
+
+ lookup.lookup = svc.knowledgeRepo.GetKnowledgeByIDOnly
+ require.NoError(t, svc.ProcessWikiIngest(context.Background(), task))
+ count, err := svc.pendingRepo.PendingCount(context.Background(), wikiTaskType, wikiTaskScope, "kb-1")
+ require.NoError(t, err)
+ assert.Zero(t, count)
+ kn, err = svc.knowledgeRepo.GetKnowledgeByIDOnly(context.Background(), "k-1")
+ require.NoError(t, err)
+ assert.Equal(t, types.ParseStatusCompleted, kn.ParseStatus)
+ assert.Zero(t, kn.PendingSubtasksCount)
+ })
+ }
+}
+
+func TestWikiMapClosesSpanAfterLookupCancellation(t *testing.T) {
+ tracker, db := setupSpanTrackerTest(t)
+ sqlDB, err := db.DB()
+ require.NoError(t, err)
+ t.Cleanup(func() { _ = sqlDB.Close() })
+ ctx, cancel := context.WithCancel(context.Background())
+ defer cancel()
+ _, attempt, err := tracker.OpenAttempt(ctx, "k-1", "")
+ require.NoError(t, err)
+ require.NotNil(t, tracker.BeginStage(ctx, "k-1", attempt, types.StagePostProcess, nil))
+ svc := &wikiIngestService{spanTracker: tracker, knowledgeSvc: &wikiKnowledgeLookupStub{
+ lookup: func(ctx context.Context, _ string) (*types.Knowledge, error) {
+ cancel()
+ return nil, ctx.Err()
+ },
+ }}
+ _, _, err = svc.mapOneDocument(ctx, nil, WikiIngestPayload{KnowledgeBaseID: "kb-1"},
+ WikiPendingOp{KnowledgeID: "k-1"}, nil)
+ require.ErrorIs(t, err, context.Canceled)
+ var span types.KnowledgeProcessingSpan
+ require.NoError(t, db.Where("name = ?", "postprocess.wiki").First(&span).Error)
+ assert.Equal(t, types.SpanStatusFailed, span.Status, "interrupted map must not remain running in the trace")
+ assert.NotNil(t, span.FinishedAt)
+}
+
+func TestWikiLookupFailureDoesNotBlockHealthySibling(t *testing.T) {
+ svc, db := newWikiLookupRecoveryService(t, "kb-1", nil)
+ seedWikiLookupTask(t, svc, db, "kb-1", "unreadable")
+ task := seedWikiLookupTask(t, svc, db, "kb-1", "healthy")
+ lookup := svc.knowledgeSvc.(*wikiKnowledgeLookupStub)
+ lookup.lookup = func(ctx context.Context, id string) (*types.Knowledge, error) {
+ if id == "unreadable" {
+ return nil, errors.New("temporary read failure")
+ }
+ return svc.knowledgeRepo.GetKnowledgeByIDOnly(ctx, id)
+ }
+ require.NoError(t, svc.ProcessWikiIngest(context.Background(), task))
+ var ops []types.TaskPendingOp
+ require.NoError(t, db.Find(&ops).Error)
+ require.Len(t, ops, 1)
+ assert.Equal(t, "unreadable", ops[0].DedupKey)
+ healthy, err := svc.knowledgeRepo.GetKnowledgeByIDOnly(context.Background(), "healthy")
+ require.NoError(t, err)
+ assert.Equal(t, types.ParseStatusCompleted, healthy.ParseStatus)
+}
+
+func TestWikiPersistentLookupFailureExhaustsRetryBudget(t *testing.T) {
+ svc, db := newWikiLookupRecoveryService(t, "kb-1", nil)
+ require.NoError(t, db.AutoMigrate(&types.TaskDeadLetter{}))
+ svc.deadLetterRepo = apprepo.NewTaskDeadLetterRepository(db)
+ task := seedWikiLookupTask(t, svc, db, "kb-1", "unreadable")
+ svc.knowledgeSvc.(*wikiKnowledgeLookupStub).lookup = func(context.Context, string) (*types.Knowledge, error) {
+ return nil, errors.New("persistent read failure")
+ }
+ for range wikiMaxFailRetries + 1 {
+ require.NoError(t, svc.ProcessWikiIngest(context.Background(), task))
+ }
+ count, err := svc.pendingRepo.PendingCount(context.Background(), wikiTaskType, wikiTaskScope, "kb-1")
+ require.NoError(t, err)
+ assert.Zero(t, count, "persistent failure must not retry forever")
+ var letters []types.TaskDeadLetter
+ require.NoError(t, db.Find(&letters).Error)
+ require.Len(t, letters, 1)
+ assert.Equal(t, "unreadable", letters[0].RelatedID)
+ assert.Equal(t, wikiMaxFailRetries+1, letters[0].FailCount)
+ kn, err := svc.knowledgeRepo.GetKnowledgeByIDOnly(context.Background(), "unreadable")
+ require.NoError(t, err)
+ assert.Zero(t, kn.PendingSubtasksCount, "exhausted work must release the finalizing slot")
+ assert.NotEqual(t, types.ParseStatusFinalizing, kn.ParseStatus)
+}
diff --git a/internal/application/service/wiki_ingest_http_cancellation_test.go b/internal/application/service/wiki_ingest_http_cancellation_test.go
new file mode 100644
index 000000000..0dbe4ae83
--- /dev/null
+++ b/internal/application/service/wiki_ingest_http_cancellation_test.go
@@ -0,0 +1,95 @@
+package service
+
+import (
+ "context"
+ "io"
+ "net/http"
+ "net/http/httptest"
+ "sync/atomic"
+ "testing"
+ "time"
+
+ "github.com/Tencent/WeKnora/internal/models/api"
+ "github.com/Tencent/WeKnora/internal/models/api/openaicompletions"
+ "github.com/Tencent/WeKnora/internal/types"
+ "github.com/stretchr/testify/require"
+)
+
+// Exercise the production HTTP client through Wiki's request coalescing layer.
+// A model that never sends headers or stops in the middle of a response must
+// not prevent cancellation or a subsequent independent request from completing.
+func TestWikiHTTPModelCancellationAndRecovery(t *testing.T) {
+ withSSRFWhitelist(t, "127.0.0.1")
+ for _, partialBody := range []bool{false, true} {
+ name := "before_headers"
+ if partialBody {
+ name = "during_body"
+ }
+ t.Run(name, func(t *testing.T) {
+ started, stopped := make(chan struct{}), make(chan struct{})
+ shutdown := make(chan struct{})
+ var calls atomic.Int32
+ server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ _, _ = io.Copy(io.Discard, r.Body)
+ if calls.Add(1) == 1 {
+ if partialBody {
+ w.Header().Set("Content-Type", "application/json")
+ _, _ = io.WriteString(w, `{"choices":[`)
+ w.(http.Flusher).Flush()
+ }
+ close(started)
+ select {
+ case <-r.Context().Done():
+ case <-shutdown:
+ }
+ close(stopped)
+ return
+ }
+ w.Header().Set("Content-Type", "application/json")
+ _, _ = io.WriteString(w, `{"choices":[{"message":{"content":"recovered"},"finish_reason":"stop"}]}`)
+ }))
+ t.Cleanup(func() {
+ close(shutdown)
+ server.Close()
+ })
+ model := openaicompletions.New(openaicompletions.Config{
+ Endpoint: api.Endpoint{
+ BaseURL: server.URL, Model: "fixture", ModelID: "fixture", Client: server.Client(),
+ },
+ Settings: api.DefaultOpenAICompletions(),
+ })
+ svc := &wikiIngestService{}
+ parent := context.WithValue(context.Background(), types.TenantIDContextKey, uint64(7))
+ ctx, cancel := context.WithCancel(parent)
+ defer cancel()
+ result := make(chan error, 1)
+ go func() {
+ _, err := svc.generateWithTemplate(ctx, model, "stalled request", nil)
+ result <- err
+ }()
+ select {
+ case <-started:
+ case <-time.After(3 * time.Second):
+ t.Fatal("model request did not reach the HTTP server")
+ }
+ cancel()
+ select {
+ case err := <-result:
+ require.ErrorIs(t, err, context.Canceled)
+ case <-time.After(3 * time.Second):
+ t.Fatal("Wiki model call did not exit on cancellation")
+ }
+ select {
+ case <-stopped:
+ case <-time.After(3 * time.Second):
+ t.Fatal("the underlying HTTP request remained active after cancellation")
+ }
+ recoveryCtx, recoveryCancel := context.WithTimeout(parent, 3*time.Second)
+ defer recoveryCancel()
+ content, err := svc.generateWithTemplate(recoveryCtx, model, "independent recovery request", nil)
+ require.NoError(t, err)
+ require.Equal(t, "recovered", content)
+ require.EqualValues(t, 2, calls.Load())
+ })
+ }
+}