Open-source LLM knowledge platform: turn raw documents into a queryable RAG, an autonomous reasoning agent, and a self-maintaining Wiki.
Go agentagenticaichatbotdsh-pluginembeddingsevaluationgenerative-aigolangknowledge-basellmmulti-tenantollamaopenaiquestion-answeringragrerankingsemantic-searchvector-searchwiki
fix(lite): reclaim superseded reparse chunks (#4595)
Slai.life640fcb618af7
diff --git a/internal/application/repository/chunk.go b/internal/application/repository/chunk.goindex b6ad7b3da..66f16ea5e 100644--- a/internal/application/repository/chunk.go+++ b/internal/application/repository/chunk.go@@ -645,13 +645,70 @@ func (r *chunkRepository) DeleteChunks(ctx context.Context, tenantID uint64, idsreturn nil}-// DeleteChunksByKnowledgeID deletes all chunks for a knowledge ID+// DeleteChunksByKnowledgeID soft-deletes all active chunks for a knowledge ID.+// The rows remain available for recovery until a later successful replacement+// proves that a newer chunk with the same document position is now active.func (r *chunkRepository) DeleteChunksByKnowledgeID(ctx context.Context, tenantID uint64, knowledgeID string) error {return r.db.WithContext(ctx).Where("tenant_id = ? AND knowledge_id = ?", tenantID, knowledgeID,).Delete(&types.Chunk{}).Error}+// PurgeSupersededChunksByKnowledgeID permanently removes only soft-deleted+// chunks that have an active replacement from a later parse at the same+// document position and chunk type. This intentionally does not purge an+// arbitrary soft-deleted row: a user-deleted chunk, or a row from a failed or+// cancelled replacement, remains recoverable until a successful replacement+// supersedes it.+//+// The revision snapshots belong to the obsolete chunk ID and are removed in+// the same transaction. SQLite's chunk_images projection is maintained by its+// existing DELETE trigger; other stores have already removed their indexes+// before this method is called.+func (r *chunkRepository) PurgeSupersededChunksByKnowledgeID(+ ctx context.Context, tenantID uint64, knowledgeID string,+) (int64, error) {+ var chunkIDs []string+ var purged int64++ err := r.db.WithContext(ctx).Transaction(func(tx *gorm.DB) error {+ // A replacement must have been created after the previous row was+ // soft-deleted. Comparing against deleted_at avoids treating a manual+ // delete as superseded merely because an older live chunk has the same+ // index. COALESCE keeps legacy NULL chunk_type rows comparable.+ if err := tx.Unscoped().Model(&types.Chunk{}).Where(+ `chunks.tenant_id = ? AND chunks.knowledge_id = ?+ AND chunks.deleted_at IS NOT NULL+ AND EXISTS (+ SELECT 1 FROM chunks AS replacement+ WHERE replacement.tenant_id = chunks.tenant_id+ AND replacement.knowledge_id = chunks.knowledge_id+ AND replacement.deleted_at IS NULL+ AND replacement.chunk_index = chunks.chunk_index+ AND COALESCE(replacement.chunk_type, '') = COALESCE(chunks.chunk_type, '')+ AND replacement.created_at >= chunks.deleted_at+ )`,+ tenantID, knowledgeID,+ ).Pluck("chunks.id", &chunkIDs).Error; err != nil {+ return err+ }+ if len(chunkIDs) == 0 {+ return nil+ }++ if err := tx.Where("chunk_id IN ?", chunkIDs).Delete(&types.ChunkRevision{}).Error; err != nil {+ return err+ }+ result := tx.Unscoped().Where("id IN ?", chunkIDs).Delete(&types.Chunk{})+ if result.Error != nil {+ return result.Error+ }+ purged = result.RowsAffected+ return nil+ })+ return purged, err+}+// ListImageInfoByKnowledgeIDs returns non-empty image_info values for the given knowledge IDs.// No chunk_type filter — collects from text, image_ocr, and image_caption chunks.func (r *chunkRepository) ListImageInfoByKnowledgeIDs(diff --git a/internal/application/repository/chunk_sqlite_test.go b/internal/application/repository/chunk_sqlite_test.goindex d3e8c2328..956a8f926 100644--- a/internal/application/repository/chunk_sqlite_test.go+++ b/internal/application/repository/chunk_sqlite_test.go@@ -4,6 +4,7 @@ import ("context""fmt""testing"+ "time""unicode/utf8""github.com/Tencent/WeKnora/internal/types"@@ -19,7 +20,7 @@ func setupChunkTestDB(t *testing.T) *gorm.DB {t.Helper()db, err := gorm.Open(sqlite.Open(":memory:"), &gorm.Config{})require.NoError(t, err)- require.NoError(t, db.AutoMigrate(&types.Chunk{}, &types.KnowledgeTag{}))+ require.NoError(t, db.AutoMigrate(&types.Chunk{}, &types.ChunkRevision{}, &types.KnowledgeTag{}))return db}@@ -178,6 +179,89 @@ func TestKnowledgeTag_SQLite_SeqIDAutoAssigned(t *testing.T) {assert.NotEqual(t, tag1.SeqID, tag2.SeqID)}+func TestPurgeSupersededChunksByKnowledgeIDRemovesOnlyReplacedHistory(t *testing.T) {+ db := setupChunkTestDB(t)+ repo := NewChunkRepository(db).(*chunkRepository)+ ctx := context.Background()+ const tenantID uint64 = 1+ const knowledgeID = "knowledge-1"++ deletedAt := time.Now().UTC().Add(-time.Minute)+ old := &types.Chunk{+ ID: "old", TenantID: tenantID, KnowledgeID: knowledgeID, KnowledgeBaseID: "kb-1",+ ChunkIndex: 0, ChunkType: types.ChunkTypeText, Content: "old", IsEnabled: true,+ CreatedAt: deletedAt.Add(-time.Minute), DeletedAt: gorm.DeletedAt{Time: deletedAt, Valid: true},+ }+ replacement := &types.Chunk{+ ID: "replacement", TenantID: tenantID, KnowledgeID: knowledgeID, KnowledgeBaseID: "kb-1",+ ChunkIndex: 0, ChunkType: types.ChunkTypeText, Content: "new", IsEnabled: true,+ CreatedAt: deletedAt.Add(time.Minute),+ }+ // This is deliberately deleted but has no active replacement. It models a+ // cancelled/failed parse or a user deletion and must stay recoverable.+ keep := &types.Chunk{+ ID: "keep", TenantID: tenantID, KnowledgeID: knowledgeID, KnowledgeBaseID: "kb-1",+ ChunkIndex: 1, ChunkType: types.ChunkTypeText, Content: "keep", IsEnabled: true,+ CreatedAt: deletedAt.Add(-time.Minute), DeletedAt: gorm.DeletedAt{Time: deletedAt, Valid: true},+ }+ require.NoError(t, db.Unscoped().Create([]*types.Chunk{old, replacement, keep}).Error)+ require.NoError(t, db.Create(&types.ChunkRevision{+ ID: "old-revision", TenantID: tenantID, KnowledgeID: knowledgeID, KnowledgeBaseID: "kb-1",+ ChunkID: old.ID, Revision: 1, Content: "old", EditedAt: deletedAt,+ }).Error)+ require.NoError(t, db.Create(&types.ChunkRevision{+ ID: "keep-revision", TenantID: tenantID, KnowledgeID: knowledgeID, KnowledgeBaseID: "kb-1",+ ChunkID: keep.ID, Revision: 1, Content: "keep", EditedAt: deletedAt,+ }).Error)++ purged, err := repo.PurgeSupersededChunksByKnowledgeID(ctx, tenantID, knowledgeID)+ require.NoError(t, err)+ require.Equal(t, int64(1), purged)++ var oldCount, keepCount, replacementCount, oldRevisions, keepRevisions int64+ require.NoError(t, db.Unscoped().Model(&types.Chunk{}).Where("id = ?", old.ID).Count(&oldCount).Error)+ require.NoError(t, db.Unscoped().Model(&types.Chunk{}).Where("id = ?", keep.ID).Count(&keepCount).Error)+ require.NoError(t, db.Model(&types.Chunk{}).Where("id = ?", replacement.ID).Count(&replacementCount).Error)+ require.NoError(t, db.Model(&types.ChunkRevision{}).Where("chunk_id = ?", old.ID).Count(&oldRevisions).Error)+ require.NoError(t, db.Model(&types.ChunkRevision{}).Where("chunk_id = ?", keep.ID).Count(&keepRevisions).Error)+ assert.Zero(t, oldCount, "the superseded chunk must be physically reclaimed")+ assert.Equal(t, int64(1), keepCount, "unreplaced history must remain recoverable")+ assert.Equal(t, int64(1), replacementCount)+ assert.Zero(t, oldRevisions, "revisions of a purged chunk must not become orphaned")+ assert.Equal(t, int64(1), keepRevisions)+}++func TestPurgeSupersededChunksByKnowledgeIDDoesNotPurgeEarlierActiveChunk(t *testing.T) {+ db := setupChunkTestDB(t)+ repo := NewChunkRepository(db).(*chunkRepository)+ ctx := context.Background()+ const tenantID uint64 = 1+ const knowledgeID = "knowledge-1"++ // The live row predates the deletion, so it cannot be a replacement for it.+ liveCreated := time.Now().UTC().Add(-2 * time.Minute)+ deletedAt := time.Now().UTC().Add(-time.Minute)+ old := &types.Chunk{+ ID: "old", TenantID: tenantID, KnowledgeID: knowledgeID, KnowledgeBaseID: "kb-1",+ ChunkIndex: 0, ChunkType: types.ChunkTypeText, Content: "old", IsEnabled: true,+ CreatedAt: deletedAt.Add(-time.Minute), DeletedAt: gorm.DeletedAt{Time: deletedAt, Valid: true},+ }+ unrelatedLive := &types.Chunk{+ ID: "live", TenantID: tenantID, KnowledgeID: knowledgeID, KnowledgeBaseID: "kb-1",+ ChunkIndex: 0, ChunkType: types.ChunkTypeText, Content: "live", IsEnabled: true,+ CreatedAt: liveCreated,+ }+ require.NoError(t, db.Unscoped().Create([]*types.Chunk{old, unrelatedLive}).Error)++ purged, err := repo.PurgeSupersededChunksByKnowledgeID(ctx, tenantID, knowledgeID)+ require.NoError(t, err)+ assert.Zero(t, purged)++ var count int64+ require.NoError(t, db.Unscoped().Model(&types.Chunk{}).Where("id = ?", old.ID).Count(&count).Error)+ assert.Equal(t, int64(1), count)+}+func TestCreateChunks_SQLite_SeqIDAfterSoftDelete(t *testing.T) {db := setupChunkTestDB(t)repo := NewChunkRepository(db)diff --git a/internal/application/service/knowledge_process.go b/internal/application/service/knowledge_process.goindex 7216d858d..1e2b81714 100644--- a/internal/application/service/knowledge_process.go+++ b/internal/application/service/knowledge_process.go@@ -388,6 +388,13 @@ func (s *knowledgeService) deleteUnindexedChunks(ctx context.Context, knowledge}}+// supersededChunkPurger is intentionally narrower than ChunkRepository: only+// persistent repositories that can prove a newer active replacement should+// physically reclaim soft-deleted chunk history.+type supersededChunkPurger interface {+ PurgeSupersededChunksByKnowledgeID(context.Context, uint64, string) (int64, error)+}+// processChunks processes chunks and creates embeddings for knowledge contentfunc (s *knowledgeService) processChunks(ctx context.Context,kb *types.KnowledgeBase, knowledge *types.Knowledge, chunks []types.ParsedChunk,@@ -857,6 +864,20 @@ func (s *knowledgeService) processChunks(ctx context.Context,return nil}+ // Reparse has now written the replacement chunks and, when enabled, their+ // replacement indexes. Only at this successful point is it safe to reclaim+ // old soft-deleted rows. Keep this optional at the interface boundary so+ // existing non-SQL repositories and narrow test doubles retain their+ // lifecycle behavior; the concrete SQL chunk repository implements it.+ if purger, ok := s.chunkRepo.(supersededChunkPurger); ok {+ purged, purgeErr := purger.PurgeSupersededChunksByKnowledgeID(ctx, knowledge.TenantID, knowledge.ID)+ if purgeErr != nil {+ logger.Warnf(ctx, "Failed to purge superseded chunks for knowledge %s: %v", knowledge.ID, purgeErr)+ } else if purged > 0 {+ logger.Infof(ctx, "Purged %d superseded chunks for knowledge %s", purged, knowledge.ID)+ }+ }+now := time.Now()finalizeIndexedKnowledgeState(knowledge,diff --git a/internal/application/service/knowledge_process_index_retry_test.go b/internal/application/service/knowledge_process_index_retry_test.goindex e6703a723..d25a09fa6 100644--- a/internal/application/service/knowledge_process_index_retry_test.go+++ b/internal/application/service/knowledge_process_index_retry_test.go@@ -114,6 +114,7 @@ func TestProcessChunksRetriesBatchIndexFailureBeforeLastAttempt(t *testing.T) {// Partial chunks and vectors are still removed; the retry rebuilds them.require.Equal(t, 2, chunks.deletes)require.Equal(t, 2, engine.indexDeletes)+ require.Zero(t, chunks.purges, "a failed index must retain soft-deleted history for recovery")}func TestProcessChunksFailsKnowledgeOnLastBatchIndexAttempt(t *testing.T) {@@ -136,4 +137,5 @@ func TestProcessChunksFailsKnowledgeOnLastBatchIndexAttempt(t *testing.T) {require.Equal(t, "embedding endpoint unavailable", last.ErrorMessage)require.Equal(t, 2, chunks.deletes)require.Equal(t, 2, engine.indexDeletes)+ require.Zero(t, chunks.purges, "a terminal index failure must retain soft-deleted history for recovery")}diff --git a/internal/application/service/knowledge_process_parent_child_test.go b/internal/application/service/knowledge_process_parent_child_test.goindex e4df0190e..d7b601b42 100644--- a/internal/application/service/knowledge_process_parent_child_test.go+++ b/internal/application/service/knowledge_process_parent_child_test.go@@ -31,12 +31,18 @@ func (r *parentChildKnowledgeRepo) UpdateKnowledge(type parentChildChunkService struct {interfaces.ChunkRepositorycreated []*types.Chunk+ purges int}func (s *parentChildChunkService) DeleteChunksByKnowledgeID(context.Context, uint64, string) error {return nil}+func (s *parentChildChunkService) PurgeSupersededChunksByKnowledgeID(context.Context, uint64, string) (int64, error) {+ s.purges+++ return 0, nil+}+func (s *parentChildChunkService) CreateChunks(_ context.Context, chunks []*types.Chunk) error {s.created = append([]*types.Chunk(nil), chunks...)return nil@@ -191,6 +197,8 @@ func TestProcessChunksIndexesEveryTextChild(t *testing.T) {}}require.Len(t, textChunkIDs, 2)+ require.Equal(t, 1, chunkService.purges,+ "successful processing must reclaim superseded history only after indexing")indexedSourceIDs := make([]string, 0, len(retrieveEngine.indexed))for _, info := range retrieveEngine.indexed {