Skip to content

Open-source LLM knowledge platform: turn raw documents into a queryable RAG, an autonomous reasoning agent, and a self-maintaining Wiki.

fix(lite): reclaim superseded reparse chunks (#4595)

Slai.life640fcb618af7
diff --git a/internal/application/repository/chunk.go b/internal/application/repository/chunk.go
index b6ad7b3da..66f16ea5e 100644
--- a/internal/application/repository/chunk.go
+++ b/internal/application/repository/chunk.go
@@ -645,13 +645,70 @@ func (r *chunkRepository) DeleteChunks(ctx context.Context, tenantID uint64, ids
return nil
}
-// DeleteChunksByKnowledgeID deletes all chunks for a knowledge ID
+// DeleteChunksByKnowledgeID soft-deletes all active chunks for a knowledge ID.
+// The rows remain available for recovery until a later successful replacement
+// proves that a newer chunk with the same document position is now active.
func (r *chunkRepository) DeleteChunksByKnowledgeID(ctx context.Context, tenantID uint64, knowledgeID string) error {
return r.db.WithContext(ctx).Where(
"tenant_id = ? AND knowledge_id = ?", tenantID, knowledgeID,
).Delete(&types.Chunk{}).Error
}
+// PurgeSupersededChunksByKnowledgeID permanently removes only soft-deleted
+// chunks that have an active replacement from a later parse at the same
+// document position and chunk type. This intentionally does not purge an
+// arbitrary soft-deleted row: a user-deleted chunk, or a row from a failed or
+// cancelled replacement, remains recoverable until a successful replacement
+// supersedes it.
+//
+// The revision snapshots belong to the obsolete chunk ID and are removed in
+// the same transaction. SQLite's chunk_images projection is maintained by its
+// existing DELETE trigger; other stores have already removed their indexes
+// before this method is called.
+func (r *chunkRepository) PurgeSupersededChunksByKnowledgeID(
+ ctx context.Context, tenantID uint64, knowledgeID string,
+) (int64, error) {
+ var chunkIDs []string
+ var purged int64
+
+ err := r.db.WithContext(ctx).Transaction(func(tx *gorm.DB) error {
+ // A replacement must have been created after the previous row was
+ // soft-deleted. Comparing against deleted_at avoids treating a manual
+ // delete as superseded merely because an older live chunk has the same
+ // index. COALESCE keeps legacy NULL chunk_type rows comparable.
+ if err := tx.Unscoped().Model(&types.Chunk{}).Where(
+ `chunks.tenant_id = ? AND chunks.knowledge_id = ?
+ AND chunks.deleted_at IS NOT NULL
+ AND EXISTS (
+ SELECT 1 FROM chunks AS replacement
+ WHERE replacement.tenant_id = chunks.tenant_id
+ AND replacement.knowledge_id = chunks.knowledge_id
+ AND replacement.deleted_at IS NULL
+ AND replacement.chunk_index = chunks.chunk_index
+ AND COALESCE(replacement.chunk_type, '') = COALESCE(chunks.chunk_type, '')
+ AND replacement.created_at >= chunks.deleted_at
+ )`,
+ tenantID, knowledgeID,
+ ).Pluck("chunks.id", &chunkIDs).Error; err != nil {
+ return err
+ }
+ if len(chunkIDs) == 0 {
+ return nil
+ }
+
+ if err := tx.Where("chunk_id IN ?", chunkIDs).Delete(&types.ChunkRevision{}).Error; err != nil {
+ return err
+ }
+ result := tx.Unscoped().Where("id IN ?", chunkIDs).Delete(&types.Chunk{})
+ if result.Error != nil {
+ return result.Error
+ }
+ purged = result.RowsAffected
+ return nil
+ })
+ return purged, err
+}
+
// ListImageInfoByKnowledgeIDs returns non-empty image_info values for the given knowledge IDs.
// No chunk_type filter — collects from text, image_ocr, and image_caption chunks.
func (r *chunkRepository) ListImageInfoByKnowledgeIDs(
diff --git a/internal/application/repository/chunk_sqlite_test.go b/internal/application/repository/chunk_sqlite_test.go
index d3e8c2328..956a8f926 100644
--- a/internal/application/repository/chunk_sqlite_test.go
+++ b/internal/application/repository/chunk_sqlite_test.go
@@ -4,6 +4,7 @@ import (
"context"
"fmt"
"testing"
+ "time"
"unicode/utf8"
"github.com/Tencent/WeKnora/internal/types"
@@ -19,7 +20,7 @@ func setupChunkTestDB(t *testing.T) *gorm.DB {
t.Helper()
db, err := gorm.Open(sqlite.Open(":memory:"), &gorm.Config{})
require.NoError(t, err)
- require.NoError(t, db.AutoMigrate(&types.Chunk{}, &types.KnowledgeTag{}))
+ require.NoError(t, db.AutoMigrate(&types.Chunk{}, &types.ChunkRevision{}, &types.KnowledgeTag{}))
return db
}
@@ -178,6 +179,89 @@ func TestKnowledgeTag_SQLite_SeqIDAutoAssigned(t *testing.T) {
assert.NotEqual(t, tag1.SeqID, tag2.SeqID)
}
+func TestPurgeSupersededChunksByKnowledgeIDRemovesOnlyReplacedHistory(t *testing.T) {
+ db := setupChunkTestDB(t)
+ repo := NewChunkRepository(db).(*chunkRepository)
+ ctx := context.Background()
+ const tenantID uint64 = 1
+ const knowledgeID = "knowledge-1"
+
+ deletedAt := time.Now().UTC().Add(-time.Minute)
+ old := &types.Chunk{
+ ID: "old", TenantID: tenantID, KnowledgeID: knowledgeID, KnowledgeBaseID: "kb-1",
+ ChunkIndex: 0, ChunkType: types.ChunkTypeText, Content: "old", IsEnabled: true,
+ CreatedAt: deletedAt.Add(-time.Minute), DeletedAt: gorm.DeletedAt{Time: deletedAt, Valid: true},
+ }
+ replacement := &types.Chunk{
+ ID: "replacement", TenantID: tenantID, KnowledgeID: knowledgeID, KnowledgeBaseID: "kb-1",
+ ChunkIndex: 0, ChunkType: types.ChunkTypeText, Content: "new", IsEnabled: true,
+ CreatedAt: deletedAt.Add(time.Minute),
+ }
+ // This is deliberately deleted but has no active replacement. It models a
+ // cancelled/failed parse or a user deletion and must stay recoverable.
+ keep := &types.Chunk{
+ ID: "keep", TenantID: tenantID, KnowledgeID: knowledgeID, KnowledgeBaseID: "kb-1",
+ ChunkIndex: 1, ChunkType: types.ChunkTypeText, Content: "keep", IsEnabled: true,
+ CreatedAt: deletedAt.Add(-time.Minute), DeletedAt: gorm.DeletedAt{Time: deletedAt, Valid: true},
+ }
+ require.NoError(t, db.Unscoped().Create([]*types.Chunk{old, replacement, keep}).Error)
+ require.NoError(t, db.Create(&types.ChunkRevision{
+ ID: "old-revision", TenantID: tenantID, KnowledgeID: knowledgeID, KnowledgeBaseID: "kb-1",
+ ChunkID: old.ID, Revision: 1, Content: "old", EditedAt: deletedAt,
+ }).Error)
+ require.NoError(t, db.Create(&types.ChunkRevision{
+ ID: "keep-revision", TenantID: tenantID, KnowledgeID: knowledgeID, KnowledgeBaseID: "kb-1",
+ ChunkID: keep.ID, Revision: 1, Content: "keep", EditedAt: deletedAt,
+ }).Error)
+
+ purged, err := repo.PurgeSupersededChunksByKnowledgeID(ctx, tenantID, knowledgeID)
+ require.NoError(t, err)
+ require.Equal(t, int64(1), purged)
+
+ var oldCount, keepCount, replacementCount, oldRevisions, keepRevisions int64
+ require.NoError(t, db.Unscoped().Model(&types.Chunk{}).Where("id = ?", old.ID).Count(&oldCount).Error)
+ require.NoError(t, db.Unscoped().Model(&types.Chunk{}).Where("id = ?", keep.ID).Count(&keepCount).Error)
+ require.NoError(t, db.Model(&types.Chunk{}).Where("id = ?", replacement.ID).Count(&replacementCount).Error)
+ require.NoError(t, db.Model(&types.ChunkRevision{}).Where("chunk_id = ?", old.ID).Count(&oldRevisions).Error)
+ require.NoError(t, db.Model(&types.ChunkRevision{}).Where("chunk_id = ?", keep.ID).Count(&keepRevisions).Error)
+ assert.Zero(t, oldCount, "the superseded chunk must be physically reclaimed")
+ assert.Equal(t, int64(1), keepCount, "unreplaced history must remain recoverable")
+ assert.Equal(t, int64(1), replacementCount)
+ assert.Zero(t, oldRevisions, "revisions of a purged chunk must not become orphaned")
+ assert.Equal(t, int64(1), keepRevisions)
+}
+
+func TestPurgeSupersededChunksByKnowledgeIDDoesNotPurgeEarlierActiveChunk(t *testing.T) {
+ db := setupChunkTestDB(t)
+ repo := NewChunkRepository(db).(*chunkRepository)
+ ctx := context.Background()
+ const tenantID uint64 = 1
+ const knowledgeID = "knowledge-1"
+
+ // The live row predates the deletion, so it cannot be a replacement for it.
+ liveCreated := time.Now().UTC().Add(-2 * time.Minute)
+ deletedAt := time.Now().UTC().Add(-time.Minute)
+ old := &types.Chunk{
+ ID: "old", TenantID: tenantID, KnowledgeID: knowledgeID, KnowledgeBaseID: "kb-1",
+ ChunkIndex: 0, ChunkType: types.ChunkTypeText, Content: "old", IsEnabled: true,
+ CreatedAt: deletedAt.Add(-time.Minute), DeletedAt: gorm.DeletedAt{Time: deletedAt, Valid: true},
+ }
+ unrelatedLive := &types.Chunk{
+ ID: "live", TenantID: tenantID, KnowledgeID: knowledgeID, KnowledgeBaseID: "kb-1",
+ ChunkIndex: 0, ChunkType: types.ChunkTypeText, Content: "live", IsEnabled: true,
+ CreatedAt: liveCreated,
+ }
+ require.NoError(t, db.Unscoped().Create([]*types.Chunk{old, unrelatedLive}).Error)
+
+ purged, err := repo.PurgeSupersededChunksByKnowledgeID(ctx, tenantID, knowledgeID)
+ require.NoError(t, err)
+ assert.Zero(t, purged)
+
+ var count int64
+ require.NoError(t, db.Unscoped().Model(&types.Chunk{}).Where("id = ?", old.ID).Count(&count).Error)
+ assert.Equal(t, int64(1), count)
+}
+
func TestCreateChunks_SQLite_SeqIDAfterSoftDelete(t *testing.T) {
db := setupChunkTestDB(t)
repo := NewChunkRepository(db)
diff --git a/internal/application/service/knowledge_process.go b/internal/application/service/knowledge_process.go
index 7216d858d..1e2b81714 100644
--- a/internal/application/service/knowledge_process.go
+++ b/internal/application/service/knowledge_process.go
@@ -388,6 +388,13 @@ func (s *knowledgeService) deleteUnindexedChunks(ctx context.Context, knowledge
}
}
+// supersededChunkPurger is intentionally narrower than ChunkRepository: only
+// persistent repositories that can prove a newer active replacement should
+// physically reclaim soft-deleted chunk history.
+type supersededChunkPurger interface {
+ PurgeSupersededChunksByKnowledgeID(context.Context, uint64, string) (int64, error)
+}
+
// processChunks processes chunks and creates embeddings for knowledge content
func (s *knowledgeService) processChunks(ctx context.Context,
kb *types.KnowledgeBase, knowledge *types.Knowledge, chunks []types.ParsedChunk,
@@ -857,6 +864,20 @@ func (s *knowledgeService) processChunks(ctx context.Context,
return nil
}
+ // Reparse has now written the replacement chunks and, when enabled, their
+ // replacement indexes. Only at this successful point is it safe to reclaim
+ // old soft-deleted rows. Keep this optional at the interface boundary so
+ // existing non-SQL repositories and narrow test doubles retain their
+ // lifecycle behavior; the concrete SQL chunk repository implements it.
+ if purger, ok := s.chunkRepo.(supersededChunkPurger); ok {
+ purged, purgeErr := purger.PurgeSupersededChunksByKnowledgeID(ctx, knowledge.TenantID, knowledge.ID)
+ if purgeErr != nil {
+ logger.Warnf(ctx, "Failed to purge superseded chunks for knowledge %s: %v", knowledge.ID, purgeErr)
+ } else if purged > 0 {
+ logger.Infof(ctx, "Purged %d superseded chunks for knowledge %s", purged, knowledge.ID)
+ }
+ }
+
now := time.Now()
finalizeIndexedKnowledgeState(
knowledge,
diff --git a/internal/application/service/knowledge_process_index_retry_test.go b/internal/application/service/knowledge_process_index_retry_test.go
index e6703a723..d25a09fa6 100644
--- a/internal/application/service/knowledge_process_index_retry_test.go
+++ b/internal/application/service/knowledge_process_index_retry_test.go
@@ -114,6 +114,7 @@ func TestProcessChunksRetriesBatchIndexFailureBeforeLastAttempt(t *testing.T) {
// Partial chunks and vectors are still removed; the retry rebuilds them.
require.Equal(t, 2, chunks.deletes)
require.Equal(t, 2, engine.indexDeletes)
+ require.Zero(t, chunks.purges, "a failed index must retain soft-deleted history for recovery")
}
func TestProcessChunksFailsKnowledgeOnLastBatchIndexAttempt(t *testing.T) {
@@ -136,4 +137,5 @@ func TestProcessChunksFailsKnowledgeOnLastBatchIndexAttempt(t *testing.T) {
require.Equal(t, "embedding endpoint unavailable", last.ErrorMessage)
require.Equal(t, 2, chunks.deletes)
require.Equal(t, 2, engine.indexDeletes)
+ require.Zero(t, chunks.purges, "a terminal index failure must retain soft-deleted history for recovery")
}
diff --git a/internal/application/service/knowledge_process_parent_child_test.go b/internal/application/service/knowledge_process_parent_child_test.go
index e4df0190e..d7b601b42 100644
--- a/internal/application/service/knowledge_process_parent_child_test.go
+++ b/internal/application/service/knowledge_process_parent_child_test.go
@@ -31,12 +31,18 @@ func (r *parentChildKnowledgeRepo) UpdateKnowledge(
type parentChildChunkService struct {
interfaces.ChunkRepository
created []*types.Chunk
+ purges int
}
func (s *parentChildChunkService) DeleteChunksByKnowledgeID(context.Context, uint64, string) error {
return nil
}
+func (s *parentChildChunkService) PurgeSupersededChunksByKnowledgeID(context.Context, uint64, string) (int64, error) {
+ s.purges++
+ return 0, nil
+}
+
func (s *parentChildChunkService) CreateChunks(_ context.Context, chunks []*types.Chunk) error {
s.created = append([]*types.Chunk(nil), chunks...)
return nil
@@ -191,6 +197,8 @@ func TestProcessChunksIndexesEveryTextChild(t *testing.T) {
}
}
require.Len(t, textChunkIDs, 2)
+ require.Equal(t, 1, chunkService.purges,
+ "successful processing must reclaim superseded history only after indexing")
indexedSourceIDs := make([]string, 0, len(retrieveEngine.indexed))
for _, info := range retrieveEngine.indexed {