Open-source LLM knowledge platform: turn raw documents into a queryable RAG, an autonomous reasoning agent, and a self-maintaining Wiki.
Go agentagenticaichatbotdsh-pluginembeddingsevaluationgenerative-aigolangknowledge-basellmmulti-tenantollamaopenaiquestion-answeringragrerankingsemantic-searchvector-searchwiki
fix(datasource): make knowledge replacement transactional (#4473)
younggglcy98fb3f91c223
diff --git a/internal/application/repository/knowledge.go b/internal/application/repository/knowledge.goindex 9cfd90fd1..e3bd55cb5 100644--- a/internal/application/repository/knowledge.go+++ b/internal/application/repository/knowledge.go@@ -434,6 +434,10 @@ func (r *knowledgeRepository) CheckKnowledgeExists(Where("tenant_id = ? AND knowledge_base_id = ? AND parse_status NOT IN ?",tenantID, kbID, []string{"failed", "deleting"})+ if params.ExcludeKnowledgeID != "" {+ query = query.Where("id <> ?", params.ExcludeKnowledgeID)+ }+switch params.Type {case "file":if params.DataSourceID != "" && params.ExternalID != "" {diff --git a/internal/application/repository/knowledge_replace.go b/internal/application/repository/knowledge_replace.gonew file mode 100644index 000000000..dec694ef2--- /dev/null+++ b/internal/application/repository/knowledge_replace.go@@ -0,0 +1,42 @@+package repository++import (+ "context"+ "fmt"++ "github.com/Tencent/WeKnora/internal/types"+ "gorm.io/gorm"+)++// ReplaceKnowledge keeps the previous row live if inserting its replacement+// fails. No file, index, queue or other remote operation runs in this transaction.+func (r *knowledgeRepository) ReplaceKnowledge(+ ctx context.Context, before, after *types.Knowledge, tagIDs []string,+) error {+ if before == nil || after == nil || before.ID == "" || after.ID == "" || before.ID == after.ID ||+ before.TenantID != after.TenantID || before.KnowledgeBaseID != after.KnowledgeBaseID {+ return fmt.Errorf("invalid knowledge replacement")+ }+ return r.db.WithContext(ctx).Transaction(func(tx *gorm.DB) error {+ // The primary-key lookup locks only the source row. Comparing the snapshot+ // prevents a second sync from replacing a row another writer has retired+ // or changed while this sync was preparing its replacement.+ result := tx.Where("id = ? AND tenant_id = ? AND knowledge_base_id = ? AND updated_at = ?",+ before.ID, before.TenantID, before.KnowledgeBaseID, before.UpdatedAt).+ Delete(&types.Knowledge{})+ if result.Error != nil {+ return result.Error+ }+ if result.RowsAffected != 1 {+ return fmt.Errorf("knowledge changed during replacement: %s", before.ID)+ }+ repo := &knowledgeRepository{db: tx}+ if err := repo.CreateKnowledge(ctx, after); err != nil {+ return err+ }+ if len(tagIDs) > 0 {+ return repo.SetKnowledgeTags(ctx, after.ID, tagIDs)+ }+ return nil+ })+}diff --git a/internal/application/repository/knowledge_replace_postgres_test.go b/internal/application/repository/knowledge_replace_postgres_test.gonew file mode 100644index 000000000..592a5e72d--- /dev/null+++ b/internal/application/repository/knowledge_replace_postgres_test.go@@ -0,0 +1,74 @@+package repository++import (+ "context"+ "os"+ "strings"+ "testing"++ "github.com/Tencent/WeKnora/internal/types"+ "github.com/google/uuid"+ "github.com/stretchr/testify/require"+ "gorm.io/driver/postgres"+ "gorm.io/gorm"+)++func TestReplaceKnowledgePostgresRollbackAndCommit(t *testing.T) {+ dsn := os.Getenv("WEKNORA_REPOSITORY_TEST_POSTGRES_DSN")+ if dsn == "" {+ t.Skip("set WEKNORA_REPOSITORY_TEST_POSTGRES_DSN for PostgreSQL replacement tests")+ }+ admin, err := gorm.Open(postgres.Open(dsn), &gorm.Config{})+ require.NoError(t, err)+ adminSQL, err := admin.DB()+ require.NoError(t, err)+ schema := "knowledge_replace_" + strings.ReplaceAll(uuid.NewString(), "-", "")+ require.NoError(t, admin.Exec("CREATE SCHEMA "+schema).Error)+ t.Cleanup(func() {+ require.NoError(t, admin.Exec("DROP SCHEMA "+schema+" CASCADE").Error)+ require.NoError(t, adminSQL.Close())+ })+ separator := " "+ if strings.Contains(dsn, "://") {+ separator = "?"+ if strings.Contains(dsn, "?") {+ separator = "&"+ }+ }+ db, err := gorm.Open(postgres.Open(dsn+separator+"search_path="+schema), &gorm.Config{})+ require.NoError(t, err)+ sqlDB, err := db.DB()+ require.NoError(t, err)+ t.Cleanup(func() { require.NoError(t, sqlDB.Close()) })+ require.NoError(t, db.AutoMigrate(&types.Knowledge{}, &types.KnowledgeTagRelation{}))+ ctx := context.Background()+ repo := NewKnowledgeRepository(db)+ old := &types.Knowledge{ID: "old", TenantID: 7, KnowledgeBaseID: "kb", Type: "file"}+ occupied := &types.Knowledge{ID: "occupied", TenantID: 7, KnowledgeBaseID: "kb", Type: "file"}+ require.NoError(t, repo.CreateKnowledge(ctx, old))+ require.NoError(t, repo.CreateKnowledge(ctx, occupied))+ old, err = repo.GetKnowledgeByID(ctx, 7, old.ID)+ require.NoError(t, err)+ require.NoError(t, repo.SetKnowledgeTags(ctx, old.ID, []string{"old-tag"}))+ replacement := &types.Knowledge{ID: occupied.ID, TenantID: 7, KnowledgeBaseID: "kb", Type: "file"}+ require.Error(t, repo.ReplaceKnowledge(ctx, old, replacement, []string{"new-tag"}))+ retained, err := repo.GetKnowledgeByID(ctx, 7, old.ID)+ require.NoError(t, err, "failed insertion must roll back the retirement")+ require.False(t, retained.DeletedAt.Valid)+ var tags []types.KnowledgeTagRelation+ require.NoError(t, db.Where("knowledge_id = ?", old.ID).Find(&tags).Error)+ require.Len(t, tags, 1)+ require.Equal(t, "old-tag", tags[0].TagID)+ replacement.ID = "new"+ require.NoError(t, repo.ReplaceKnowledge(ctx, old, replacement, []string{"new-tag"}))+ _, err = repo.GetKnowledgeByID(ctx, 7, old.ID)+ require.Error(t, err)+ _, err = repo.GetKnowledgeByID(ctx, 7, replacement.ID)+ require.NoError(t, err)+ tags = nil+ require.NoError(t, db.Where("knowledge_id = ?", replacement.ID).Find(&tags).Error)+ require.Len(t, tags, 1)+ require.Equal(t, "new-tag", tags[0].TagID)+ replacement.ID = "stale-replacement"+ require.ErrorContains(t, repo.ReplaceKnowledge(ctx, old, replacement, nil), "knowledge changed")+}diff --git a/internal/application/repository/knowledge_replace_test.go b/internal/application/repository/knowledge_replace_test.gonew file mode 100644index 000000000..8e39ca1d1--- /dev/null+++ b/internal/application/repository/knowledge_replace_test.go@@ -0,0 +1,160 @@+package repository++import (+ "context"+ "path/filepath"+ "testing"+ "time"++ "github.com/Tencent/WeKnora/internal/types"+ "github.com/Tencent/WeKnora/internal/types/interfaces"+ "github.com/stretchr/testify/require"+ "gorm.io/driver/sqlite"+ "gorm.io/gorm"+)++func replacementFixture(t *testing.T) (*gorm.DB, interfaces.KnowledgeRepository, *types.Knowledge, *types.Knowledge) {+ t.Helper()+ db, err := gorm.Open(sqlite.Open(filepath.Join(t.TempDir(), "replacement.db")), &gorm.Config{})+ require.NoError(t, err)+ sqlDB, err := db.DB()+ require.NoError(t, err)+ t.Cleanup(func() { require.NoError(t, sqlDB.Close()) })+ require.NoError(t, db.AutoMigrate(&types.Knowledge{}, &types.KnowledgeTagRelation{}))+ repo := NewKnowledgeRepository(db)+ before := &types.Knowledge{+ ID: "old", TenantID: 7, KnowledgeBaseID: "kb-1", Type: "file",+ FilePath: "old.md", ParseStatus: types.ParseStatusCompleted, EnableStatus: "enabled",+ Metadata: types.JSON(`{"datasource_id":"ds-1","external_id":"doc-1"}`),+ }+ require.NoError(t, repo.CreateKnowledge(context.Background(), before))+ before, err = repo.GetKnowledgeByID(context.Background(), 7, before.ID)+ require.NoError(t, err)+ after := &types.Knowledge{+ ID: "new", TenantID: 7, KnowledgeBaseID: "kb-1", Type: "file",+ FilePath: "new.md", ParseStatus: types.ParseStatusPending, Metadata: before.Metadata,+ }+ return db, repo, before, after+}++func TestReplaceKnowledgeRollsBackRetirementWhenInsertFails(t *testing.T) {+ db, repo, before, after := replacementFixture(t)+ require.NoError(t, db.Exec(`CREATE TRIGGER fail_replacement BEFORE INSERT ON knowledges+ WHEN NEW.id = 'new' BEGIN SELECT RAISE(FAIL, 'forced insert failure'); END`).Error)++ err := repo.ReplaceKnowledge(context.Background(), before, after, nil)+ require.ErrorContains(t, err, "forced insert failure")+ persisted, err := repo.GetKnowledgeByID(context.Background(), 7, before.ID)+ require.NoError(t, err)+ require.Equal(t, "old.md", persisted.FilePath)+ require.Equal(t, types.ParseStatusCompleted, persisted.ParseStatus)+ require.False(t, persisted.DeletedAt.Valid)+ _, err = repo.GetKnowledgeByID(context.Background(), 7, after.ID)+ require.Error(t, err)+}++func TestReplaceKnowledgeCommitsBothRowsAndRejectsStaleReplacement(t *testing.T) {+ db, repo, before, after := replacementFixture(t)+ require.NoError(t, repo.ReplaceKnowledge(context.Background(), before, after, nil))+ _, err := repo.GetKnowledgeByID(context.Background(), 7, before.ID)+ require.Error(t, err)+ persisted, err := repo.GetKnowledgeByID(context.Background(), 7, after.ID)+ require.NoError(t, err)+ require.Equal(t, after.Metadata, persisted.Metadata)+ var retired types.Knowledge+ require.NoError(t, db.Unscoped().First(&retired, "id = ?", before.ID).Error)+ require.True(t, retired.DeletedAt.Valid, "retain a tombstone until resource cleanup succeeds")++ after.ID = "racing-replacement"+ require.ErrorContains(t, repo.ReplaceKnowledge(context.Background(), before, after, nil), "knowledge changed")+ var count int64+ require.NoError(t, db.Model(&types.Knowledge{}).Count(&count).Error)+ require.EqualValues(t, 1, count)+}++func TestReplaceKnowledgeRejectsChangedOrForeignSnapshot(t *testing.T) {+ for _, change := range []string{"updated", "tenant", "kb", "same-id", "delete-failure"} {+ t.Run(change, func(t *testing.T) {+ db, repo, before, after := replacementFixture(t)+ switch change {+ case "updated":+ require.NoError(t, repo.UpdateKnowledgeColumn(context.Background(), before.ID,+ "updated_at", before.UpdatedAt.Add(time.Second)))+ case "tenant":+ after.TenantID+++ case "kb":+ after.KnowledgeBaseID = "kb-other"+ case "same-id":+ after.ID = before.ID+ case "delete-failure":+ require.NoError(t, db.Exec(`CREATE TRIGGER fail_retirement BEFORE UPDATE OF deleted_at ON knowledges+ BEGIN SELECT RAISE(FAIL, 'forced delete failure'); END`).Error)+ }+ require.Error(t, repo.ReplaceKnowledge(context.Background(), before, after, nil))+ _, err := repo.GetKnowledgeByID(context.Background(), 7, before.ID)+ require.NoError(t, err)+ var count int64+ require.NoError(t, db.Unscoped().Model(&types.Knowledge{}).Count(&count).Error)+ require.EqualValues(t, 1, count)+ })+ }+}++func TestCheckKnowledgeExistsExcludesOnlyReplacementSource(t *testing.T) {+ for _, kind := range []string{"file", "url"} {+ t.Run(kind, func(t *testing.T) {+ _, repo, before, _ := replacementFixture(t)+ require.NoError(t, repo.UpdateKnowledgeColumns(context.Background(), before.ID,+ map[string]interface{}{+ "type": kind, "file_hash": "same-hash", "file_type": "md", "source": "https://example.com/doc",+ }))+ params := &types.KnowledgeCheckParams{+ Type: kind, FileHash: "same-hash", FileType: "md",+ URL: "https://example.com/doc", ExcludeKnowledgeID: before.ID,+ }+ exists, _, err := repo.CheckKnowledgeExists(context.Background(), 7, "kb-1", params)+ require.NoError(t, err)+ require.False(t, exists)+ other := &types.Knowledge{+ ID: "other", TenantID: 7, KnowledgeBaseID: "kb-1", Type: kind,+ FileHash: "same-hash", FileType: "md", Source: params.URL, ParseStatus: types.ParseStatusCompleted,+ }+ require.NoError(t, repo.CreateKnowledge(context.Background(), other))+ exists, duplicate, err := repo.CheckKnowledgeExists(context.Background(), 7, "kb-1", params)+ require.NoError(t, err)+ require.True(t, exists)+ require.Equal(t, other.ID, duplicate.ID)+ })+ }+}++func TestReplaceKnowledgeTagsCommitOrRollbackWithRows(t *testing.T) {+ for _, fail := range []bool{false, true} {+ t.Run(map[bool]string{false: "commit", true: "tag failure"}[fail], func(t *testing.T) {+ db, repo, before, after := replacementFixture(t)+ require.NoError(t, repo.SetKnowledgeTags(context.Background(), before.ID, []string{"old-tag"}))+ if fail {+ require.NoError(t, db.Exec(`CREATE TRIGGER fail_new_tags BEFORE INSERT ON knowledge_tag_relations+ WHEN NEW.knowledge_id = 'new' BEGIN SELECT RAISE(FAIL, 'forced tag failure'); END`).Error)+ }+ err := repo.ReplaceKnowledge(context.Background(), before, after, []string{"new-tag"})+ if fail {+ require.ErrorContains(t, err, "forced tag failure")+ _, err = repo.GetKnowledgeByID(context.Background(), 7, before.ID)+ require.NoError(t, err)+ _, err = repo.GetKnowledgeByID(context.Background(), 7, after.ID)+ require.Error(t, err)+ } else {+ require.NoError(t, err)+ }+ var relations []types.KnowledgeTagRelation+ require.NoError(t, db.Where("knowledge_id = ?", after.ID).Find(&relations).Error)+ if fail {+ require.Empty(t, relations)+ } else {+ require.Len(t, relations, 1)+ require.Equal(t, "new-tag", relations[0].TagID)+ }+ })+ }+}diff --git a/internal/application/service/datasource_atomic_replace_test.go b/internal/application/service/datasource_atomic_replace_test.gonew file mode 100644index 000000000..e92e539b7--- /dev/null+++ b/internal/application/service/datasource_atomic_replace_test.go@@ -0,0 +1,78 @@+package service++import (+ "context"+ "errors"+ "testing"++ "github.com/Tencent/WeKnora/internal/types"+ "github.com/Tencent/WeKnora/internal/types/interfaces"+ "github.com/stretchr/testify/require"+)++type atomicIngestRepo struct {+ interfaces.KnowledgeRepository+ row *types.Knowledge+ err error+}++func (r *atomicIngestRepo) FindByDataSourceExternalID(+ context.Context, uint64, string, string, string,+) (*types.Knowledge, error) {+ return r.row, r.err+}++func (r *atomicIngestRepo) HardDeleteKnowledge(context.Context, uint64, string) error {+ r.row = nil+ return nil+}++type atomicIngestService struct {+ sweepFakeKS+ replacements int+}++func (s *atomicIngestService) ReplaceKnowledgeFromDataSource(+ context.Context, string, *types.FetchedItem, map[string]string, []string, string,+) (*types.Knowledge, error) {+ s.replacements+++ return nil, s.createErr+}++func (s *atomicIngestService) CreateKnowledgeFromURL(+ context.Context, string, string, string, string, *bool, string, []string, string, *types.KnowledgeProcessOverrides,+) (*types.Knowledge, error) {+ s.events = append(s.events, "create-url")+ return nil, s.createErr+}++func TestIngestItemReplacementFailureKeepsPreviousKnowledge(t *testing.T) {+ for _, item := range []*types.FetchedItem{+ {ExternalID: "doc-1", Content: []byte("new content"), FileName: "doc.md"},+ {ExternalID: "doc-1", URL: "https://example.com/doc"},+ } {+ t.Run(item.FileName+item.URL, func(t *testing.T) {+ old := &types.Knowledge{ID: "old-knowledge", Type: "file"}+ repo := &atomicIngestRepo{row: old}+ ks := &atomicIngestService{sweepFakeKS: sweepFakeKS{repo: repo, createErr: errors.New("insert failed")}}+ s := &DataSourceService{knowledgeService: ks}+ ds := &types.DataSource{ID: "ds-1", TenantID: 7, KnowledgeBaseID: "kb-1"}++ _, err := s.ingestItem(context.Background(), ds, item, nil)+ require.Error(t, err)+ require.Same(t, old, repo.row, "a failed replacement must retain the previous row")+ require.Empty(t, ks.deleted, "old resources must not be deleted before replacement commits")+ require.Equal(t, 1, ks.replacements)+ })+ }+}++func TestIngestItemLookupFailureDoesNotCreateDuplicate(t *testing.T) {+ repo := &atomicIngestRepo{err: errors.New("lookup failed")}+ ks := &atomicIngestService{sweepFakeKS: sweepFakeKS{repo: repo}}+ s := &DataSourceService{knowledgeService: ks}+ _, err := s.ingestItem(context.Background(), &types.DataSource{ID: "ds-1"},+ &types.FetchedItem{ExternalID: "doc-1", Content: []byte("content"), FileName: "doc.md"}, nil)+ require.ErrorContains(t, err, "lookup failed")+ require.Empty(t, ks.events)+}diff --git a/internal/application/service/datasource_conflict_skip_test.go b/internal/application/service/datasource_conflict_skip_test.goindex ca509c5e0..1300e7561 100644--- a/internal/application/service/datasource_conflict_skip_test.go+++ b/internal/application/service/datasource_conflict_skip_test.go@@ -60,12 +60,24 @@ func (conflictSkipConnector) FetchIncremental(return items, nil, err}+type conflictSkipKnowledgeService struct {+ *sweepFakeKS+ replacements int+}++func (s *conflictSkipKnowledgeService) ReplaceKnowledgeFromDataSource(+ _ context.Context, _ string, _ *types.FetchedItem, _ map[string]string, _ []string, _ string,+) (*types.Knowledge, error) {+ s.replacements+++ return &types.Knowledge{ID: "new-knowledge"}, nil+}+// newConflictSkipHarness wires a full-sync ProcessSync fixture whose data// source carries the given conflict strategy and whose KB already contains the// synced item under the same external_id.func newConflictSkipHarness(t *testing.T, strategy string,-) (*DataSourceService, *types.DataSource, *deletionLookupKnowledgeRepo, *sweepFakeKS, *types.SyncLog) {+) (*DataSourceService, *types.DataSource, *deletionLookupKnowledgeRepo, *conflictSkipKnowledgeService, *types.SyncLog) {t.Helper()configJSON, err := (&types.DataSourceConfig{Type: conflictSkipConnectorType}).ToJSON()require.NoError(t, err)@@ -83,7 +95,7 @@ func newConflictSkipHarness(}existing := &types.Knowledge{ID: "knowledge-exists", FileName: "existing.md"}repo := &deletionLookupKnowledgeRepo{knowledge: existing}- ks := &sweepFakeKS{repo: repo}+ ks := &conflictSkipKnowledgeService{sweepFakeKS: &sweepFakeKS{repo: repo}}syncLog := &types.SyncLog{ID: "log-conflict-skip",DataSourceID: ds.ID,@@ -134,6 +146,7 @@ func TestProcessSync_SkipStrategyKeepsExistingCountsSkipped(t *testing.T) {log := runConflictSkipSync(t, svc, ds, syncLog)assert.Empty(t, repo.hardDeleted, "skip strategy must not hard-delete the existing knowledge")+ assert.Zero(t, ks.replacements)assert.NotContains(t, ks.events, "delete:knowledge-exists")assert.NotContains(t, ks.events, "create:existing.md")assert.Equal(t, 0, log.ItemsCreated)@@ -142,15 +155,15 @@ func TestProcessSync_SkipStrategyKeepsExistingCountsSkipped(t *testing.T) {}// Default strategy (overwrite) keeps the historical replace behavior: the-// existing item is deleted and re-created, counted as updated.+// existing item is replaced transactionally, counted as updated.func TestProcessSync_OverwriteStrategyReplacesExisting(t *testing.T) {svc, ds, repo, ks, syncLog := newConflictSkipHarness(t, types.ConflictStrategyOverwrite)log := runConflictSkipSync(t, svc, ds, syncLog)- assert.Equal(t, []string{"knowledge-exists"}, repo.hardDeleted)- assert.Contains(t, ks.events, "delete:knowledge-exists")- assert.Contains(t, ks.events, "create:existing.md")+ assert.Empty(t, repo.hardDeleted)+ assert.Empty(t, ks.events, "the caller must not delete or create outside the replacement transaction")+ assert.Equal(t, 1, ks.replacements)assert.Equal(t, 0, log.ItemsCreated)assert.Equal(t, 1, log.ItemsUpdated)assert.Equal(t, 0, log.ItemsSkipped)diff --git a/internal/application/service/datasource_service.go b/internal/application/service/datasource_service.goindex f651201f5..660aaf5c6 100644--- a/internal/application/service/datasource_service.go+++ b/internal/application/service/datasource_service.go@@ -1374,7 +1374,7 @@ func (s *DataSourceService) validateDataSourceConfig(ctx context.Context, ds *ty}// ingestItem writes a single FetchedItem into the knowledge base.-// If a knowledge item with the same external_id already exists, it is deleted first (update = delete + re-create).+// Existing datasource items are replaced atomically at the knowledge-row boundary.//// Routing logic:// - Has Content bytes → CreateKnowledgeFromFile (走完整的文档解析 pipeline)@@ -1412,18 +1412,16 @@ func (s *DataSourceService) ingestItem(ctx context.Context, ds *types.DataSourcemetadata[k] = v}- // Check if a knowledge item with this external_id already exists → delete it first (update)+ // Existing items are replaced through a short DB transaction. Validation and+ // file storage must succeed before the previous row or resources are retired.isUpdate := falseif item.ExternalID != "" {repo := s.knowledgeService.GetRepository()- // Scope the lookup to items owned by this data source so identical- // external IDs from two data sources cannot collide or overwrite each- // other during updates.existing, err := repo.FindByDataSourceExternalID(ctx, ds.TenantID, ds.KnowledgeBaseID, ds.ID, item.ExternalID)if err != nil {- logger.Warnf(ctx, "failed to check existing knowledge for external_id=%s: %v", item.ExternalID, err)- // Non-fatal: proceed with creation (may produce duplicate)- } else if existing != nil {+ return false, fmt.Errorf("check existing datasource knowledge: %w", err)+ }+ if existing != nil {// Conflict strategy "skip" (跳过已存在): the item already exists in// the KB, so leave it untouched and report the conflict as a skip// via the shared DuplicateKnowledgeError path (Tencent/WeKnora#4640 —@@ -1442,15 +1440,13 @@ func (s *DataSourceService) ingestItem(ctx context.Context, ds *types.DataSourceKnowledge: existing,}}- logger.Infof(ctx, "found existing knowledge %s for external_id=%s, deleting for update", existing.ID, item.ExternalID)- if err := s.knowledgeService.DeleteKnowledge(ctx, existing.ID); err != nil {- logger.Warnf(ctx, "failed to delete existing knowledge %s: %v", existing.ID, err)- } else {- if herr := repo.HardDeleteKnowledge(ctx, ds.TenantID, existing.ID); herr != nil {- logger.Warnf(ctx, "failed to hard-delete replaced knowledge %s: %v", existing.ID, herr)- }- isUpdate = true+ _, err := s.knowledgeService.ReplaceKnowledgeFromDataSource(+ ctx, existing.ID, item, metadata, tagIDs, channel)+ var dupErr *types.DuplicateKnowledgeError+ if err == nil || (errors.As(err, &dupErr) && dupIsSameNode(dupErr, item)) {+ s.sweepStaleSubtree(ctx, ds, item)}+ return err == nil, err}}diff --git a/internal/application/service/knowledge_create.go b/internal/application/service/knowledge_create.goindex 33adbfc71..3cd21ffd8 100644--- a/internal/application/service/knowledge_create.go+++ b/internal/application/service/knowledge_create.go@@ -25,8 +25,19 @@ import (// CreateKnowledgeFromFile creates a knowledge entry from an uploaded filefunc (s *knowledgeService) CreateKnowledgeFromFile(ctx context.Context,- kbID string, file *multipart.FileHeader, metadata map[string]string, enableMultimodel *bool, customFileName string, tagIDs []string, channel string,+ kbID string, file *multipart.FileHeader, metadata map[string]string, enableMultimodel *bool,+ customFileName string, tagIDs []string, channel string,processOverrides *types.KnowledgeProcessOverrides,+) (*types.Knowledge, error) {+ return s.createKnowledgeFromFile(ctx, kbID, file, metadata, enableMultimodel, customFileName,+ tagIDs, channel, processOverrides, nil)+}++func (s *knowledgeService) createKnowledgeFromFile(ctx context.Context,+ kbID string, file *multipart.FileHeader, metadata map[string]string, enableMultimodel *bool,+ customFileName string, tagIDs []string, channel string,+ processOverrides *types.KnowledgeProcessOverrides,+ replacement *knowledgeReplacement,) (*types.Knowledge, error) {logger.Info(ctx, "Start creating knowledge from file")@@ -78,10 +89,9 @@ func (s *knowledgeService) CreateKnowledgeFromFile(ctx context.Context,return nil, ErrInvalidFileType}- // JSON content validation stays in the HTTP upload handler (and- // ReplaceKnowledgeFile), not here: datasource sync may have already- // deleted the previous knowledge for this external_id, and IM swallows- // create errors — an early 400 would drop the document with no failed row.+ // Fresh imports keep JSON validation in the HTTP upload handler (IM needs+ // a failed row for asynchronous parse errors). Replacement entry points+ // validate JSON before retiring the previous source.// Calculate file hash for deduplicationlogger.Info(ctx, "Calculating file hash")@@ -108,7 +118,7 @@ func (s *knowledgeService) CreateKnowledgeFromFile(ctx context.Context,checkParams.DataSourceID = metadata["datasource_id"]checkParams.ExternalID = metadata["external_id"]}- exists, existingKnowledge, err := s.repo.CheckKnowledgeExists(ctx, tenantID, kbID, checkParams)+ exists, existingKnowledge, err := s.checkKnowledgeForCreation(ctx, tenantID, kbID, replacement, checkParams)if err != nil {logger.Errorf(ctx, "Failed to check knowledge existence: %v", err)return nil, err@@ -125,7 +135,7 @@ func (s *knowledgeService) CreateKnowledgeFromFile(ctx context.Context,// Check storage quotatenantInfo := ctx.Value(types.TenantInfoContextKey).(*types.Tenant)- if tenantInfo.StorageQuota > 0 && tenantInfo.StorageUsed >= tenantInfo.StorageQuota {+ if tenantInfo.StorageQuota > 0 && knowledgeCreationStorageUsed(tenantInfo, replacement) >= tenantInfo.StorageQuota {logger.Error(ctx, "Storage quota exceeded")return nil, types.NewStorageQuotaExceededError()}@@ -205,15 +215,18 @@ func (s *knowledgeService) CreateKnowledgeFromFile(ctx context.Context,// Save knowledge record to database after the file is safely stored.logger.Info(ctx, "Saving knowledge record to database")- if err := s.repo.CreateKnowledge(ctx, knowledge); err != nil {+ if err := s.persistKnowledgeForCreation(ctx, knowledge, replacement); err != nil {logger.Errorf(ctx, "Failed to create knowledge record, ID: %s, error: %v", knowledge.ID, err)- if deleteErr := fileSvc.DeleteFile(ctx, filePath); deleteErr != nil {- logger.Errorf(ctx, "Failed to delete saved file after knowledge creation failed, path: %s, error: %v", filePath, deleteErr)+ if replacement == nil || !replacement.keepFile {+ if deleteErr := fileSvc.DeleteFile(context.WithoutCancel(ctx), filePath); deleteErr != nil {+ logger.Errorf(ctx, "Failed to delete saved file after knowledge creation failed, path: %s, error: %v",+ filePath, deleteErr)+ }}return nil, err}// Set tag relations- if err := s.setAndAttachKnowledgeTags(ctx, tenantID, kbID, knowledge, tagIDs); err != nil {+ if err := s.attachCreationTags(ctx, knowledge, tagIDs, replacement); err != nil {logger.Errorf(ctx, "Failed to set knowledge tags, knowledge ID: %s, error: %v", knowledge.ID, err)return nil, err}@@ -309,16 +322,28 @@ func isFileURL(rawURL, fileName, fileType string) bool {}func (s *knowledgeService) CreateKnowledgeFromURL(ctx context.Context,- kbID string, rawURL string, fileName string, fileType string, enableMultimodel *bool, title string, tagIDs []string, channel string,+ kbID string, rawURL string, fileName string, fileType string, enableMultimodel *bool,+ title string, tagIDs []string, channel string,processOverrides *types.KnowledgeProcessOverrides,+) (*types.Knowledge, error) {+ return s.createKnowledgeFromURL(ctx, kbID, rawURL, fileName, fileType, enableMultimodel,+ title, tagIDs, channel, processOverrides, nil)+}++func (s *knowledgeService) createKnowledgeFromURL(ctx context.Context,+ kbID string, rawURL string, fileName string, fileType string, enableMultimodel *bool,+ title string, tagIDs []string, channel string,+ processOverrides *types.KnowledgeProcessOverrides,+ replacement *knowledgeReplacement,) (*types.Knowledge, error) {logger.Info(ctx, "Start creating knowledge from URL")logger.Infof(ctx, "Knowledge base ID: %s, URL: %s", kbID, rawURL)// Route to file_url logic when the URL points to a downloadable fileif isFileURL(rawURL, fileName, fileType) {- return s.createKnowledgeFromFileURL(- ctx, kbID, rawURL, fileName, fileType, enableMultimodel, title, tagIDs, channel, processOverrides,+ return s.createKnowledgeFromFileURLWithReplacement(+ ctx, kbID, rawURL, fileName, fileType, enableMultimodel,+ title, tagIDs, channel, processOverrides, replacement,)}@@ -353,11 +378,12 @@ func (s *knowledgeService) CreateKnowledgeFromURL(ctx context.Context,tenantID := ctx.Value(types.TenantIDContextKey).(uint64)logger.Infof(ctx, "Checking if URL exists, tenant ID: %d", tenantID)fileHash := calculateStr(url)- exists, existingKnowledge, err := s.repo.CheckKnowledgeExists(ctx, tenantID, kbID, &types.KnowledgeCheckParams{- Type: "url",- URL: url,- FileHash: fileHash,- })+ exists, existingKnowledge, err := s.checkKnowledgeForCreation(ctx, tenantID, kbID, replacement,+ &types.KnowledgeCheckParams{+ Type: "url",+ URL: url,+ FileHash: fileHash,+ })if err != nil {logger.Errorf(ctx, "Failed to check knowledge existence: %v", err)return nil, err@@ -376,7 +402,7 @@ func (s *knowledgeService) CreateKnowledgeFromURL(ctx context.Context,// Check storage quotatenantInfo := ctx.Value(types.TenantInfoContextKey).(*types.Tenant)- if tenantInfo.StorageQuota > 0 && tenantInfo.StorageUsed >= tenantInfo.StorageQuota {+ if tenantInfo.StorageQuota > 0 && knowledgeCreationStorageUsed(tenantInfo, replacement) >= tenantInfo.StorageQuota {logger.Error(ctx, "Storage quota exceeded")return nil, types.NewStorageQuotaExceededError()}@@ -407,12 +433,12 @@ func (s *knowledgeService) CreateKnowledgeFromURL(ctx context.Context,return nil, err}- if err := s.repo.CreateKnowledge(ctx, knowledge); err != nil {+ if err := s.persistKnowledgeForCreation(ctx, knowledge, replacement); err != nil {logger.Errorf(ctx, "Failed to create knowledge record: %v", err)return nil, err}// Set tag relations- if err := s.setAndAttachKnowledgeTags(ctx, tenantID, kbID, knowledge, tagIDs); err != nil {+ if err := s.attachCreationTags(ctx, knowledge, tagIDs, replacement); err != nil {logger.Errorf(ctx, "Failed to set knowledge tags, knowledge ID: %s, error: %v", knowledge.ID, err)return nil, err}@@ -512,18 +538,20 @@ func extractFileNameFromContentDisposition(header string) string {}// createKnowledgeFromFileURL is the internal implementation for file URL knowledge creation.-// Called by CreateKnowledgeFromURL when the URL is detected as a direct file download.+// The replacement-aware variant shares the pipeline for datasource updates.func (s *knowledgeService) createKnowledgeFromFileURL(- ctx context.Context,- kbID string,- fileURL string,- fileName string,- fileType string,- enableMultimodel *bool,- title string,- tagIDs []string,- channel string,+ ctx context.Context, kbID, fileURL, fileName, fileType string,+ enableMultimodel *bool, title string, tagIDs []string, channel string,processOverrides *types.KnowledgeProcessOverrides,+) (*types.Knowledge, error) {+ return s.createKnowledgeFromFileURLWithReplacement(ctx, kbID, fileURL, fileName, fileType,+ enableMultimodel, title, tagIDs, channel, processOverrides, nil)+}++func (s *knowledgeService) createKnowledgeFromFileURLWithReplacement(+ ctx context.Context, kbID, fileURL, fileName, fileType string,+ enableMultimodel *bool, title string, tagIDs []string, channel string,+ processOverrides *types.KnowledgeProcessOverrides, replacement *knowledgeReplacement,) (*types.Knowledge, error) {logger.Info(ctx, "Start creating knowledge from file URL")logger.Infof(ctx, "Knowledge base ID: %s, file URL: %s", kbID, fileURL)@@ -590,11 +618,12 @@ func (s *knowledgeService) createKnowledgeFromFileURL(// Check for duplicate (by URL hash)tenantID := ctx.Value(types.TenantIDContextKey).(uint64)fileHash := calculateStr(fileURL)- exists, existingKnowledge, err := s.repo.CheckKnowledgeExists(ctx, tenantID, kbID, &types.KnowledgeCheckParams{- Type: "file_url",- URL: fileURL,- FileHash: fileHash,- })+ exists, existingKnowledge, err := s.checkKnowledgeForCreation(ctx, tenantID, kbID, replacement,+ &types.KnowledgeCheckParams{+ Type: "file_url",+ URL: fileURL,+ FileHash: fileHash,+ })if err != nil {logger.Errorf(ctx, "Failed to check knowledge existence: %v", err)return nil, err@@ -612,7 +641,7 @@ func (s *knowledgeService) createKnowledgeFromFileURL(// Check storage quotatenantInfo := ctx.Value(types.TenantInfoContextKey).(*types.Tenant)- if tenantInfo.StorageQuota > 0 && tenantInfo.StorageUsed >= tenantInfo.StorageQuota {+ if tenantInfo.StorageQuota > 0 && knowledgeCreationStorageUsed(tenantInfo, replacement) >= tenantInfo.StorageQuota {logger.Error(ctx, "Storage quota exceeded")return nil, types.NewStorageQuotaExceededError()}@@ -650,12 +679,12 @@ func (s *knowledgeService) createKnowledgeFromFileURL(}}- if err := s.repo.CreateKnowledge(ctx, knowledge); err != nil {+ if err := s.persistKnowledgeForCreation(ctx, knowledge, replacement); err != nil {logger.Errorf(ctx, "Failed to create knowledge record: %v", err)return nil, err}// Set tag relations- if err := s.setAndAttachKnowledgeTags(ctx, tenantID, kbID, knowledge, tagIDs); err != nil {+ if err := s.attachCreationTags(ctx, knowledge, tagIDs, replacement); err != nil {logger.Errorf(ctx, "Failed to set knowledge tags, knowledge ID: %s, error: %v", knowledge.ID, err)return nil, err}diff --git a/internal/application/service/knowledge_datasource_replace.go b/internal/application/service/knowledge_datasource_replace.gonew file mode 100644index 000000000..fbe2fd2b0--- /dev/null+++ b/internal/application/service/knowledge_datasource_replace.go@@ -0,0 +1,141 @@+package service++import (+ "context"+ "fmt"++ "github.com/Tencent/WeKnora/internal/logger"+ "github.com/Tencent/WeKnora/internal/types"+)++// knowledgeReplacement is local to a single creation pipeline; a successful+// persist switches the live rows before that pipeline can enqueue a worker.+type knowledgeReplacement struct {+ previous *types.Knowledge+ metadata map[string]string+ tagIDs []string+ committed bool+ keepFile bool // a failed commit/read may still have published the new path+}++func (s *knowledgeService) checkKnowledgeForCreation(ctx context.Context, tenantID uint64, kbID string,+ replacement *knowledgeReplacement, params *types.KnowledgeCheckParams,+) (bool, *types.Knowledge, error) {+ if replacement != nil {+ params.ExcludeKnowledgeID = replacement.previous.ID+ }+ return s.repo.CheckKnowledgeExists(ctx, tenantID, kbID, params)+}++func (s *knowledgeService) persistKnowledgeForCreation(ctx context.Context, knowledge *types.Knowledge,+ replacement *knowledgeReplacement,+) error {+ if replacement == nil {+ return s.repo.CreateKnowledge(ctx, knowledge)+ }+ // URL creation normally has no datasource metadata. Include it in the same+ // INSERT so a committed replacement is immediately discoverable on retry.+ metadata, err := mergeKnowledgeMetadata(knowledge.Metadata, replacement.metadata)+ if err != nil {+ return err+ }+ knowledge.Metadata = metadata+ if err := s.repo.ReplaceKnowledge(ctx, replacement.previous, knowledge, replacement.tagIDs); err != nil {+ // A commit can succeed while its acknowledgement is lost. Only discard+ // the staged file when a read confirms the previous row is still live.+ cleanupCtx := context.WithoutCancel(ctx)+ old, oldErr := s.repo.GetKnowledgeByID(cleanupCtx, replacement.previous.TenantID, replacement.previous.ID)+ if oldErr == nil && old != nil {+ return err+ }+ current, readErr := s.repo.GetKnowledgeByID(cleanupCtx, knowledge.TenantID, knowledge.ID)+ if readErr != nil || current == nil {+ replacement.keepFile = true+ return err+ }+ logger.Warnf(ctx, "Knowledge replacement %s committed despite reported error: %v", knowledge.ID, err)+ }+ replacement.committed = true+ return nil+}++// ReplaceKnowledgeFromDataSource preserves the previous row and its resources+// until validation, file storage and the replacement INSERT succeed. The DB+// transaction covers both knowledge rows and new tag relations; parsing remains asynchronous+// and cleanup of the retired row's external resources is best-effort after commit.+func (s *knowledgeService) ReplaceKnowledgeFromDataSource(ctx context.Context, knowledgeID string,+ item *types.FetchedItem, metadata map[string]string, tagIDs []string, channel string,+) (*types.Knowledge, error) {+ if item == nil || (len(item.Content) == 0 && item.URL == "") {+ return nil, fmt.Errorf("item has neither content nor URL")+ }+ plan, err := s.planKnowledgeDelete(ctx, []string{knowledgeID})+ if err != nil {+ return nil, err+ }+ previous := plan.knowledge[0]+ stored := previous.GetMetadata()+ if metadata["datasource_id"] == "" || metadata["external_id"] == "" ||+ stored["datasource_id"] != metadata["datasource_id"] || stored["external_id"] != metadata["external_id"] {+ return nil, fmt.Errorf("datasource knowledge identity changed")+ }+ if err := s.validateKnowledgeTagIDs(plan.ctx, previous.TenantID, previous.KnowledgeBaseID, tagIDs); err != nil {+ return nil, err+ }+ replacement := &knowledgeReplacement{previous: previous, metadata: metadata, tagIDs: tagIDs}+ var created *types.Knowledge+ if len(item.Content) > 0 {+ file, buildErr := bytesToFileHeader(item.Content, item.FileName)+ if buildErr != nil {+ return nil, fmt.Errorf("build file header: %w", buildErr)+ }+ if err := ValidateJSONUploadContent(item.FileName, file); err != nil {+ return nil, err+ }+ created, err = s.createKnowledgeFromFile(plan.ctx, previous.KnowledgeBaseID, file, metadata,+ nil, item.FileName, tagIDs, channel, nil, replacement)+ } else {+ created, err = s.createKnowledgeFromURL(plan.ctx, previous.KnowledgeBaseID, item.URL, item.FileName,+ "", nil, item.Title, tagIDs, channel, nil, replacement)+ }+ if !replacement.committed {+ return created, err+ }++ // Never run the delete cascade before the INSERT commits: rollback cannot+ // restore an object-store file, vector index, graph or wiki contribution.+ // The old row is already soft-deleted, so do not mark/save it again (which+ // could resurrect it). Keep the tombstone if external cleanup fails.+ plan.rowsDeleted = true+ plan.ctx = context.WithoutCancel(plan.ctx)+ if cleanupErr := s.executeKnowledgeDelete(plan, true); cleanupErr != nil {+ logger.Warnf(plan.ctx, "Replacement committed; failed to clean retired knowledge %s: %v",+ knowledgeID, cleanupErr)+ } else if cleanupErr := s.repo.HardDeleteKnowledge(plan.ctx, previous.TenantID, knowledgeID); cleanupErr != nil {+ logger.Warnf(plan.ctx, "Replacement committed; failed to remove retired knowledge %s: %v",+ knowledgeID, cleanupErr)+ }+ return created, err+}++// Replacement tags were inserted with the row transaction. Normal uploads keep+// their existing validation/persistence behavior.+func (s *knowledgeService) attachCreationTags(ctx context.Context, knowledge *types.Knowledge,+ tagIDs []string, replacement *knowledgeReplacement,+) error {+ if replacement == nil {+ return s.setAndAttachKnowledgeTags(ctx, knowledge.TenantID, knowledge.KnowledgeBaseID, knowledge, tagIDs)+ }+ s.attachTagsToKnowledge(ctx, knowledge)+ return nil+}++// Retirement used to free this storage before creation checked quota. Preserve+// that allowance without changing the shared tenant snapshot before commit.+func knowledgeCreationStorageUsed(tenant *types.Tenant, replacement *knowledgeReplacement) int64 {+ used := tenant.StorageUsed+ if replacement != nil {+ used -= replacement.previous.StorageSize+ }+ return used+}diff --git a/internal/application/service/knowledge_datasource_replace_test.go b/internal/application/service/knowledge_datasource_replace_test.gonew file mode 100644index 000000000..76a1850a8--- /dev/null+++ b/internal/application/service/knowledge_datasource_replace_test.go@@ -0,0 +1,212 @@+package service++import (+ "context"+ "encoding/json"+ "errors"+ "path/filepath"+ "testing"++ "github.com/Tencent/WeKnora/internal/application/repository"+ "github.com/Tencent/WeKnora/internal/types"+ "github.com/Tencent/WeKnora/internal/types/interfaces"+ "github.com/hibiken/asynq"+ "github.com/stretchr/testify/require"+ "gorm.io/driver/sqlite"+ "gorm.io/gorm"+)++type datasourceReplacementChunks struct {+ replaceFileChunks+ deleted int+ err error+}++func (c *datasourceReplacementChunks) DeleteByKnowledgeList(context.Context, uint64, []string) error {+ c.deleted+++ return c.err+}++type datasourceReplacementTenant struct{ interfaces.TenantRepository }++func (datasourceReplacementTenant) AdjustStorageUsed(context.Context, uint64, int64) error {+ return nil+}++type datasourceReplacementQueue struct {+ t *testing.T+ repo interfaces.KnowledgeRepository+ oldID string+ queued int+}++func (q *datasourceReplacementQueue) Enqueue(task *asynq.Task, _ ...asynq.Option) (*asynq.TaskInfo, error) {+ if task.Type() == types.TypeDocumentProcess {+ var payload types.DocumentProcessPayload+ require.NoError(q.t, json.Unmarshal(task.Payload(), &payload))+ _, err := q.repo.GetKnowledgeByID(context.Background(), 7, q.oldID)+ require.Error(q.t, err, "retirement must commit before a worker can start")+ created, err := q.repo.GetKnowledgeByID(context.Background(), 7, payload.KnowledgeID)+ require.NoError(q.t, err, "replacement must commit before a worker can start")+ require.Equal(q.t, "ds-1", created.GetMetadata()["datasource_id"])+ require.Equal(q.t, "doc-1", created.GetMetadata()["external_id"])+ q.queued+++ }+ return &asynq.TaskInfo{ID: "task-1", Queue: types.QueueDefault}, nil+}++func datasourceReplacementFixture(t *testing.T) (+ *replaceFileHarness, *gorm.DB, *datasourceReplacementChunks, *datasourceReplacementQueue,+) {+ t.Helper()+ h := newReplaceFileHarness(t)+ db, err := gorm.Open(sqlite.Open(filepath.Join(t.TempDir(), "datasource.db")), &gorm.Config{})+ require.NoError(t, err)+ sqlDB, err := db.DB()+ require.NoError(t, err)+ t.Cleanup(func() { require.NoError(t, sqlDB.Close()) })+ require.NoError(t, db.AutoMigrate(&types.Knowledge{}, &types.KnowledgeTag{}, &types.KnowledgeTagRelation{}))+ repo := repository.NewKnowledgeRepository(db)+ h.original.Metadata = types.JSON(`{"datasource_id":"ds-1","external_id":"doc-1"}`)+ require.NoError(t, repo.CreateKnowledge(h.ctx, &h.original))+ h.svc.repo = repo+ chunks := &datasourceReplacementChunks{}+ h.svc.chunkRepo = chunks+ h.svc.tenantRepo = datasourceReplacementTenant{}+ queue := &datasourceReplacementQueue{t: t, repo: repo, oldID: h.original.ID}+ h.svc.task = queue+ return h, db, chunks, queue+}++func TestDataSourceReplacementPreservesOldResourcesOnFailure(t *testing.T) {+ for _, stage := range []string{"validation", "json", "storage", "insert"} {+ t.Run(stage, func(t *testing.T) {+ h, db, chunks, queue := datasourceReplacementFixture(t)+ item := &types.FetchedItem{ExternalID: "doc-1", Content: []byte("new content"), FileName: "doc.md"}+ switch stage {+ case "validation":+ item.FileName = "doc.unsupported"+ case "json":+ item.FileName, item.Content = "doc.json", []byte("invalid JSON")+ case "storage":+ h.store.saveErr = errors.New("storage unavailable")+ case "insert":+ require.NoError(t, db.Exec(`CREATE TRIGGER fail_replacement BEFORE INSERT ON knowledges+ BEGIN SELECT RAISE(FAIL, 'forced insert failure'); END`).Error)+ }+ ds := &DataSourceService{knowledgeService: h.svc}+ _, err := ds.ingestItem(h.ctx,+ &types.DataSource{ID: "ds-1", TenantID: 7, KnowledgeBaseID: "kb-1"}, item, nil)+ require.Error(t, err)+ old, err := h.svc.repo.GetKnowledgeByID(h.ctx, 7, h.original.ID)+ require.NoError(t, err)+ require.Equal(t, h.original.FilePath, old.FilePath)+ require.Equal(t, types.ParseStatusCompleted, old.ParseStatus)+ require.Zero(t, chunks.deleted)+ require.Zero(t, queue.queued)+ for _, path := range h.store.deleted {+ require.NotEqual(t, h.original.FilePath, path)+ }+ if stage == "insert" {+ require.Equal(t, []string{"new/file.md"}, h.store.deleted, "discard only the staged new file")+ }+ })+ }+}++func TestDataSourceReplacementCommitsBeforeEnqueueAndCleanup(t *testing.T) {+ for _, kind := range []string{"file", "url", "file_url"} {+ t.Run(kind, func(t *testing.T) {+ h, _, chunks, queue := datasourceReplacementFixture(t)+ tenant := h.ctx.Value(types.TenantInfoContextKey).(*types.Tenant)+ tenant.StorageQuota, tenant.StorageUsed = 100, 100+ require.NoError(t, h.svc.repo.UpdateKnowledgeColumn(h.ctx, h.original.ID, "storage_size", int64(10)))+ item := &types.FetchedItem{ExternalID: "doc-1", FileName: "doc.md"}+ switch kind {+ case "file":+ item.Content = []byte("new content")+ case "url":+ item.URL, item.FileName = "https://example.com/doc", ""+ case "file_url":+ item.URL = "https://example.com/doc.md"+ }+ ds := &DataSourceService{knowledgeService: h.svc}+ updated, err := ds.ingestItem(h.ctx,+ &types.DataSource{ID: "ds-1", TenantID: 7, KnowledgeBaseID: "kb-1"}, item, nil)+ require.NoError(t, err)+ require.True(t, updated)+ require.Equal(t, 1, queue.queued)+ require.Equal(t, 1, chunks.deleted)+ require.Contains(t, h.store.deleted, h.original.FilePath)+ created, err := h.svc.repo.FindByDataSourceExternalID(h.ctx, 7, "kb-1", "ds-1", "doc-1")+ require.NoError(t, err)+ require.NotNil(t, created)+ require.Equal(t, kind, created.Type)+ require.NotEqual(t, h.original.ID, created.ID)+ })+ }+}++func TestDataSourceReplacementCleanupFailureKeepsCommittedReplacement(t *testing.T) {+ h, db, chunks, queue := datasourceReplacementFixture(t)+ chunks.err = errors.New("index cleanup unavailable")+ ds := &DataSourceService{knowledgeService: h.svc}+ _, err := ds.ingestItem(h.ctx, &types.DataSource{ID: "ds-1", TenantID: 7, KnowledgeBaseID: "kb-1"},+ &types.FetchedItem{ExternalID: "doc-1", Content: []byte("new content"), FileName: "doc.md"}, nil)+ require.NoError(t, err)+ require.Equal(t, 1, queue.queued)+ require.NotContains(t, h.store.deleted, h.original.FilePath)+ var retired types.Knowledge+ require.NoError(t, db.Unscoped().First(&retired, "id = ?", h.original.ID).Error)+ require.True(t, retired.DeletedAt.Valid)+ created, err := h.svc.repo.FindByDataSourceExternalID(h.ctx, 7, "kb-1", "ds-1", "doc-1")+ require.NoError(t, err)+ require.NotNil(t, created)+}++type uncertainReplacementRepo struct {+ interfaces.KnowledgeRepository+ committed bool+ failReads bool+}++func (r *uncertainReplacementRepo) ReplaceKnowledge(+ ctx context.Context, before, after *types.Knowledge, tags []string,+) error {+ if err := r.KnowledgeRepository.ReplaceKnowledge(ctx, before, after, tags); err != nil {+ return err+ }+ r.committed = true+ return errors.New("commit acknowledgement lost")+}++func (r *uncertainReplacementRepo) GetKnowledgeByID(+ ctx context.Context, tenant uint64, id string,+) (*types.Knowledge, error) {+ if r.committed && r.failReads {+ return nil, errors.New("database unavailable")+ }+ return r.KnowledgeRepository.GetKnowledgeByID(ctx, tenant, id)+}++func TestDataSourceReplacementRetainsFileOnUncertainCommit(t *testing.T) {+ for _, failReads := range []bool{false, true} {+ t.Run(map[bool]string{false: "commit confirmed", true: "reads unavailable"}[failReads], func(t *testing.T) {+ h, _, chunks, queue := datasourceReplacementFixture(t)+ h.svc.repo = &uncertainReplacementRepo{KnowledgeRepository: h.svc.repo, failReads: failReads}+ ds := &DataSourceService{knowledgeService: h.svc}+ _, err := ds.ingestItem(h.ctx, &types.DataSource{ID: "ds-1", TenantID: 7, KnowledgeBaseID: "kb-1"},+ &types.FetchedItem{ExternalID: "doc-1", Content: []byte("new content"), FileName: "doc.md"}, nil)+ require.NotContains(t, h.store.deleted, "new/file.md", "a committed row may reference this file")+ if failReads {+ require.ErrorContains(t, err, "commit acknowledgement lost")+ require.Zero(t, queue.queued)+ require.Zero(t, chunks.deleted)+ } else {+ require.NoError(t, err)+ require.Equal(t, 1, queue.queued)+ require.Equal(t, 1, chunks.deleted)+ }+ })+ }+}diff --git a/internal/application/service/knowledge_delete.go b/internal/application/service/knowledge_delete.goindex 4761d5693..92d6f6423 100644--- a/internal/application/service/knowledge_delete.go+++ b/internal/application/service/knowledge_delete.go@@ -488,10 +488,12 @@ func (s *knowledgeService) executeKnowledgeDelete(plan *knowledgeDeletePlan, sinprev := knowledge.ParseStatusbefore := *knowledgeknowledge.ParseStatus = types.ParseStatusDeleting- if err := s.repo.UpdateKnowledgeForTransfer(ctx, &before, knowledge); err != nil {- logger.GetLogger(ctx).WithField("error", err).WithField("knowledge_id", knowledge.ID).- Errorf("DeleteKnowledgeList failed to mark as deleting")- return err+ if !plan.rowsDeleted {+ if err := s.repo.UpdateKnowledgeForTransfer(ctx, &before, knowledge); err != nil {+ logger.GetLogger(ctx).WithField("error", err).WithField("knowledge_id", knowledge.ID).+ Errorf("DeleteKnowledgeList failed to mark as deleting")+ return err+ }}if prev == types.ParseStatusPending || prev == types.ParseStatusProcessing {inFlightIDs = append(inFlightIDs, knowledge.ID)@@ -615,8 +617,10 @@ func (s *knowledgeService) executeKnowledgeDelete(plan *knowledgeDeletePlan, sin// gone avoids "file missing but row present" zombies that break reparse /// re-delete when an earlier cleanup step failed (issue #2192). A failure below// only orphans storage.- if err := s.repo.DeleteKnowledgeList(ctx, tenantInfo.ID, ids); err != nil {- return err+ if !plan.rowsDeleted {+ if err := s.repo.DeleteKnowledgeList(ctx, tenantInfo.ID, ids); err != nil {+ return err+ }}storageAdjust := int64(0)diff --git a/internal/application/service/knowledge_delete_plan.go b/internal/application/service/knowledge_delete_plan.goindex f08f17aea..c3b2546f4 100644--- a/internal/application/service/knowledge_delete_plan.go+++ b/internal/application/service/knowledge_delete_plan.go@@ -72,12 +72,14 @@ func deleteReferencedKnowledge(}type knowledgeDeletePlan struct {- ctx context.Context- ids []string- knowledge []*types.Knowledge- kbs map[string]*types.KnowledgeBase- files map[string]interfaces.FileService- imageInfo []interfaces.ChunkImageInfo+ // A datasource replacement already soft-deleted the old row atomically.+ rowsDeleted bool+ ctx context.Context+ ids []string+ knowledge []*types.Knowledge+ kbs map[string]*types.KnowledgeBase+ files map[string]interfaces.FileService+ imageInfo []interfaces.ChunkImageInfo}func (s *knowledgeService) planKnowledgeDelete(ctx context.Context, ids []string) (*knowledgeDeletePlan, error) {diff --git a/internal/types/interfaces/knowledge.go b/internal/types/interfaces/knowledge.goindex 32448da3b..361674c5c 100644--- a/internal/types/interfaces/knowledge.go+++ b/internal/types/interfaces/knowledge.go@@ -11,6 +11,11 @@ import (// KnowledgeService defines the interface for knowledge services.type KnowledgeService interface {+ // ReplaceKnowledgeFromDataSource atomically retires the previous row and inserts+ // its replacement before enqueueing parsing or deleting the previous resources.+ ReplaceKnowledgeFromDataSource(ctx context.Context, knowledgeID string, item *types.FetchedItem,+ metadata map[string]string, tagIDs []string, channel string) (*types.Knowledge, error)+// CreateKnowledgeFromFile creates knowledge from a file.// channel identifies the ingestion channel (e.g. "web", "api", "wechat"); empty defaults to "web".CreateKnowledgeFromFile(@@ -242,6 +247,11 @@ type KnowledgeService interface {// KnowledgeRepository defines the interface for knowledge repositories.type KnowledgeRepository interface {+ // ReplaceKnowledge soft-deletes before and inserts after with its tag relations+ // in one short transaction.+ // A stale before snapshot is rejected without deleting or inserting either row.+ ReplaceKnowledge(ctx context.Context, before, after *types.Knowledge, tagIDs []string) error+CreateKnowledge(ctx context.Context, knowledge *types.Knowledge) errorGetKnowledgeByID(ctx context.Context, tenantID uint64, id string) (*types.Knowledge, error)// GetKnowledgeByIDOnly returns knowledge by ID without tenant filter (for permission resolution).diff --git a/internal/types/knowledge.go b/internal/types/knowledge.goindex d76952939..aad0fbf8a 100644--- a/internal/types/knowledge.go+++ b/internal/types/knowledge.go@@ -547,6 +547,8 @@ func (k *Knowledge) SetProcessOverrides(o *KnowledgeProcessOverrides) error {// KnowledgeCheckParams defines parameters used to check if knowledge already exists.type KnowledgeCheckParams struct {+ // ExcludeKnowledgeID ignores only the row being replaced during duplicate checks.+ ExcludeKnowledgeID string// File parametersFileName string// FileType scopes file-hash deduplication; callers checking file uploads should set it.