Skip to content

Open-source LLM knowledge platform: turn raw documents into a queryable RAG, an autonomous reasoning agent, and a self-maintaining Wiki.

fix(datasource): make knowledge replacement transactional (#4473)

younggglcy98fb3f91c223
diff --git a/internal/application/repository/knowledge.go b/internal/application/repository/knowledge.go
index 9cfd90fd1..e3bd55cb5 100644
--- a/internal/application/repository/knowledge.go
+++ b/internal/application/repository/knowledge.go
@@ -434,6 +434,10 @@ func (r *knowledgeRepository) CheckKnowledgeExists(
Where("tenant_id = ? AND knowledge_base_id = ? AND parse_status NOT IN ?",
tenantID, kbID, []string{"failed", "deleting"})
+ if params.ExcludeKnowledgeID != "" {
+ query = query.Where("id <> ?", params.ExcludeKnowledgeID)
+ }
+
switch params.Type {
case "file":
if params.DataSourceID != "" && params.ExternalID != "" {
diff --git a/internal/application/repository/knowledge_replace.go b/internal/application/repository/knowledge_replace.go
new file mode 100644
index 000000000..dec694ef2
--- /dev/null
+++ b/internal/application/repository/knowledge_replace.go
@@ -0,0 +1,42 @@
+package repository
+
+import (
+ "context"
+ "fmt"
+
+ "github.com/Tencent/WeKnora/internal/types"
+ "gorm.io/gorm"
+)
+
+// ReplaceKnowledge keeps the previous row live if inserting its replacement
+// fails. No file, index, queue or other remote operation runs in this transaction.
+func (r *knowledgeRepository) ReplaceKnowledge(
+ ctx context.Context, before, after *types.Knowledge, tagIDs []string,
+) error {
+ if before == nil || after == nil || before.ID == "" || after.ID == "" || before.ID == after.ID ||
+ before.TenantID != after.TenantID || before.KnowledgeBaseID != after.KnowledgeBaseID {
+ return fmt.Errorf("invalid knowledge replacement")
+ }
+ return r.db.WithContext(ctx).Transaction(func(tx *gorm.DB) error {
+ // The primary-key lookup locks only the source row. Comparing the snapshot
+ // prevents a second sync from replacing a row another writer has retired
+ // or changed while this sync was preparing its replacement.
+ result := tx.Where("id = ? AND tenant_id = ? AND knowledge_base_id = ? AND updated_at = ?",
+ before.ID, before.TenantID, before.KnowledgeBaseID, before.UpdatedAt).
+ Delete(&types.Knowledge{})
+ if result.Error != nil {
+ return result.Error
+ }
+ if result.RowsAffected != 1 {
+ return fmt.Errorf("knowledge changed during replacement: %s", before.ID)
+ }
+ repo := &knowledgeRepository{db: tx}
+ if err := repo.CreateKnowledge(ctx, after); err != nil {
+ return err
+ }
+ if len(tagIDs) > 0 {
+ return repo.SetKnowledgeTags(ctx, after.ID, tagIDs)
+ }
+ return nil
+ })
+}
diff --git a/internal/application/repository/knowledge_replace_postgres_test.go b/internal/application/repository/knowledge_replace_postgres_test.go
new file mode 100644
index 000000000..592a5e72d
--- /dev/null
+++ b/internal/application/repository/knowledge_replace_postgres_test.go
@@ -0,0 +1,74 @@
+package repository
+
+import (
+ "context"
+ "os"
+ "strings"
+ "testing"
+
+ "github.com/Tencent/WeKnora/internal/types"
+ "github.com/google/uuid"
+ "github.com/stretchr/testify/require"
+ "gorm.io/driver/postgres"
+ "gorm.io/gorm"
+)
+
+func TestReplaceKnowledgePostgresRollbackAndCommit(t *testing.T) {
+ dsn := os.Getenv("WEKNORA_REPOSITORY_TEST_POSTGRES_DSN")
+ if dsn == "" {
+ t.Skip("set WEKNORA_REPOSITORY_TEST_POSTGRES_DSN for PostgreSQL replacement tests")
+ }
+ admin, err := gorm.Open(postgres.Open(dsn), &gorm.Config{})
+ require.NoError(t, err)
+ adminSQL, err := admin.DB()
+ require.NoError(t, err)
+ schema := "knowledge_replace_" + strings.ReplaceAll(uuid.NewString(), "-", "")
+ require.NoError(t, admin.Exec("CREATE SCHEMA "+schema).Error)
+ t.Cleanup(func() {
+ require.NoError(t, admin.Exec("DROP SCHEMA "+schema+" CASCADE").Error)
+ require.NoError(t, adminSQL.Close())
+ })
+ separator := " "
+ if strings.Contains(dsn, "://") {
+ separator = "?"
+ if strings.Contains(dsn, "?") {
+ separator = "&"
+ }
+ }
+ db, err := gorm.Open(postgres.Open(dsn+separator+"search_path="+schema), &gorm.Config{})
+ require.NoError(t, err)
+ sqlDB, err := db.DB()
+ require.NoError(t, err)
+ t.Cleanup(func() { require.NoError(t, sqlDB.Close()) })
+ require.NoError(t, db.AutoMigrate(&types.Knowledge{}, &types.KnowledgeTagRelation{}))
+ ctx := context.Background()
+ repo := NewKnowledgeRepository(db)
+ old := &types.Knowledge{ID: "old", TenantID: 7, KnowledgeBaseID: "kb", Type: "file"}
+ occupied := &types.Knowledge{ID: "occupied", TenantID: 7, KnowledgeBaseID: "kb", Type: "file"}
+ require.NoError(t, repo.CreateKnowledge(ctx, old))
+ require.NoError(t, repo.CreateKnowledge(ctx, occupied))
+ old, err = repo.GetKnowledgeByID(ctx, 7, old.ID)
+ require.NoError(t, err)
+ require.NoError(t, repo.SetKnowledgeTags(ctx, old.ID, []string{"old-tag"}))
+ replacement := &types.Knowledge{ID: occupied.ID, TenantID: 7, KnowledgeBaseID: "kb", Type: "file"}
+ require.Error(t, repo.ReplaceKnowledge(ctx, old, replacement, []string{"new-tag"}))
+ retained, err := repo.GetKnowledgeByID(ctx, 7, old.ID)
+ require.NoError(t, err, "failed insertion must roll back the retirement")
+ require.False(t, retained.DeletedAt.Valid)
+ var tags []types.KnowledgeTagRelation
+ require.NoError(t, db.Where("knowledge_id = ?", old.ID).Find(&tags).Error)
+ require.Len(t, tags, 1)
+ require.Equal(t, "old-tag", tags[0].TagID)
+ replacement.ID = "new"
+ require.NoError(t, repo.ReplaceKnowledge(ctx, old, replacement, []string{"new-tag"}))
+ _, err = repo.GetKnowledgeByID(ctx, 7, old.ID)
+ require.Error(t, err)
+ _, err = repo.GetKnowledgeByID(ctx, 7, replacement.ID)
+ require.NoError(t, err)
+ tags = nil
+ require.NoError(t, db.Where("knowledge_id = ?", replacement.ID).Find(&tags).Error)
+ require.Len(t, tags, 1)
+ require.Equal(t, "new-tag", tags[0].TagID)
+ replacement.ID = "stale-replacement"
+ require.ErrorContains(t, repo.ReplaceKnowledge(ctx, old, replacement, nil), "knowledge changed")
+}
diff --git a/internal/application/repository/knowledge_replace_test.go b/internal/application/repository/knowledge_replace_test.go
new file mode 100644
index 000000000..8e39ca1d1
--- /dev/null
+++ b/internal/application/repository/knowledge_replace_test.go
@@ -0,0 +1,160 @@
+package repository
+
+import (
+ "context"
+ "path/filepath"
+ "testing"
+ "time"
+
+ "github.com/Tencent/WeKnora/internal/types"
+ "github.com/Tencent/WeKnora/internal/types/interfaces"
+ "github.com/stretchr/testify/require"
+ "gorm.io/driver/sqlite"
+ "gorm.io/gorm"
+)
+
+func replacementFixture(t *testing.T) (*gorm.DB, interfaces.KnowledgeRepository, *types.Knowledge, *types.Knowledge) {
+ t.Helper()
+ db, err := gorm.Open(sqlite.Open(filepath.Join(t.TempDir(), "replacement.db")), &gorm.Config{})
+ require.NoError(t, err)
+ sqlDB, err := db.DB()
+ require.NoError(t, err)
+ t.Cleanup(func() { require.NoError(t, sqlDB.Close()) })
+ require.NoError(t, db.AutoMigrate(&types.Knowledge{}, &types.KnowledgeTagRelation{}))
+ repo := NewKnowledgeRepository(db)
+ before := &types.Knowledge{
+ ID: "old", TenantID: 7, KnowledgeBaseID: "kb-1", Type: "file",
+ FilePath: "old.md", ParseStatus: types.ParseStatusCompleted, EnableStatus: "enabled",
+ Metadata: types.JSON(`{"datasource_id":"ds-1","external_id":"doc-1"}`),
+ }
+ require.NoError(t, repo.CreateKnowledge(context.Background(), before))
+ before, err = repo.GetKnowledgeByID(context.Background(), 7, before.ID)
+ require.NoError(t, err)
+ after := &types.Knowledge{
+ ID: "new", TenantID: 7, KnowledgeBaseID: "kb-1", Type: "file",
+ FilePath: "new.md", ParseStatus: types.ParseStatusPending, Metadata: before.Metadata,
+ }
+ return db, repo, before, after
+}
+
+func TestReplaceKnowledgeRollsBackRetirementWhenInsertFails(t *testing.T) {
+ db, repo, before, after := replacementFixture(t)
+ require.NoError(t, db.Exec(`CREATE TRIGGER fail_replacement BEFORE INSERT ON knowledges
+ WHEN NEW.id = 'new' BEGIN SELECT RAISE(FAIL, 'forced insert failure'); END`).Error)
+
+ err := repo.ReplaceKnowledge(context.Background(), before, after, nil)
+ require.ErrorContains(t, err, "forced insert failure")
+ persisted, err := repo.GetKnowledgeByID(context.Background(), 7, before.ID)
+ require.NoError(t, err)
+ require.Equal(t, "old.md", persisted.FilePath)
+ require.Equal(t, types.ParseStatusCompleted, persisted.ParseStatus)
+ require.False(t, persisted.DeletedAt.Valid)
+ _, err = repo.GetKnowledgeByID(context.Background(), 7, after.ID)
+ require.Error(t, err)
+}
+
+func TestReplaceKnowledgeCommitsBothRowsAndRejectsStaleReplacement(t *testing.T) {
+ db, repo, before, after := replacementFixture(t)
+ require.NoError(t, repo.ReplaceKnowledge(context.Background(), before, after, nil))
+ _, err := repo.GetKnowledgeByID(context.Background(), 7, before.ID)
+ require.Error(t, err)
+ persisted, err := repo.GetKnowledgeByID(context.Background(), 7, after.ID)
+ require.NoError(t, err)
+ require.Equal(t, after.Metadata, persisted.Metadata)
+ var retired types.Knowledge
+ require.NoError(t, db.Unscoped().First(&retired, "id = ?", before.ID).Error)
+ require.True(t, retired.DeletedAt.Valid, "retain a tombstone until resource cleanup succeeds")
+
+ after.ID = "racing-replacement"
+ require.ErrorContains(t, repo.ReplaceKnowledge(context.Background(), before, after, nil), "knowledge changed")
+ var count int64
+ require.NoError(t, db.Model(&types.Knowledge{}).Count(&count).Error)
+ require.EqualValues(t, 1, count)
+}
+
+func TestReplaceKnowledgeRejectsChangedOrForeignSnapshot(t *testing.T) {
+ for _, change := range []string{"updated", "tenant", "kb", "same-id", "delete-failure"} {
+ t.Run(change, func(t *testing.T) {
+ db, repo, before, after := replacementFixture(t)
+ switch change {
+ case "updated":
+ require.NoError(t, repo.UpdateKnowledgeColumn(context.Background(), before.ID,
+ "updated_at", before.UpdatedAt.Add(time.Second)))
+ case "tenant":
+ after.TenantID++
+ case "kb":
+ after.KnowledgeBaseID = "kb-other"
+ case "same-id":
+ after.ID = before.ID
+ case "delete-failure":
+ require.NoError(t, db.Exec(`CREATE TRIGGER fail_retirement BEFORE UPDATE OF deleted_at ON knowledges
+ BEGIN SELECT RAISE(FAIL, 'forced delete failure'); END`).Error)
+ }
+ require.Error(t, repo.ReplaceKnowledge(context.Background(), before, after, nil))
+ _, err := repo.GetKnowledgeByID(context.Background(), 7, before.ID)
+ require.NoError(t, err)
+ var count int64
+ require.NoError(t, db.Unscoped().Model(&types.Knowledge{}).Count(&count).Error)
+ require.EqualValues(t, 1, count)
+ })
+ }
+}
+
+func TestCheckKnowledgeExistsExcludesOnlyReplacementSource(t *testing.T) {
+ for _, kind := range []string{"file", "url"} {
+ t.Run(kind, func(t *testing.T) {
+ _, repo, before, _ := replacementFixture(t)
+ require.NoError(t, repo.UpdateKnowledgeColumns(context.Background(), before.ID,
+ map[string]interface{}{
+ "type": kind, "file_hash": "same-hash", "file_type": "md", "source": "https://example.com/doc",
+ }))
+ params := &types.KnowledgeCheckParams{
+ Type: kind, FileHash: "same-hash", FileType: "md",
+ URL: "https://example.com/doc", ExcludeKnowledgeID: before.ID,
+ }
+ exists, _, err := repo.CheckKnowledgeExists(context.Background(), 7, "kb-1", params)
+ require.NoError(t, err)
+ require.False(t, exists)
+ other := &types.Knowledge{
+ ID: "other", TenantID: 7, KnowledgeBaseID: "kb-1", Type: kind,
+ FileHash: "same-hash", FileType: "md", Source: params.URL, ParseStatus: types.ParseStatusCompleted,
+ }
+ require.NoError(t, repo.CreateKnowledge(context.Background(), other))
+ exists, duplicate, err := repo.CheckKnowledgeExists(context.Background(), 7, "kb-1", params)
+ require.NoError(t, err)
+ require.True(t, exists)
+ require.Equal(t, other.ID, duplicate.ID)
+ })
+ }
+}
+
+func TestReplaceKnowledgeTagsCommitOrRollbackWithRows(t *testing.T) {
+ for _, fail := range []bool{false, true} {
+ t.Run(map[bool]string{false: "commit", true: "tag failure"}[fail], func(t *testing.T) {
+ db, repo, before, after := replacementFixture(t)
+ require.NoError(t, repo.SetKnowledgeTags(context.Background(), before.ID, []string{"old-tag"}))
+ if fail {
+ require.NoError(t, db.Exec(`CREATE TRIGGER fail_new_tags BEFORE INSERT ON knowledge_tag_relations
+ WHEN NEW.knowledge_id = 'new' BEGIN SELECT RAISE(FAIL, 'forced tag failure'); END`).Error)
+ }
+ err := repo.ReplaceKnowledge(context.Background(), before, after, []string{"new-tag"})
+ if fail {
+ require.ErrorContains(t, err, "forced tag failure")
+ _, err = repo.GetKnowledgeByID(context.Background(), 7, before.ID)
+ require.NoError(t, err)
+ _, err = repo.GetKnowledgeByID(context.Background(), 7, after.ID)
+ require.Error(t, err)
+ } else {
+ require.NoError(t, err)
+ }
+ var relations []types.KnowledgeTagRelation
+ require.NoError(t, db.Where("knowledge_id = ?", after.ID).Find(&relations).Error)
+ if fail {
+ require.Empty(t, relations)
+ } else {
+ require.Len(t, relations, 1)
+ require.Equal(t, "new-tag", relations[0].TagID)
+ }
+ })
+ }
+}
diff --git a/internal/application/service/datasource_atomic_replace_test.go b/internal/application/service/datasource_atomic_replace_test.go
new file mode 100644
index 000000000..e92e539b7
--- /dev/null
+++ b/internal/application/service/datasource_atomic_replace_test.go
@@ -0,0 +1,78 @@
+package service
+
+import (
+ "context"
+ "errors"
+ "testing"
+
+ "github.com/Tencent/WeKnora/internal/types"
+ "github.com/Tencent/WeKnora/internal/types/interfaces"
+ "github.com/stretchr/testify/require"
+)
+
+type atomicIngestRepo struct {
+ interfaces.KnowledgeRepository
+ row *types.Knowledge
+ err error
+}
+
+func (r *atomicIngestRepo) FindByDataSourceExternalID(
+ context.Context, uint64, string, string, string,
+) (*types.Knowledge, error) {
+ return r.row, r.err
+}
+
+func (r *atomicIngestRepo) HardDeleteKnowledge(context.Context, uint64, string) error {
+ r.row = nil
+ return nil
+}
+
+type atomicIngestService struct {
+ sweepFakeKS
+ replacements int
+}
+
+func (s *atomicIngestService) ReplaceKnowledgeFromDataSource(
+ context.Context, string, *types.FetchedItem, map[string]string, []string, string,
+) (*types.Knowledge, error) {
+ s.replacements++
+ return nil, s.createErr
+}
+
+func (s *atomicIngestService) CreateKnowledgeFromURL(
+ context.Context, string, string, string, string, *bool, string, []string, string, *types.KnowledgeProcessOverrides,
+) (*types.Knowledge, error) {
+ s.events = append(s.events, "create-url")
+ return nil, s.createErr
+}
+
+func TestIngestItemReplacementFailureKeepsPreviousKnowledge(t *testing.T) {
+ for _, item := range []*types.FetchedItem{
+ {ExternalID: "doc-1", Content: []byte("new content"), FileName: "doc.md"},
+ {ExternalID: "doc-1", URL: "https://example.com/doc"},
+ } {
+ t.Run(item.FileName+item.URL, func(t *testing.T) {
+ old := &types.Knowledge{ID: "old-knowledge", Type: "file"}
+ repo := &atomicIngestRepo{row: old}
+ ks := &atomicIngestService{sweepFakeKS: sweepFakeKS{repo: repo, createErr: errors.New("insert failed")}}
+ s := &DataSourceService{knowledgeService: ks}
+ ds := &types.DataSource{ID: "ds-1", TenantID: 7, KnowledgeBaseID: "kb-1"}
+
+ _, err := s.ingestItem(context.Background(), ds, item, nil)
+ require.Error(t, err)
+ require.Same(t, old, repo.row, "a failed replacement must retain the previous row")
+ require.Empty(t, ks.deleted, "old resources must not be deleted before replacement commits")
+ require.Equal(t, 1, ks.replacements)
+ })
+ }
+}
+
+func TestIngestItemLookupFailureDoesNotCreateDuplicate(t *testing.T) {
+ repo := &atomicIngestRepo{err: errors.New("lookup failed")}
+ ks := &atomicIngestService{sweepFakeKS: sweepFakeKS{repo: repo}}
+ s := &DataSourceService{knowledgeService: ks}
+ _, err := s.ingestItem(context.Background(), &types.DataSource{ID: "ds-1"},
+ &types.FetchedItem{ExternalID: "doc-1", Content: []byte("content"), FileName: "doc.md"}, nil)
+ require.ErrorContains(t, err, "lookup failed")
+ require.Empty(t, ks.events)
+}
diff --git a/internal/application/service/datasource_conflict_skip_test.go b/internal/application/service/datasource_conflict_skip_test.go
index ca509c5e0..1300e7561 100644
--- a/internal/application/service/datasource_conflict_skip_test.go
+++ b/internal/application/service/datasource_conflict_skip_test.go
@@ -60,12 +60,24 @@ func (conflictSkipConnector) FetchIncremental(
return items, nil, err
}
+type conflictSkipKnowledgeService struct {
+ *sweepFakeKS
+ replacements int
+}
+
+func (s *conflictSkipKnowledgeService) ReplaceKnowledgeFromDataSource(
+ _ context.Context, _ string, _ *types.FetchedItem, _ map[string]string, _ []string, _ string,
+) (*types.Knowledge, error) {
+ s.replacements++
+ return &types.Knowledge{ID: "new-knowledge"}, nil
+}
+
// newConflictSkipHarness wires a full-sync ProcessSync fixture whose data
// source carries the given conflict strategy and whose KB already contains the
// synced item under the same external_id.
func newConflictSkipHarness(
t *testing.T, strategy string,
-) (*DataSourceService, *types.DataSource, *deletionLookupKnowledgeRepo, *sweepFakeKS, *types.SyncLog) {
+) (*DataSourceService, *types.DataSource, *deletionLookupKnowledgeRepo, *conflictSkipKnowledgeService, *types.SyncLog) {
t.Helper()
configJSON, err := (&types.DataSourceConfig{Type: conflictSkipConnectorType}).ToJSON()
require.NoError(t, err)
@@ -83,7 +95,7 @@ func newConflictSkipHarness(
}
existing := &types.Knowledge{ID: "knowledge-exists", FileName: "existing.md"}
repo := &deletionLookupKnowledgeRepo{knowledge: existing}
- ks := &sweepFakeKS{repo: repo}
+ ks := &conflictSkipKnowledgeService{sweepFakeKS: &sweepFakeKS{repo: repo}}
syncLog := &types.SyncLog{
ID: "log-conflict-skip",
DataSourceID: ds.ID,
@@ -134,6 +146,7 @@ func TestProcessSync_SkipStrategyKeepsExistingCountsSkipped(t *testing.T) {
log := runConflictSkipSync(t, svc, ds, syncLog)
assert.Empty(t, repo.hardDeleted, "skip strategy must not hard-delete the existing knowledge")
+ assert.Zero(t, ks.replacements)
assert.NotContains(t, ks.events, "delete:knowledge-exists")
assert.NotContains(t, ks.events, "create:existing.md")
assert.Equal(t, 0, log.ItemsCreated)
@@ -142,15 +155,15 @@ func TestProcessSync_SkipStrategyKeepsExistingCountsSkipped(t *testing.T) {
}
// Default strategy (overwrite) keeps the historical replace behavior: the
-// existing item is deleted and re-created, counted as updated.
+// existing item is replaced transactionally, counted as updated.
func TestProcessSync_OverwriteStrategyReplacesExisting(t *testing.T) {
svc, ds, repo, ks, syncLog := newConflictSkipHarness(t, types.ConflictStrategyOverwrite)
log := runConflictSkipSync(t, svc, ds, syncLog)
- assert.Equal(t, []string{"knowledge-exists"}, repo.hardDeleted)
- assert.Contains(t, ks.events, "delete:knowledge-exists")
- assert.Contains(t, ks.events, "create:existing.md")
+ assert.Empty(t, repo.hardDeleted)
+ assert.Empty(t, ks.events, "the caller must not delete or create outside the replacement transaction")
+ assert.Equal(t, 1, ks.replacements)
assert.Equal(t, 0, log.ItemsCreated)
assert.Equal(t, 1, log.ItemsUpdated)
assert.Equal(t, 0, log.ItemsSkipped)
diff --git a/internal/application/service/datasource_service.go b/internal/application/service/datasource_service.go
index f651201f5..660aaf5c6 100644
--- a/internal/application/service/datasource_service.go
+++ b/internal/application/service/datasource_service.go
@@ -1374,7 +1374,7 @@ func (s *DataSourceService) validateDataSourceConfig(ctx context.Context, ds *ty
}
// ingestItem writes a single FetchedItem into the knowledge base.
-// If a knowledge item with the same external_id already exists, it is deleted first (update = delete + re-create).
+// Existing datasource items are replaced atomically at the knowledge-row boundary.
//
// Routing logic:
// - Has Content bytes → CreateKnowledgeFromFile (走完整的文档解析 pipeline)
@@ -1412,18 +1412,16 @@ func (s *DataSourceService) ingestItem(ctx context.Context, ds *types.DataSource
metadata[k] = v
}
- // Check if a knowledge item with this external_id already exists → delete it first (update)
+ // Existing items are replaced through a short DB transaction. Validation and
+ // file storage must succeed before the previous row or resources are retired.
isUpdate := false
if item.ExternalID != "" {
repo := s.knowledgeService.GetRepository()
- // Scope the lookup to items owned by this data source so identical
- // external IDs from two data sources cannot collide or overwrite each
- // other during updates.
existing, err := repo.FindByDataSourceExternalID(ctx, ds.TenantID, ds.KnowledgeBaseID, ds.ID, item.ExternalID)
if err != nil {
- logger.Warnf(ctx, "failed to check existing knowledge for external_id=%s: %v", item.ExternalID, err)
- // Non-fatal: proceed with creation (may produce duplicate)
- } else if existing != nil {
+ return false, fmt.Errorf("check existing datasource knowledge: %w", err)
+ }
+ if existing != nil {
// Conflict strategy "skip" (跳过已存在): the item already exists in
// the KB, so leave it untouched and report the conflict as a skip
// via the shared DuplicateKnowledgeError path (Tencent/WeKnora#4640 —
@@ -1442,15 +1440,13 @@ func (s *DataSourceService) ingestItem(ctx context.Context, ds *types.DataSource
Knowledge: existing,
}
}
- logger.Infof(ctx, "found existing knowledge %s for external_id=%s, deleting for update", existing.ID, item.ExternalID)
- if err := s.knowledgeService.DeleteKnowledge(ctx, existing.ID); err != nil {
- logger.Warnf(ctx, "failed to delete existing knowledge %s: %v", existing.ID, err)
- } else {
- if herr := repo.HardDeleteKnowledge(ctx, ds.TenantID, existing.ID); herr != nil {
- logger.Warnf(ctx, "failed to hard-delete replaced knowledge %s: %v", existing.ID, herr)
- }
- isUpdate = true
+ _, err := s.knowledgeService.ReplaceKnowledgeFromDataSource(
+ ctx, existing.ID, item, metadata, tagIDs, channel)
+ var dupErr *types.DuplicateKnowledgeError
+ if err == nil || (errors.As(err, &dupErr) && dupIsSameNode(dupErr, item)) {
+ s.sweepStaleSubtree(ctx, ds, item)
}
+ return err == nil, err
}
}
diff --git a/internal/application/service/knowledge_create.go b/internal/application/service/knowledge_create.go
index 33adbfc71..3cd21ffd8 100644
--- a/internal/application/service/knowledge_create.go
+++ b/internal/application/service/knowledge_create.go
@@ -25,8 +25,19 @@ import (
// CreateKnowledgeFromFile creates a knowledge entry from an uploaded file
func (s *knowledgeService) CreateKnowledgeFromFile(ctx context.Context,
- kbID string, file *multipart.FileHeader, metadata map[string]string, enableMultimodel *bool, customFileName string, tagIDs []string, channel string,
+ kbID string, file *multipart.FileHeader, metadata map[string]string, enableMultimodel *bool,
+ customFileName string, tagIDs []string, channel string,
processOverrides *types.KnowledgeProcessOverrides,
+) (*types.Knowledge, error) {
+ return s.createKnowledgeFromFile(ctx, kbID, file, metadata, enableMultimodel, customFileName,
+ tagIDs, channel, processOverrides, nil)
+}
+
+func (s *knowledgeService) createKnowledgeFromFile(ctx context.Context,
+ kbID string, file *multipart.FileHeader, metadata map[string]string, enableMultimodel *bool,
+ customFileName string, tagIDs []string, channel string,
+ processOverrides *types.KnowledgeProcessOverrides,
+ replacement *knowledgeReplacement,
) (*types.Knowledge, error) {
logger.Info(ctx, "Start creating knowledge from file")
@@ -78,10 +89,9 @@ func (s *knowledgeService) CreateKnowledgeFromFile(ctx context.Context,
return nil, ErrInvalidFileType
}
- // JSON content validation stays in the HTTP upload handler (and
- // ReplaceKnowledgeFile), not here: datasource sync may have already
- // deleted the previous knowledge for this external_id, and IM swallows
- // create errors — an early 400 would drop the document with no failed row.
+ // Fresh imports keep JSON validation in the HTTP upload handler (IM needs
+ // a failed row for asynchronous parse errors). Replacement entry points
+ // validate JSON before retiring the previous source.
// Calculate file hash for deduplication
logger.Info(ctx, "Calculating file hash")
@@ -108,7 +118,7 @@ func (s *knowledgeService) CreateKnowledgeFromFile(ctx context.Context,
checkParams.DataSourceID = metadata["datasource_id"]
checkParams.ExternalID = metadata["external_id"]
}
- exists, existingKnowledge, err := s.repo.CheckKnowledgeExists(ctx, tenantID, kbID, checkParams)
+ exists, existingKnowledge, err := s.checkKnowledgeForCreation(ctx, tenantID, kbID, replacement, checkParams)
if err != nil {
logger.Errorf(ctx, "Failed to check knowledge existence: %v", err)
return nil, err
@@ -125,7 +135,7 @@ func (s *knowledgeService) CreateKnowledgeFromFile(ctx context.Context,
// Check storage quota
tenantInfo := ctx.Value(types.TenantInfoContextKey).(*types.Tenant)
- if tenantInfo.StorageQuota > 0 && tenantInfo.StorageUsed >= tenantInfo.StorageQuota {
+ if tenantInfo.StorageQuota > 0 && knowledgeCreationStorageUsed(tenantInfo, replacement) >= tenantInfo.StorageQuota {
logger.Error(ctx, "Storage quota exceeded")
return nil, types.NewStorageQuotaExceededError()
}
@@ -205,15 +215,18 @@ func (s *knowledgeService) CreateKnowledgeFromFile(ctx context.Context,
// Save knowledge record to database after the file is safely stored.
logger.Info(ctx, "Saving knowledge record to database")
- if err := s.repo.CreateKnowledge(ctx, knowledge); err != nil {
+ if err := s.persistKnowledgeForCreation(ctx, knowledge, replacement); err != nil {
logger.Errorf(ctx, "Failed to create knowledge record, ID: %s, error: %v", knowledge.ID, err)
- if deleteErr := fileSvc.DeleteFile(ctx, filePath); deleteErr != nil {
- logger.Errorf(ctx, "Failed to delete saved file after knowledge creation failed, path: %s, error: %v", filePath, deleteErr)
+ if replacement == nil || !replacement.keepFile {
+ if deleteErr := fileSvc.DeleteFile(context.WithoutCancel(ctx), filePath); deleteErr != nil {
+ logger.Errorf(ctx, "Failed to delete saved file after knowledge creation failed, path: %s, error: %v",
+ filePath, deleteErr)
+ }
}
return nil, err
}
// Set tag relations
- if err := s.setAndAttachKnowledgeTags(ctx, tenantID, kbID, knowledge, tagIDs); err != nil {
+ if err := s.attachCreationTags(ctx, knowledge, tagIDs, replacement); err != nil {
logger.Errorf(ctx, "Failed to set knowledge tags, knowledge ID: %s, error: %v", knowledge.ID, err)
return nil, err
}
@@ -309,16 +322,28 @@ func isFileURL(rawURL, fileName, fileType string) bool {
}
func (s *knowledgeService) CreateKnowledgeFromURL(ctx context.Context,
- kbID string, rawURL string, fileName string, fileType string, enableMultimodel *bool, title string, tagIDs []string, channel string,
+ kbID string, rawURL string, fileName string, fileType string, enableMultimodel *bool,
+ title string, tagIDs []string, channel string,
processOverrides *types.KnowledgeProcessOverrides,
+) (*types.Knowledge, error) {
+ return s.createKnowledgeFromURL(ctx, kbID, rawURL, fileName, fileType, enableMultimodel,
+ title, tagIDs, channel, processOverrides, nil)
+}
+
+func (s *knowledgeService) createKnowledgeFromURL(ctx context.Context,
+ kbID string, rawURL string, fileName string, fileType string, enableMultimodel *bool,
+ title string, tagIDs []string, channel string,
+ processOverrides *types.KnowledgeProcessOverrides,
+ replacement *knowledgeReplacement,
) (*types.Knowledge, error) {
logger.Info(ctx, "Start creating knowledge from URL")
logger.Infof(ctx, "Knowledge base ID: %s, URL: %s", kbID, rawURL)
// Route to file_url logic when the URL points to a downloadable file
if isFileURL(rawURL, fileName, fileType) {
- return s.createKnowledgeFromFileURL(
- ctx, kbID, rawURL, fileName, fileType, enableMultimodel, title, tagIDs, channel, processOverrides,
+ return s.createKnowledgeFromFileURLWithReplacement(
+ ctx, kbID, rawURL, fileName, fileType, enableMultimodel,
+ title, tagIDs, channel, processOverrides, replacement,
)
}
@@ -353,11 +378,12 @@ func (s *knowledgeService) CreateKnowledgeFromURL(ctx context.Context,
tenantID := ctx.Value(types.TenantIDContextKey).(uint64)
logger.Infof(ctx, "Checking if URL exists, tenant ID: %d", tenantID)
fileHash := calculateStr(url)
- exists, existingKnowledge, err := s.repo.CheckKnowledgeExists(ctx, tenantID, kbID, &types.KnowledgeCheckParams{
- Type: "url",
- URL: url,
- FileHash: fileHash,
- })
+ exists, existingKnowledge, err := s.checkKnowledgeForCreation(ctx, tenantID, kbID, replacement,
+ &types.KnowledgeCheckParams{
+ Type: "url",
+ URL: url,
+ FileHash: fileHash,
+ })
if err != nil {
logger.Errorf(ctx, "Failed to check knowledge existence: %v", err)
return nil, err
@@ -376,7 +402,7 @@ func (s *knowledgeService) CreateKnowledgeFromURL(ctx context.Context,
// Check storage quota
tenantInfo := ctx.Value(types.TenantInfoContextKey).(*types.Tenant)
- if tenantInfo.StorageQuota > 0 && tenantInfo.StorageUsed >= tenantInfo.StorageQuota {
+ if tenantInfo.StorageQuota > 0 && knowledgeCreationStorageUsed(tenantInfo, replacement) >= tenantInfo.StorageQuota {
logger.Error(ctx, "Storage quota exceeded")
return nil, types.NewStorageQuotaExceededError()
}
@@ -407,12 +433,12 @@ func (s *knowledgeService) CreateKnowledgeFromURL(ctx context.Context,
return nil, err
}
- if err := s.repo.CreateKnowledge(ctx, knowledge); err != nil {
+ if err := s.persistKnowledgeForCreation(ctx, knowledge, replacement); err != nil {
logger.Errorf(ctx, "Failed to create knowledge record: %v", err)
return nil, err
}
// Set tag relations
- if err := s.setAndAttachKnowledgeTags(ctx, tenantID, kbID, knowledge, tagIDs); err != nil {
+ if err := s.attachCreationTags(ctx, knowledge, tagIDs, replacement); err != nil {
logger.Errorf(ctx, "Failed to set knowledge tags, knowledge ID: %s, error: %v", knowledge.ID, err)
return nil, err
}
@@ -512,18 +538,20 @@ func extractFileNameFromContentDisposition(header string) string {
}
// createKnowledgeFromFileURL is the internal implementation for file URL knowledge creation.
-// Called by CreateKnowledgeFromURL when the URL is detected as a direct file download.
+// The replacement-aware variant shares the pipeline for datasource updates.
func (s *knowledgeService) createKnowledgeFromFileURL(
- ctx context.Context,
- kbID string,
- fileURL string,
- fileName string,
- fileType string,
- enableMultimodel *bool,
- title string,
- tagIDs []string,
- channel string,
+ ctx context.Context, kbID, fileURL, fileName, fileType string,
+ enableMultimodel *bool, title string, tagIDs []string, channel string,
processOverrides *types.KnowledgeProcessOverrides,
+) (*types.Knowledge, error) {
+ return s.createKnowledgeFromFileURLWithReplacement(ctx, kbID, fileURL, fileName, fileType,
+ enableMultimodel, title, tagIDs, channel, processOverrides, nil)
+}
+
+func (s *knowledgeService) createKnowledgeFromFileURLWithReplacement(
+ ctx context.Context, kbID, fileURL, fileName, fileType string,
+ enableMultimodel *bool, title string, tagIDs []string, channel string,
+ processOverrides *types.KnowledgeProcessOverrides, replacement *knowledgeReplacement,
) (*types.Knowledge, error) {
logger.Info(ctx, "Start creating knowledge from file URL")
logger.Infof(ctx, "Knowledge base ID: %s, file URL: %s", kbID, fileURL)
@@ -590,11 +618,12 @@ func (s *knowledgeService) createKnowledgeFromFileURL(
// Check for duplicate (by URL hash)
tenantID := ctx.Value(types.TenantIDContextKey).(uint64)
fileHash := calculateStr(fileURL)
- exists, existingKnowledge, err := s.repo.CheckKnowledgeExists(ctx, tenantID, kbID, &types.KnowledgeCheckParams{
- Type: "file_url",
- URL: fileURL,
- FileHash: fileHash,
- })
+ exists, existingKnowledge, err := s.checkKnowledgeForCreation(ctx, tenantID, kbID, replacement,
+ &types.KnowledgeCheckParams{
+ Type: "file_url",
+ URL: fileURL,
+ FileHash: fileHash,
+ })
if err != nil {
logger.Errorf(ctx, "Failed to check knowledge existence: %v", err)
return nil, err
@@ -612,7 +641,7 @@ func (s *knowledgeService) createKnowledgeFromFileURL(
// Check storage quota
tenantInfo := ctx.Value(types.TenantInfoContextKey).(*types.Tenant)
- if tenantInfo.StorageQuota > 0 && tenantInfo.StorageUsed >= tenantInfo.StorageQuota {
+ if tenantInfo.StorageQuota > 0 && knowledgeCreationStorageUsed(tenantInfo, replacement) >= tenantInfo.StorageQuota {
logger.Error(ctx, "Storage quota exceeded")
return nil, types.NewStorageQuotaExceededError()
}
@@ -650,12 +679,12 @@ func (s *knowledgeService) createKnowledgeFromFileURL(
}
}
- if err := s.repo.CreateKnowledge(ctx, knowledge); err != nil {
+ if err := s.persistKnowledgeForCreation(ctx, knowledge, replacement); err != nil {
logger.Errorf(ctx, "Failed to create knowledge record: %v", err)
return nil, err
}
// Set tag relations
- if err := s.setAndAttachKnowledgeTags(ctx, tenantID, kbID, knowledge, tagIDs); err != nil {
+ if err := s.attachCreationTags(ctx, knowledge, tagIDs, replacement); err != nil {
logger.Errorf(ctx, "Failed to set knowledge tags, knowledge ID: %s, error: %v", knowledge.ID, err)
return nil, err
}
diff --git a/internal/application/service/knowledge_datasource_replace.go b/internal/application/service/knowledge_datasource_replace.go
new file mode 100644
index 000000000..fbe2fd2b0
--- /dev/null
+++ b/internal/application/service/knowledge_datasource_replace.go
@@ -0,0 +1,141 @@
+package service
+
+import (
+ "context"
+ "fmt"
+
+ "github.com/Tencent/WeKnora/internal/logger"
+ "github.com/Tencent/WeKnora/internal/types"
+)
+
+// knowledgeReplacement is local to a single creation pipeline; a successful
+// persist switches the live rows before that pipeline can enqueue a worker.
+type knowledgeReplacement struct {
+ previous *types.Knowledge
+ metadata map[string]string
+ tagIDs []string
+ committed bool
+ keepFile bool // a failed commit/read may still have published the new path
+}
+
+func (s *knowledgeService) checkKnowledgeForCreation(ctx context.Context, tenantID uint64, kbID string,
+ replacement *knowledgeReplacement, params *types.KnowledgeCheckParams,
+) (bool, *types.Knowledge, error) {
+ if replacement != nil {
+ params.ExcludeKnowledgeID = replacement.previous.ID
+ }
+ return s.repo.CheckKnowledgeExists(ctx, tenantID, kbID, params)
+}
+
+func (s *knowledgeService) persistKnowledgeForCreation(ctx context.Context, knowledge *types.Knowledge,
+ replacement *knowledgeReplacement,
+) error {
+ if replacement == nil {
+ return s.repo.CreateKnowledge(ctx, knowledge)
+ }
+ // URL creation normally has no datasource metadata. Include it in the same
+ // INSERT so a committed replacement is immediately discoverable on retry.
+ metadata, err := mergeKnowledgeMetadata(knowledge.Metadata, replacement.metadata)
+ if err != nil {
+ return err
+ }
+ knowledge.Metadata = metadata
+ if err := s.repo.ReplaceKnowledge(ctx, replacement.previous, knowledge, replacement.tagIDs); err != nil {
+ // A commit can succeed while its acknowledgement is lost. Only discard
+ // the staged file when a read confirms the previous row is still live.
+ cleanupCtx := context.WithoutCancel(ctx)
+ old, oldErr := s.repo.GetKnowledgeByID(cleanupCtx, replacement.previous.TenantID, replacement.previous.ID)
+ if oldErr == nil && old != nil {
+ return err
+ }
+ current, readErr := s.repo.GetKnowledgeByID(cleanupCtx, knowledge.TenantID, knowledge.ID)
+ if readErr != nil || current == nil {
+ replacement.keepFile = true
+ return err
+ }
+ logger.Warnf(ctx, "Knowledge replacement %s committed despite reported error: %v", knowledge.ID, err)
+ }
+ replacement.committed = true
+ return nil
+}
+
+// ReplaceKnowledgeFromDataSource preserves the previous row and its resources
+// until validation, file storage and the replacement INSERT succeed. The DB
+// transaction covers both knowledge rows and new tag relations; parsing remains asynchronous
+// and cleanup of the retired row's external resources is best-effort after commit.
+func (s *knowledgeService) ReplaceKnowledgeFromDataSource(ctx context.Context, knowledgeID string,
+ item *types.FetchedItem, metadata map[string]string, tagIDs []string, channel string,
+) (*types.Knowledge, error) {
+ if item == nil || (len(item.Content) == 0 && item.URL == "") {
+ return nil, fmt.Errorf("item has neither content nor URL")
+ }
+ plan, err := s.planKnowledgeDelete(ctx, []string{knowledgeID})
+ if err != nil {
+ return nil, err
+ }
+ previous := plan.knowledge[0]
+ stored := previous.GetMetadata()
+ if metadata["datasource_id"] == "" || metadata["external_id"] == "" ||
+ stored["datasource_id"] != metadata["datasource_id"] || stored["external_id"] != metadata["external_id"] {
+ return nil, fmt.Errorf("datasource knowledge identity changed")
+ }
+ if err := s.validateKnowledgeTagIDs(plan.ctx, previous.TenantID, previous.KnowledgeBaseID, tagIDs); err != nil {
+ return nil, err
+ }
+ replacement := &knowledgeReplacement{previous: previous, metadata: metadata, tagIDs: tagIDs}
+ var created *types.Knowledge
+ if len(item.Content) > 0 {
+ file, buildErr := bytesToFileHeader(item.Content, item.FileName)
+ if buildErr != nil {
+ return nil, fmt.Errorf("build file header: %w", buildErr)
+ }
+ if err := ValidateJSONUploadContent(item.FileName, file); err != nil {
+ return nil, err
+ }
+ created, err = s.createKnowledgeFromFile(plan.ctx, previous.KnowledgeBaseID, file, metadata,
+ nil, item.FileName, tagIDs, channel, nil, replacement)
+ } else {
+ created, err = s.createKnowledgeFromURL(plan.ctx, previous.KnowledgeBaseID, item.URL, item.FileName,
+ "", nil, item.Title, tagIDs, channel, nil, replacement)
+ }
+ if !replacement.committed {
+ return created, err
+ }
+
+ // Never run the delete cascade before the INSERT commits: rollback cannot
+ // restore an object-store file, vector index, graph or wiki contribution.
+ // The old row is already soft-deleted, so do not mark/save it again (which
+ // could resurrect it). Keep the tombstone if external cleanup fails.
+ plan.rowsDeleted = true
+ plan.ctx = context.WithoutCancel(plan.ctx)
+ if cleanupErr := s.executeKnowledgeDelete(plan, true); cleanupErr != nil {
+ logger.Warnf(plan.ctx, "Replacement committed; failed to clean retired knowledge %s: %v",
+ knowledgeID, cleanupErr)
+ } else if cleanupErr := s.repo.HardDeleteKnowledge(plan.ctx, previous.TenantID, knowledgeID); cleanupErr != nil {
+ logger.Warnf(plan.ctx, "Replacement committed; failed to remove retired knowledge %s: %v",
+ knowledgeID, cleanupErr)
+ }
+ return created, err
+}
+
+// Replacement tags were inserted with the row transaction. Normal uploads keep
+// their existing validation/persistence behavior.
+func (s *knowledgeService) attachCreationTags(ctx context.Context, knowledge *types.Knowledge,
+ tagIDs []string, replacement *knowledgeReplacement,
+) error {
+ if replacement == nil {
+ return s.setAndAttachKnowledgeTags(ctx, knowledge.TenantID, knowledge.KnowledgeBaseID, knowledge, tagIDs)
+ }
+ s.attachTagsToKnowledge(ctx, knowledge)
+ return nil
+}
+
+// Retirement used to free this storage before creation checked quota. Preserve
+// that allowance without changing the shared tenant snapshot before commit.
+func knowledgeCreationStorageUsed(tenant *types.Tenant, replacement *knowledgeReplacement) int64 {
+ used := tenant.StorageUsed
+ if replacement != nil {
+ used -= replacement.previous.StorageSize
+ }
+ return used
+}
diff --git a/internal/application/service/knowledge_datasource_replace_test.go b/internal/application/service/knowledge_datasource_replace_test.go
new file mode 100644
index 000000000..76a1850a8
--- /dev/null
+++ b/internal/application/service/knowledge_datasource_replace_test.go
@@ -0,0 +1,212 @@
+package service
+
+import (
+ "context"
+ "encoding/json"
+ "errors"
+ "path/filepath"
+ "testing"
+
+ "github.com/Tencent/WeKnora/internal/application/repository"
+ "github.com/Tencent/WeKnora/internal/types"
+ "github.com/Tencent/WeKnora/internal/types/interfaces"
+ "github.com/hibiken/asynq"
+ "github.com/stretchr/testify/require"
+ "gorm.io/driver/sqlite"
+ "gorm.io/gorm"
+)
+
+type datasourceReplacementChunks struct {
+ replaceFileChunks
+ deleted int
+ err error
+}
+
+func (c *datasourceReplacementChunks) DeleteByKnowledgeList(context.Context, uint64, []string) error {
+ c.deleted++
+ return c.err
+}
+
+type datasourceReplacementTenant struct{ interfaces.TenantRepository }
+
+func (datasourceReplacementTenant) AdjustStorageUsed(context.Context, uint64, int64) error {
+ return nil
+}
+
+type datasourceReplacementQueue struct {
+ t *testing.T
+ repo interfaces.KnowledgeRepository
+ oldID string
+ queued int
+}
+
+func (q *datasourceReplacementQueue) Enqueue(task *asynq.Task, _ ...asynq.Option) (*asynq.TaskInfo, error) {
+ if task.Type() == types.TypeDocumentProcess {
+ var payload types.DocumentProcessPayload
+ require.NoError(q.t, json.Unmarshal(task.Payload(), &payload))
+ _, err := q.repo.GetKnowledgeByID(context.Background(), 7, q.oldID)
+ require.Error(q.t, err, "retirement must commit before a worker can start")
+ created, err := q.repo.GetKnowledgeByID(context.Background(), 7, payload.KnowledgeID)
+ require.NoError(q.t, err, "replacement must commit before a worker can start")
+ require.Equal(q.t, "ds-1", created.GetMetadata()["datasource_id"])
+ require.Equal(q.t, "doc-1", created.GetMetadata()["external_id"])
+ q.queued++
+ }
+ return &asynq.TaskInfo{ID: "task-1", Queue: types.QueueDefault}, nil
+}
+
+func datasourceReplacementFixture(t *testing.T) (
+ *replaceFileHarness, *gorm.DB, *datasourceReplacementChunks, *datasourceReplacementQueue,
+) {
+ t.Helper()
+ h := newReplaceFileHarness(t)
+ db, err := gorm.Open(sqlite.Open(filepath.Join(t.TempDir(), "datasource.db")), &gorm.Config{})
+ require.NoError(t, err)
+ sqlDB, err := db.DB()
+ require.NoError(t, err)
+ t.Cleanup(func() { require.NoError(t, sqlDB.Close()) })
+ require.NoError(t, db.AutoMigrate(&types.Knowledge{}, &types.KnowledgeTag{}, &types.KnowledgeTagRelation{}))
+ repo := repository.NewKnowledgeRepository(db)
+ h.original.Metadata = types.JSON(`{"datasource_id":"ds-1","external_id":"doc-1"}`)
+ require.NoError(t, repo.CreateKnowledge(h.ctx, &h.original))
+ h.svc.repo = repo
+ chunks := &datasourceReplacementChunks{}
+ h.svc.chunkRepo = chunks
+ h.svc.tenantRepo = datasourceReplacementTenant{}
+ queue := &datasourceReplacementQueue{t: t, repo: repo, oldID: h.original.ID}
+ h.svc.task = queue
+ return h, db, chunks, queue
+}
+
+func TestDataSourceReplacementPreservesOldResourcesOnFailure(t *testing.T) {
+ for _, stage := range []string{"validation", "json", "storage", "insert"} {
+ t.Run(stage, func(t *testing.T) {
+ h, db, chunks, queue := datasourceReplacementFixture(t)
+ item := &types.FetchedItem{ExternalID: "doc-1", Content: []byte("new content"), FileName: "doc.md"}
+ switch stage {
+ case "validation":
+ item.FileName = "doc.unsupported"
+ case "json":
+ item.FileName, item.Content = "doc.json", []byte("invalid JSON")
+ case "storage":
+ h.store.saveErr = errors.New("storage unavailable")
+ case "insert":
+ require.NoError(t, db.Exec(`CREATE TRIGGER fail_replacement BEFORE INSERT ON knowledges
+ BEGIN SELECT RAISE(FAIL, 'forced insert failure'); END`).Error)
+ }
+ ds := &DataSourceService{knowledgeService: h.svc}
+ _, err := ds.ingestItem(h.ctx,
+ &types.DataSource{ID: "ds-1", TenantID: 7, KnowledgeBaseID: "kb-1"}, item, nil)
+ require.Error(t, err)
+ old, err := h.svc.repo.GetKnowledgeByID(h.ctx, 7, h.original.ID)
+ require.NoError(t, err)
+ require.Equal(t, h.original.FilePath, old.FilePath)
+ require.Equal(t, types.ParseStatusCompleted, old.ParseStatus)
+ require.Zero(t, chunks.deleted)
+ require.Zero(t, queue.queued)
+ for _, path := range h.store.deleted {
+ require.NotEqual(t, h.original.FilePath, path)
+ }
+ if stage == "insert" {
+ require.Equal(t, []string{"new/file.md"}, h.store.deleted, "discard only the staged new file")
+ }
+ })
+ }
+}
+
+func TestDataSourceReplacementCommitsBeforeEnqueueAndCleanup(t *testing.T) {
+ for _, kind := range []string{"file", "url", "file_url"} {
+ t.Run(kind, func(t *testing.T) {
+ h, _, chunks, queue := datasourceReplacementFixture(t)
+ tenant := h.ctx.Value(types.TenantInfoContextKey).(*types.Tenant)
+ tenant.StorageQuota, tenant.StorageUsed = 100, 100
+ require.NoError(t, h.svc.repo.UpdateKnowledgeColumn(h.ctx, h.original.ID, "storage_size", int64(10)))
+ item := &types.FetchedItem{ExternalID: "doc-1", FileName: "doc.md"}
+ switch kind {
+ case "file":
+ item.Content = []byte("new content")
+ case "url":
+ item.URL, item.FileName = "https://example.com/doc", ""
+ case "file_url":
+ item.URL = "https://example.com/doc.md"
+ }
+ ds := &DataSourceService{knowledgeService: h.svc}
+ updated, err := ds.ingestItem(h.ctx,
+ &types.DataSource{ID: "ds-1", TenantID: 7, KnowledgeBaseID: "kb-1"}, item, nil)
+ require.NoError(t, err)
+ require.True(t, updated)
+ require.Equal(t, 1, queue.queued)
+ require.Equal(t, 1, chunks.deleted)
+ require.Contains(t, h.store.deleted, h.original.FilePath)
+ created, err := h.svc.repo.FindByDataSourceExternalID(h.ctx, 7, "kb-1", "ds-1", "doc-1")
+ require.NoError(t, err)
+ require.NotNil(t, created)
+ require.Equal(t, kind, created.Type)
+ require.NotEqual(t, h.original.ID, created.ID)
+ })
+ }
+}
+
+func TestDataSourceReplacementCleanupFailureKeepsCommittedReplacement(t *testing.T) {
+ h, db, chunks, queue := datasourceReplacementFixture(t)
+ chunks.err = errors.New("index cleanup unavailable")
+ ds := &DataSourceService{knowledgeService: h.svc}
+ _, err := ds.ingestItem(h.ctx, &types.DataSource{ID: "ds-1", TenantID: 7, KnowledgeBaseID: "kb-1"},
+ &types.FetchedItem{ExternalID: "doc-1", Content: []byte("new content"), FileName: "doc.md"}, nil)
+ require.NoError(t, err)
+ require.Equal(t, 1, queue.queued)
+ require.NotContains(t, h.store.deleted, h.original.FilePath)
+ var retired types.Knowledge
+ require.NoError(t, db.Unscoped().First(&retired, "id = ?", h.original.ID).Error)
+ require.True(t, retired.DeletedAt.Valid)
+ created, err := h.svc.repo.FindByDataSourceExternalID(h.ctx, 7, "kb-1", "ds-1", "doc-1")
+ require.NoError(t, err)
+ require.NotNil(t, created)
+}
+
+type uncertainReplacementRepo struct {
+ interfaces.KnowledgeRepository
+ committed bool
+ failReads bool
+}
+
+func (r *uncertainReplacementRepo) ReplaceKnowledge(
+ ctx context.Context, before, after *types.Knowledge, tags []string,
+) error {
+ if err := r.KnowledgeRepository.ReplaceKnowledge(ctx, before, after, tags); err != nil {
+ return err
+ }
+ r.committed = true
+ return errors.New("commit acknowledgement lost")
+}
+
+func (r *uncertainReplacementRepo) GetKnowledgeByID(
+ ctx context.Context, tenant uint64, id string,
+) (*types.Knowledge, error) {
+ if r.committed && r.failReads {
+ return nil, errors.New("database unavailable")
+ }
+ return r.KnowledgeRepository.GetKnowledgeByID(ctx, tenant, id)
+}
+
+func TestDataSourceReplacementRetainsFileOnUncertainCommit(t *testing.T) {
+ for _, failReads := range []bool{false, true} {
+ t.Run(map[bool]string{false: "commit confirmed", true: "reads unavailable"}[failReads], func(t *testing.T) {
+ h, _, chunks, queue := datasourceReplacementFixture(t)
+ h.svc.repo = &uncertainReplacementRepo{KnowledgeRepository: h.svc.repo, failReads: failReads}
+ ds := &DataSourceService{knowledgeService: h.svc}
+ _, err := ds.ingestItem(h.ctx, &types.DataSource{ID: "ds-1", TenantID: 7, KnowledgeBaseID: "kb-1"},
+ &types.FetchedItem{ExternalID: "doc-1", Content: []byte("new content"), FileName: "doc.md"}, nil)
+ require.NotContains(t, h.store.deleted, "new/file.md", "a committed row may reference this file")
+ if failReads {
+ require.ErrorContains(t, err, "commit acknowledgement lost")
+ require.Zero(t, queue.queued)
+ require.Zero(t, chunks.deleted)
+ } else {
+ require.NoError(t, err)
+ require.Equal(t, 1, queue.queued)
+ require.Equal(t, 1, chunks.deleted)
+ }
+ })
+ }
+}
diff --git a/internal/application/service/knowledge_delete.go b/internal/application/service/knowledge_delete.go
index 4761d5693..92d6f6423 100644
--- a/internal/application/service/knowledge_delete.go
+++ b/internal/application/service/knowledge_delete.go
@@ -488,10 +488,12 @@ func (s *knowledgeService) executeKnowledgeDelete(plan *knowledgeDeletePlan, sin
prev := knowledge.ParseStatus
before := *knowledge
knowledge.ParseStatus = types.ParseStatusDeleting
- if err := s.repo.UpdateKnowledgeForTransfer(ctx, &before, knowledge); err != nil {
- logger.GetLogger(ctx).WithField("error", err).WithField("knowledge_id", knowledge.ID).
- Errorf("DeleteKnowledgeList failed to mark as deleting")
- return err
+ if !plan.rowsDeleted {
+ if err := s.repo.UpdateKnowledgeForTransfer(ctx, &before, knowledge); err != nil {
+ logger.GetLogger(ctx).WithField("error", err).WithField("knowledge_id", knowledge.ID).
+ Errorf("DeleteKnowledgeList failed to mark as deleting")
+ return err
+ }
}
if prev == types.ParseStatusPending || prev == types.ParseStatusProcessing {
inFlightIDs = append(inFlightIDs, knowledge.ID)
@@ -615,8 +617,10 @@ func (s *knowledgeService) executeKnowledgeDelete(plan *knowledgeDeletePlan, sin
// gone avoids "file missing but row present" zombies that break reparse /
// re-delete when an earlier cleanup step failed (issue #2192). A failure below
// only orphans storage.
- if err := s.repo.DeleteKnowledgeList(ctx, tenantInfo.ID, ids); err != nil {
- return err
+ if !plan.rowsDeleted {
+ if err := s.repo.DeleteKnowledgeList(ctx, tenantInfo.ID, ids); err != nil {
+ return err
+ }
}
storageAdjust := int64(0)
diff --git a/internal/application/service/knowledge_delete_plan.go b/internal/application/service/knowledge_delete_plan.go
index f08f17aea..c3b2546f4 100644
--- a/internal/application/service/knowledge_delete_plan.go
+++ b/internal/application/service/knowledge_delete_plan.go
@@ -72,12 +72,14 @@ func deleteReferencedKnowledge(
}
type knowledgeDeletePlan struct {
- ctx context.Context
- ids []string
- knowledge []*types.Knowledge
- kbs map[string]*types.KnowledgeBase
- files map[string]interfaces.FileService
- imageInfo []interfaces.ChunkImageInfo
+ // A datasource replacement already soft-deleted the old row atomically.
+ rowsDeleted bool
+ ctx context.Context
+ ids []string
+ knowledge []*types.Knowledge
+ kbs map[string]*types.KnowledgeBase
+ files map[string]interfaces.FileService
+ imageInfo []interfaces.ChunkImageInfo
}
func (s *knowledgeService) planKnowledgeDelete(ctx context.Context, ids []string) (*knowledgeDeletePlan, error) {
diff --git a/internal/types/interfaces/knowledge.go b/internal/types/interfaces/knowledge.go
index 32448da3b..361674c5c 100644
--- a/internal/types/interfaces/knowledge.go
+++ b/internal/types/interfaces/knowledge.go
@@ -11,6 +11,11 @@ import (
// KnowledgeService defines the interface for knowledge services.
type KnowledgeService interface {
+ // ReplaceKnowledgeFromDataSource atomically retires the previous row and inserts
+ // its replacement before enqueueing parsing or deleting the previous resources.
+ ReplaceKnowledgeFromDataSource(ctx context.Context, knowledgeID string, item *types.FetchedItem,
+ metadata map[string]string, tagIDs []string, channel string) (*types.Knowledge, error)
+
// CreateKnowledgeFromFile creates knowledge from a file.
// channel identifies the ingestion channel (e.g. "web", "api", "wechat"); empty defaults to "web".
CreateKnowledgeFromFile(
@@ -242,6 +247,11 @@ type KnowledgeService interface {
// KnowledgeRepository defines the interface for knowledge repositories.
type KnowledgeRepository interface {
+ // ReplaceKnowledge soft-deletes before and inserts after with its tag relations
+ // in one short transaction.
+ // A stale before snapshot is rejected without deleting or inserting either row.
+ ReplaceKnowledge(ctx context.Context, before, after *types.Knowledge, tagIDs []string) error
+
CreateKnowledge(ctx context.Context, knowledge *types.Knowledge) error
GetKnowledgeByID(ctx context.Context, tenantID uint64, id string) (*types.Knowledge, error)
// GetKnowledgeByIDOnly returns knowledge by ID without tenant filter (for permission resolution).
diff --git a/internal/types/knowledge.go b/internal/types/knowledge.go
index d76952939..aad0fbf8a 100644
--- a/internal/types/knowledge.go
+++ b/internal/types/knowledge.go
@@ -547,6 +547,8 @@ func (k *Knowledge) SetProcessOverrides(o *KnowledgeProcessOverrides) error {
// KnowledgeCheckParams defines parameters used to check if knowledge already exists.
type KnowledgeCheckParams struct {
+ // ExcludeKnowledgeID ignores only the row being replaced during duplicate checks.
+ ExcludeKnowledgeID string
// File parameters
FileName string
// FileType scopes file-hash deduplication; callers checking file uploads should set it.