Skip to content

Open-source LLM knowledge platform: turn raw documents into a queryable RAG, an autonomous reasoning agent, and a self-maintaining Wiki.

fix(datasource): honor skip conflict strategy (#4643)

LuckVdbeb5a3774a95
diff --git a/internal/application/service/datasource_conflict_skip_test.go b/internal/application/service/datasource_conflict_skip_test.go
new file mode 100644
index 000000000..ca509c5e0
--- /dev/null
+++ b/internal/application/service/datasource_conflict_skip_test.go
@@ -0,0 +1,219 @@
+package service
+
+import (
+ "context"
+ "encoding/json"
+ "testing"
+ "time"
+
+ "github.com/Tencent/WeKnora/internal/datasource"
+ "github.com/Tencent/WeKnora/internal/types"
+ "github.com/hibiken/asynq"
+ "github.com/stretchr/testify/assert"
+ "github.com/stretchr/testify/require"
+)
+
+// ─────────────────────────────────────────────────────────────────────────────
+// Conflict strategy "skip" (跳过已存在): when a data source is configured with
+// ConflictStrategySkip, re-syncing an item whose external_id already exists in
+// the KB must leave the existing knowledge untouched and count the item as
+// skipped. Tencent/WeKnora#4640: ingestItem ignored ds.ConflictStrategy and
+// always replaced (delete + re-create) existing items.
+// ─────────────────────────────────────────────────────────────────────────────
+
+const conflictSkipConnectorType = "test-conflict-skip"
+
+// conflictSkipConnector fetches one normal (non-deleted) content item whose
+// external_id already exists in the KB fake.
+type conflictSkipConnector struct{}
+
+func (conflictSkipConnector) Type() string { return conflictSkipConnectorType }
+func (conflictSkipConnector) Validate(context.Context, *types.DataSourceConfig) error {
+ return nil
+}
+
+func (conflictSkipConnector) ListResources(context.Context, *types.DataSourceConfig, string) ([]types.Resource, error) {
+ return nil, nil
+}
+
+func (conflictSkipConnector) ResolveResourceAncestors(
+ context.Context, *types.DataSourceConfig, []string,
+) ([]string, error) {
+ return nil, nil
+}
+
+func (conflictSkipConnector) FetchAll(context.Context, *types.DataSourceConfig, []string) ([]types.FetchedItem, error) {
+ return []types.FetchedItem{{
+ ExternalID: "file:exists",
+ Title: "Existing Doc",
+ Content: []byte("# v2\n"),
+ ContentType: "text/markdown",
+ FileName: "existing.md",
+ SourceResourceID: "folder:1",
+ }}, nil
+}
+
+func (conflictSkipConnector) FetchIncremental(
+ context.Context, *types.DataSourceConfig, *types.SyncCursor,
+) ([]types.FetchedItem, *types.SyncCursor, error) {
+ items, err := (conflictSkipConnector{}).FetchAll(context.Background(), nil, nil)
+ return items, nil, err
+}
+
+// newConflictSkipHarness wires a full-sync ProcessSync fixture whose data
+// source carries the given conflict strategy and whose KB already contains the
+// synced item under the same external_id.
+func newConflictSkipHarness(
+ t *testing.T, strategy string,
+) (*DataSourceService, *types.DataSource, *deletionLookupKnowledgeRepo, *sweepFakeKS, *types.SyncLog) {
+ t.Helper()
+ configJSON, err := (&types.DataSourceConfig{Type: conflictSkipConnectorType}).ToJSON()
+ require.NoError(t, err)
+
+ ds := &types.DataSource{
+ ID: "ds-conflict-skip",
+ TenantID: 1,
+ KnowledgeBaseID: "kb-1",
+ Name: "Conflict Skip",
+ Type: conflictSkipConnectorType,
+ Config: configJSON,
+ SyncMode: types.SyncModeFull,
+ Status: types.DataSourceStatusActive,
+ ConflictStrategy: strategy,
+ }
+ existing := &types.Knowledge{ID: "knowledge-exists", FileName: "existing.md"}
+ repo := &deletionLookupKnowledgeRepo{knowledge: existing}
+ ks := &sweepFakeKS{repo: repo}
+ syncLog := &types.SyncLog{
+ ID: "log-conflict-skip",
+ DataSourceID: ds.ID,
+ TenantID: ds.TenantID,
+ Status: types.SyncLogStatusRunning,
+ StartedAt: time.Now().UTC(),
+ }
+ registry := datasource.NewConnectorRegistry()
+ require.NoError(t, registry.Register(conflictSkipConnector{}))
+
+ svc := &DataSourceService{
+ dsRepo: newKBDeleteDSRepo(ds.KnowledgeBaseID, ds),
+ syncLogRepo: &processSyncSyncLogRepo{logs: map[string]*types.SyncLog{syncLog.ID: syncLog}},
+ knowledgeService: ks,
+ kbService: &processSyncKBService{
+ kb: &types.KnowledgeBase{ID: ds.KnowledgeBaseID, TenantID: ds.TenantID},
+ },
+ connectorRegistry: registry,
+ tenantRepo: &processSyncTenantRepo{tenant: &types.Tenant{ID: ds.TenantID}},
+ tagService: &processSyncTagService{},
+ }
+ return svc, ds, repo, ks, syncLog
+}
+
+func runConflictSkipSync(
+ t *testing.T,
+ svc *DataSourceService,
+ ds *types.DataSource,
+ syncLog *types.SyncLog,
+) *types.SyncLog {
+ t.Helper()
+ payload, err := json.Marshal(types.DataSourceSyncPayload{
+ DataSourceID: ds.ID,
+ TenantID: ds.TenantID,
+ SyncLogID: syncLog.ID,
+ ForceFull: true,
+ })
+ require.NoError(t, err)
+ require.NoError(t, svc.ProcessSync(context.Background(), asynq.NewTask(types.TypeDataSourceSync, payload)))
+ return svc.syncLogRepo.(*processSyncSyncLogRepo).logs[syncLog.ID]
+}
+
+// With ConflictStrategySkip the existing knowledge must be left untouched:
+// no delete, no re-create, and the item counted as skipped.
+func TestProcessSync_SkipStrategyKeepsExistingCountsSkipped(t *testing.T) {
+ svc, ds, repo, ks, syncLog := newConflictSkipHarness(t, types.ConflictStrategySkip)
+
+ log := runConflictSkipSync(t, svc, ds, syncLog)
+
+ assert.Empty(t, repo.hardDeleted, "skip strategy must not hard-delete the existing knowledge")
+ assert.NotContains(t, ks.events, "delete:knowledge-exists")
+ assert.NotContains(t, ks.events, "create:existing.md")
+ assert.Equal(t, 0, log.ItemsCreated)
+ assert.Equal(t, 0, log.ItemsUpdated)
+ assert.Equal(t, 1, log.ItemsSkipped)
+}
+
+// Default strategy (overwrite) keeps the historical replace behavior: the
+// existing item is deleted and re-created, counted as updated.
+func TestProcessSync_OverwriteStrategyReplacesExisting(t *testing.T) {
+ svc, ds, repo, ks, syncLog := newConflictSkipHarness(t, types.ConflictStrategyOverwrite)
+
+ log := runConflictSkipSync(t, svc, ds, syncLog)
+
+ assert.Equal(t, []string{"knowledge-exists"}, repo.hardDeleted)
+ assert.Contains(t, ks.events, "delete:knowledge-exists")
+ assert.Contains(t, ks.events, "create:existing.md")
+ assert.Equal(t, 0, log.ItemsCreated)
+ assert.Equal(t, 1, log.ItemsUpdated)
+ assert.Equal(t, 0, log.ItemsSkipped)
+}
+
+// ingestItem-level: with skip strategy the lookup hit short-circuits into a
+// DuplicateKnowledgeError (the caller's shared skip-counting path) before any
+// delete or create happens.
+func TestIngestItem_SkipStrategyShortCircuitsAsDuplicate(t *testing.T) {
+ existing := &types.Knowledge{ID: "knowledge-exists", FileName: "existing.md"}
+ repo := &deletionLookupKnowledgeRepo{knowledge: existing}
+ ks := &sweepFakeKS{repo: repo}
+ ds := &types.DataSource{
+ ID: "ds-1",
+ TenantID: 1,
+ KnowledgeBaseID: "kb-1",
+ ConflictStrategy: types.ConflictStrategySkip,
+ }
+ svc := &DataSourceService{knowledgeService: ks}
+
+ isUpdate, err := svc.ingestItem(context.Background(), ds, &types.FetchedItem{
+ ExternalID: "file:exists",
+ Title: "Existing Doc",
+ Content: []byte("# v2\n"),
+ ContentType: "text/markdown",
+ FileName: "existing.md",
+ }, nil)
+
+ require.Error(t, err)
+ var dupErr *types.DuplicateKnowledgeError
+ require.ErrorAs(t, err, &dupErr)
+ assert.Same(t, existing, dupErr.Knowledge)
+ assert.False(t, isUpdate)
+ assert.Empty(t, repo.hardDeleted, "no hard delete may run for a skipped item")
+ assert.Empty(t, ks.events, "neither delete nor create may run for a skipped item")
+}
+
+// URL-only items follow the same skip path: the existing knowledge survives
+// and CreateKnowledgeFromURL is never called.
+func TestIngestItem_SkipStrategyKeepsExistingURLItem(t *testing.T) {
+ existing := &types.Knowledge{ID: "knowledge-url-exists", Source: "https://example.com/doc"}
+ repo := &deletionLookupKnowledgeRepo{knowledge: existing}
+ ks := &sweepFakeKS{repo: repo}
+ ds := &types.DataSource{
+ ID: "ds-1",
+ TenantID: 1,
+ KnowledgeBaseID: "kb-1",
+ ConflictStrategy: types.ConflictStrategySkip,
+ }
+ svc := &DataSourceService{knowledgeService: ks}
+
+ isUpdate, err := svc.ingestItem(context.Background(), ds, &types.FetchedItem{
+ ExternalID: "url:1",
+ URL: "https://example.com/doc",
+ Title: "Existing URL Doc",
+ }, nil)
+
+ require.Error(t, err)
+ var dupErr *types.DuplicateKnowledgeError
+ require.ErrorAs(t, err, &dupErr)
+ assert.Same(t, existing, dupErr.Knowledge)
+ assert.False(t, isUpdate)
+ assert.Empty(t, repo.hardDeleted)
+ assert.Empty(t, ks.events)
+ assert.Empty(t, repo.metadataUpdates, "CreateKnowledgeFromURL must not be reached for a skipped item")
+}
diff --git a/internal/application/service/datasource_service.go b/internal/application/service/datasource_service.go
index b8abb4865..f651201f5 100644
--- a/internal/application/service/datasource_service.go
+++ b/internal/application/service/datasource_service.go
@@ -1424,6 +1424,24 @@ func (s *DataSourceService) ingestItem(ctx context.Context, ds *types.DataSource
logger.Warnf(ctx, "failed to check existing knowledge for external_id=%s: %v", item.ExternalID, err)
// Non-fatal: proceed with creation (may produce duplicate)
} else if existing != nil {
+ // Conflict strategy "skip" (跳过已存在): the item already exists in
+ // the KB, so leave it untouched and report the conflict as a skip
+ // via the shared DuplicateKnowledgeError path (Tencent/WeKnora#4640 —
+ // this strategy was silently ignored and the item was overwritten).
+ if ds.ConflictStrategy == types.ConflictStrategySkip {
+ logger.Infof(
+ ctx,
+ "item %q (external_id=%s) already exists as knowledge %s, conflict strategy skip → skipping",
+ item.Title, item.ExternalID, existing.ID,
+ )
+ return false, &types.DuplicateKnowledgeError{
+ Message: fmt.Sprintf(
+ "knowledge already exists for external_id %s (conflict strategy: skip)",
+ item.ExternalID,
+ ),
+ Knowledge: existing,
+ }
+ }
logger.Infof(ctx, "found existing knowledge %s for external_id=%s, deleting for update", existing.ID, item.ExternalID)
if err := s.knowledgeService.DeleteKnowledge(ctx, existing.ID); err != nil {
logger.Warnf(ctx, "failed to delete existing knowledge %s: %v", existing.ID, err)