Skip to content

RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.

Fix: cross-KB document id collision in data source sync (#18116) (#18244)

Mpider-Sanaa57022bfdb1
diff --git a/api/apps/services/dataset_api_service.py b/api/apps/services/dataset_api_service.py
index 914b21cb54..aa6ba965e1 100644
--- a/api/apps/services/dataset_api_service.py
+++ b/api/apps/services/dataset_api_service.py
@@ -18,9 +18,9 @@ import logging
import os
import re
-from api.db.db_models import File
+from api.db.db_models import Connector2Kb, Document, File, SyncLogs
from api.db.joint_services.tenant_model_service import get_composite_model_name_by_ids, resolve_model_config, resolve_model_id
-from api.db.services.connector_service import Connector2KbService
+from api.db.services.connector_service import Connector2KbService, SyncLogsService
from api.db.services.document_service import DocumentService, queue_raptor_o_graphrag_tasks
from api.db.services.file2document_service import File2DocumentService
from api.db.services.file_service import FileService
@@ -30,7 +30,7 @@ from api.db.services.tenant_model_service import TenantModelService
from api.db.services.user_service import TenantService, UserService, UserTenantService
from api.utils.api_utils import deep_merge, get_parser_config, remap_dictionary_keys, verify_embedding_availability
from common import settings
-from common.constants import PAGERANK_FLD, FileSource, LLMType, RetCode, StatusEnum
+from common.constants import PAGERANK_FLD, FileSource, LLMType, RetCode, StatusEnum, TaskStatus
from common.misc_utils import thread_pool_exec, thread_pool_exec_long_time
from rag.advanced_rag.knowlege_compile.wiki import WIKI_PAGE_COMPILE_KWD
@@ -171,6 +171,15 @@ def _delete_datasets_sync(tenant_id: str, ids: list = None, delete_all: bool = F
errors = []
success_count = 0
for kb_id, kb in kb_id_instance_pairs:
+ # Cancel this dataset's queued syncs before touching its documents.
+ # Tasks are only picked up while they are SCHEDULE, so cancelling stops
+ # every run that has not started yet; a sync already in flight is not
+ # interruptible, which is what the stranded-row sweep below covers.
+ SyncLogsService.filter_update(
+ [SyncLogs.kb_id == kb_id, SyncLogs.status.in_([TaskStatus.SCHEDULE, TaskStatus.RUNNING])],
+ {"status": TaskStatus.CANCEL},
+ )
+
for doc in DocumentService.query(kb_id=kb_id):
if not DocumentService.remove_document(doc, tenant_id):
errors.append(f"Remove document '{doc.id}' error for dataset '{kb_id}'")
@@ -207,6 +216,22 @@ def _delete_datasets_sync(tenant_id: str, ids: list = None, delete_all: bool = F
if not KnowledgebaseService.delete_by_id(kb_id):
errors.append(f"Delete dataset error for {kb_id}")
continue
+
+ # Unwire the data sources only once the dataset is really gone, so a
+ # failed deletion above leaves a dataset that is still linked and still
+ # syncable. Left behind, these rows keep the connector scheduler queueing
+ # syncs against a kb_id that no longer resolves, and any document such a
+ # run writes outlives its dataset -- an invisible row that later reports
+ # a cross-KB id collision against whatever dataset is linked next.
+ Connector2KbService.filter_delete([Connector2Kb.kb_id == kb_id])
+ SyncLogsService.filter_delete([SyncLogs.kb_id == kb_id])
+
+ # Sweep anything the per-document loop could not see, including rows
+ # written by a sync that was already in flight when deletion started.
+ stranded = DocumentService.filter_delete([Document.kb_id == kb_id])
+ if stranded:
+ logging.warning("delete_datasets: removed %s stranded document rows for dataset %s", stranded, kb_id)
+
success_count += 1
if not errors:
diff --git a/api/db/services/connector_service.py b/api/db/services/connector_service.py
index b940e6e663..8d6e10edbb 100644
--- a/api/db/services/connector_service.py
+++ b/api/db/services/connector_service.py
@@ -54,6 +54,47 @@ def _append_text_expr(field, suffix: str):
return fn.COALESCE(field + suffix, suffix)
+def connector_doc_id_candidates(kb_id: str, connector_id: str, external_id: str) -> tuple[str, str, str]:
+ """Every document id a connector-sourced document may legitimately carry.
+
+ A synced document's primary key is derived from its external id so that
+ re-running a sync updates the existing row instead of duplicating it. Two
+ historical derivations left ``kb_id`` out, which made the key identical for
+ every knowledge base linked to the same data source:
+
+ <= v0.25.1 hash128(external_id)
+ legacy hash128(f"{connector_id}:{external_id}")
+ current hash128(f"{kb_id}:{connector_id}:{external_id}")
+
+ Ordered oldest first, so a knowledge base holding rows from more than one
+ upgrade settles on the earliest id it owns instead of migrating forward
+ again on every sync.
+ """
+ return (
+ hash128(external_id),
+ hash128(f"{connector_id}:{external_id}"),
+ hash128(f"{kb_id}:{connector_id}:{external_id}"),
+ )
+
+
+def resolve_connector_doc_id(kb_id: str, connector_id: str, external_id: str, owned_doc_ids) -> str:
+ """Pick the document id to sync ``external_id`` into ``kb_id`` under.
+
+ ``owned_doc_ids`` must only contain ids this knowledge base already owns
+ (see ``DocumentService.list_doc_headers_by_kb_and_source_type``). A
+ KB-agnostic id is reused only when it resolves to one of those rows; when it
+ does not, the row it would hit belongs to some other knowledge base and
+ ``FileService.upload_document`` would reject the write as a cross-KB
+ collision and drop the document. Everything else gets the KB-scoped id,
+ which cannot collide by construction.
+ """
+ *kb_agnostic_doc_ids, scoped_doc_id = connector_doc_id_candidates(kb_id, connector_id, external_id)
+ for candidate in kb_agnostic_doc_ids:
+ if candidate in owned_doc_ids:
+ return candidate
+ return scoped_doc_id
+
+
class ConnectorService(CommonService):
model = Connector
@@ -186,7 +227,7 @@ class ConnectorService(CommonService):
return 0, []
source_type = f"{conn.source}/{conn.id}"
- retain_doc_ids = {doc_id for file in file_list for doc_id in (hash128(f"{connector_id}:{file.id}"), hash128(f"{kb_id}:{connector_id}:{file.id}"))}
+ retain_doc_ids = {doc_id for file in file_list for doc_id in connector_doc_id_candidates(kb_id, connector_id, file.id)}
existing_docs = DocumentService.list_doc_headers_by_kb_and_source_type(
kb_id,
source_type,
diff --git a/api/db/services/file_service.py b/api/db/services/file_service.py
index 5af5b04f1c..e92b505131 100644
--- a/api/db/services/file_service.py
+++ b/api/db/services/file_service.py
@@ -537,6 +537,43 @@ class FileService(CommonService):
logger.exception("move_file")
raise RuntimeError("Database error (File move)!")
+ @classmethod
+ def _discard_orphaned_document(cls, doc) -> bool:
+ """Drop a document stranded by a deleted knowledge base, and its debris.
+
+ Connector syncs derive document ids from the external document, so a row
+ stranded this way keeps answering ``get_by_id`` and blocks that document
+ from ever being ingested again -- while being invisible to the user,
+ because the knowledge base it names is gone. Returns whether it was
+ removed.
+
+ Mirrors the teardown ``delete_docs`` performs, minus the chunk work:
+ the chunks went with the index dropped at dataset deletion, and the
+ document's tenant is no longer resolvable through its knowledge base,
+ so there is no index left to address. Storage and row cleanup are
+ best-effort -- the point is to unblock ingestion, so debris that cannot
+ be reached must not resurrect the collision.
+ """
+ if KnowledgebaseService.get_or_none(id=doc.kb_id) is not None:
+ return False
+
+ logger.warning("Discarding orphaned document %s: its kb_id=%s no longer exists.", doc.id, doc.kb_id)
+ try:
+ bucket, location = File2DocumentService.get_storage_address(doc_id=doc.id)
+ TaskService.filter_delete([Task.doc_id == doc.id])
+ f2d = File2DocumentService.get_by_document_id(doc.id)
+ deleted_file_count = 0
+ if f2d:
+ deleted_file_count = cls.filter_delete([File.source_type == FileSource.KNOWLEDGEBASE, File.id == f2d[0].file_id])
+ File2DocumentService.delete_by_document_id(doc.id)
+ if deleted_file_count > 0:
+ settings.STORAGE_IMPL.rm(bucket, location)
+ except Exception:
+ logger.exception("Failed to fully clean up orphaned document %s; removing the row anyway", doc.id)
+
+ DocumentService.delete_by_id(doc.id)
+ return True
+
@classmethod
@DB.connection_context()
def upload_document(self, kb, file_objs, user_id, src="local", parent_path: str | None = None, parser_config_override: dict | None = None):
@@ -559,18 +596,21 @@ class FileService(CommonService):
for file in file_objs:
doc_id = file.id if hasattr(file, "id") else get_uuid()
e, doc = DocumentService.get_by_id(doc_id)
+ if e and str(doc.kb_id) != str(kb.id):
+ if not self._discard_orphaned_document(doc):
+ logger.warning(
+ "Existing document id collision detected for %s: belongs to kb_id=%s, incoming kb_id=%s. Skipping update to avoid cross-KB overwrite.",
+ doc_id,
+ doc.kb_id,
+ kb.id,
+ )
+ user_msg = f"Existing document id collision with knowledge base '{doc.kb_id}'; skipping update."
+ err.append(file.filename + ": " + user_msg)
+ continue
+ # The stranded row is gone; ingest as a fresh document.
+ e, doc = False, None
if e:
try:
- if str(doc.kb_id) != str(kb.id):
- logger.warning(
- "Existing document id collision detected for %s: belongs to kb_id=%s, incoming kb_id=%s. Skipping update to avoid cross-KB overwrite.",
- doc_id,
- doc.kb_id,
- kb.id,
- )
- user_msg = "Existing document id collision with another knowledge base; skipping update."
- err.append(file.filename + ": " + user_msg)
- continue
blob = file.read()
# Connector-supplied fingerprint (e.g. xxhash128(S3 ETag))
# takes precedence: for connector-sourced docs the bypass
diff --git a/rag/svr/sync_data_source.py b/rag/svr/sync_data_source.py
index 5f13ae4ae1..a7a847e934 100644
--- a/rag/svr/sync_data_source.py
+++ b/rag/svr/sync_data_source.py
@@ -37,8 +37,7 @@ from typing import Any
from flask import json
-from api.utils.common import hash128
-from api.db.services.connector_service import ConnectorService, SyncLogsService
+from api.db.services.connector_service import ConnectorService, SyncLogsService, resolve_connector_doc_id
from api.db.services.document_service import DocumentService
from api.db.services.knowledgebase_service import KnowledgebaseService
from common import settings
@@ -232,10 +231,8 @@ class SyncBase:
docs = []
for doc in document_batch:
- legacy_doc_id = hash128(f"{task['connector_id']}:{doc.id}")
- new_doc_id = hash128(f"{task['kb_id']}:{task['connector_id']}:{doc.id}")
d = {
- "id": legacy_doc_id if legacy_doc_id in existing_doc_ids else new_doc_id,
+ "id": resolve_connector_doc_id(task["kb_id"], task["connector_id"], doc.id, existing_doc_ids),
"connector_id": task["connector_id"],
"source": self.SOURCE_NAME,
"semantic_identifier": doc.semantic_identifier,
@@ -392,9 +389,8 @@ class _BlobLikeBase(SyncBase):
if key_record.deleted:
continue
- legacy_doc_id = hash128(f"{task['connector_id']}:{key_record.key}")
- new_doc_id = hash128(f"{task['kb_id']}:{task['connector_id']}:{key_record.key}")
- stored = existing_fingerprints.get(legacy_doc_id, "") or existing_fingerprints.get(new_doc_id, "")
+ doc_id = resolve_connector_doc_id(task["kb_id"], task["connector_id"], key_record.key, existing_fingerprints)
+ stored = existing_fingerprints.get(doc_id, "")
if key_record.fingerprint and stored and key_record.fingerprint == stored:
bypass_count += 1
continue
diff --git a/test/unit_test/api/apps/services/test_dataset_api_service_list_datasets.py b/test/unit_test/api/apps/services/test_dataset_api_service_list_datasets.py
index f30d20aac1..52c5d4d197 100644
--- a/test/unit_test/api/apps/services/test_dataset_api_service_list_datasets.py
+++ b/test/unit_test/api/apps/services/test_dataset_api_service_list_datasets.py
@@ -96,6 +96,7 @@ def _load_list_datasets_module(monkeypatch, *, kbs, parsing_status_by_kb):
FileSource=SimpleNamespace(KNOWLEDGEBASE="knowledgebase"),
PipelineTaskType=SimpleNamespace(),
StatusEnum=SimpleNamespace(),
+ TaskStatus=SimpleNamespace(SCHEDULE="schedule", RUNNING="running", CANCEL="cancel"),
RetCode=SimpleNamespace(),
ModelTypeBinary=_StubModelTypeBinary,
)
@@ -107,7 +108,10 @@ def _load_list_datasets_module(monkeypatch, *, kbs, parsing_status_by_kb):
_stub(
monkeypatch,
"api.db.db_models",
+ Connector2Kb=SimpleNamespace(kb_id="kb_id"),
+ Document=SimpleNamespace(kb_id="kb_id"),
File=SimpleNamespace(),
+ SyncLogs=SimpleNamespace(kb_id="kb_id"),
)
_stub(
monkeypatch,
@@ -140,6 +144,7 @@ def _load_list_datasets_module(monkeypatch, *, kbs, parsing_status_by_kb):
monkeypatch,
"api.db.services.connector_service",
Connector2KbService=SimpleNamespace(),
+ SyncLogsService=SimpleNamespace(),
)
_stub(
monkeypatch,
diff --git a/test/unit_test/api/apps/services/test_delete_datasets.py b/test/unit_test/api/apps/services/test_delete_datasets.py
index 12862fcc5a..0ba30ce741 100644
--- a/test/unit_test/api/apps/services/test_delete_datasets.py
+++ b/test/unit_test/api/apps/services/test_delete_datasets.py
@@ -50,8 +50,16 @@ def _stub(monkeypatch, name, **attrs):
return mod
-def _load_delete_datasets_module(monkeypatch, *, f2d_rows, file_filter_delete):
+def _load_delete_datasets_module(monkeypatch, *, f2d_rows, file_filter_delete, stranded_doc_rows=0):
f2d_delete = MagicMock()
+ calls = []
+ cleanup = SimpleNamespace(
+ calls=calls,
+ doc_filter_delete=MagicMock(side_effect=lambda *_a, **_k: (calls.append("sweep_documents"), stranded_doc_rows)[1]),
+ connector2kb_filter_delete=MagicMock(side_effect=lambda *_a, **_k: calls.append("unlink_connectors")),
+ sync_logs_filter_delete=MagicMock(side_effect=lambda *_a, **_k: calls.append("delete_sync_logs")),
+ sync_logs_filter_update=MagicMock(side_effect=lambda *_a, **_k: calls.append("cancel_running_syncs")),
+ )
kb = SimpleNamespace(id="kb-1", tenant_id="tenant-1", name="test-kb")
doc = SimpleNamespace(id="doc-1")
@@ -61,6 +69,7 @@ def _load_delete_datasets_module(monkeypatch, *, f2d_rows, file_filter_delete):
DocumentService=SimpleNamespace(
query=lambda kb_id: [doc],
remove_document=lambda doc, tenant_id: True,
+ filter_delete=cleanup.doc_filter_delete,
),
queue_raptor_o_graphrag_tasks=MagicMock(),
)
@@ -81,8 +90,8 @@ def _load_delete_datasets_module(monkeypatch, *, f2d_rows, file_filter_delete):
monkeypatch,
"api.db.services.knowledgebase_service",
KnowledgebaseService=SimpleNamespace(
- get_or_none=lambda id, tenant_id: kb,
- delete_by_id=lambda kb_id: True,
+ get_or_none=lambda **_kwargs: kb,
+ delete_by_id=lambda kb_id: calls.append("delete_dataset") is None,
query=lambda **kwargs: [],
),
validate_dataset_embedding_models=lambda kbs: None,
@@ -90,7 +99,11 @@ def _load_delete_datasets_module(monkeypatch, *, f2d_rows, file_filter_delete):
_stub(
monkeypatch,
"api.db.services.connector_service",
- Connector2KbService=SimpleNamespace(),
+ Connector2KbService=SimpleNamespace(filter_delete=cleanup.connector2kb_filter_delete),
+ SyncLogsService=SimpleNamespace(
+ filter_delete=cleanup.sync_logs_filter_delete,
+ filter_update=cleanup.sync_logs_filter_update,
+ ),
)
_stub(
monkeypatch,
@@ -136,7 +149,10 @@ def _load_delete_datasets_module(monkeypatch, *, f2d_rows, file_filter_delete):
"api.db.db_models",
DB=SimpleNamespace(connection_context=lambda: lambda func: func),
TenantModel=SimpleNamespace(),
+ Connector2Kb=SimpleNamespace(kb_id="kb_id"),
+ Document=SimpleNamespace(kb_id="kb_id"),
File=SimpleNamespace(source_type="source_type", id="id", type="type", name="name"),
+ SyncLogs=SimpleNamespace(kb_id="kb_id", status=SimpleNamespace(in_=lambda _values: None)),
)
_stub(
monkeypatch,
@@ -156,6 +172,7 @@ def _load_delete_datasets_module(monkeypatch, *, f2d_rows, file_filter_delete):
StatusEnum=SimpleNamespace(),
LLMType=SimpleNamespace(),
RetCode=SimpleNamespace(),
+ TaskStatus=SimpleNamespace(SCHEDULE="schedule", RUNNING="running", CANCEL="cancel"),
ModelTypeBinary=_StubModelTypeBinary,
)
_stub(monkeypatch, "rag.advanced_rag", __path__=[])
@@ -178,14 +195,14 @@ def _load_delete_datasets_module(monkeypatch, *, f2d_rows, file_filter_delete):
module = importlib.util.module_from_spec(spec)
monkeypatch.setitem(sys.modules, "test_delete_datasets_module", module)
spec.loader.exec_module(module)
- return module, f2d_delete
+ return module, f2d_delete, cleanup
@pytest.mark.asyncio
async def test_delete_datasets_skips_file_delete_when_no_file2document(monkeypatch):
"""Documents without a File2Document row must not crash dataset deletion."""
file_filter_delete = MagicMock(return_value=0)
- module, f2d_delete = _load_delete_datasets_module(
+ module, f2d_delete, _cleanup = _load_delete_datasets_module(
monkeypatch,
f2d_rows=[],
file_filter_delete=file_filter_delete,
@@ -203,7 +220,7 @@ async def test_delete_datasets_skips_file_delete_when_no_file2document(monkeypat
async def test_delete_datasets_deletes_linked_file_when_file2document_exists(monkeypatch):
f2d_row = SimpleNamespace(file_id="file-1")
file_filter_delete = MagicMock(side_effect=[1, 0])
- module, _f2d_delete = _load_delete_datasets_module(
+ module, _f2d_delete, _cleanup = _load_delete_datasets_module(
monkeypatch,
f2d_rows=[f2d_row],
file_filter_delete=file_filter_delete,
@@ -214,3 +231,40 @@ async def test_delete_datasets_deletes_linked_file_when_file2document_exists(mon
assert ok is True
assert result == {"success_count": 1}
assert file_filter_delete.call_count == 2
+
+
+async def test_delete_datasets_unwires_connectors_and_sweeps_stranded_documents(monkeypatch):
+ """Deleting a dataset must not leave connector state pointing at it.
+
+ Regression test for #18116. Surviving Connector2Kb/SyncLogs rows keep the
+ connector scheduler queueing syncs against a kb_id that no longer resolves,
+ and a surviving document row is invisible to the user while still able to
+ trip the cross-KB id collision guard.
+ """
+ module, _f2d_delete, cleanup = _load_delete_datasets_module(
+ monkeypatch,
+ f2d_rows=[],
+ file_filter_delete=MagicMock(return_value=0),
+ stranded_doc_rows=2,
+ )
+
+ ok, result = await module.delete_datasets("tenant-1", ids=["kb-1"])
+
+ assert ok is True
+ assert result == {"success_count": 1}
+ cleanup.connector2kb_filter_delete.assert_called_once()
+ cleanup.sync_logs_filter_delete.assert_called_once()
+ cleanup.doc_filter_delete.assert_called_once()
+ cleanup.sync_logs_filter_update.assert_called_once()
+
+ # Queued syncs are cancelled while the connector mapping still exists, the
+ # mapping is dropped only once the dataset is really gone, and the sweep for
+ # rows an in-flight sync may have written runs last.
+ assert cleanup.calls == [
+ "cancel_running_syncs",
+ "delete_dataset",
+ "unlink_connectors",
+ "delete_sync_logs",
+ "sweep_documents",
+ ]
diff --git a/test/unit_test/api/apps/services/test_gaussdb_dataset_embedding.py b/test/unit_test/api/apps/services/test_gaussdb_dataset_embedding.py
index 21cc664ad1..73fef442f8 100644
--- a/test/unit_test/api/apps/services/test_gaussdb_dataset_embedding.py
+++ b/test/unit_test/api/apps/services/test_gaussdb_dataset_embedding.py
@@ -108,7 +108,14 @@ def _load_dataset_module(monkeypatch):
resolve_model_config=lambda *_args, **_kwargs: {"llm_name": "new-embd"},
resolve_model_id=lambda *_args, **_kwargs: "new-embd",
)
- _install_module(monkeypatch, "api.db.db_models", File=SimpleNamespace())
+ _install_module(
+ monkeypatch,
+ "api.db.db_models",
+ Connector2Kb=SimpleNamespace(kb_id="kb_id"),
+ Document=SimpleNamespace(kb_id="kb_id"),
+ File=SimpleNamespace(),
+ SyncLogs=SimpleNamespace(kb_id="kb_id", status=SimpleNamespace(in_=lambda _values: None)),
+ )
_install_module(
monkeypatch,
"api.db.services.document_service",
@@ -126,7 +133,12 @@ def _load_dataset_module(monkeypatch):
),
validate_dataset_embedding_models=lambda *_args, **_kwargs: None,
)
- _install_module(monkeypatch, "api.db.services.connector_service", Connector2KbService=SimpleNamespace())
+ _install_module(
+ monkeypatch,
+ "api.db.services.connector_service",
+ Connector2KbService=SimpleNamespace(),
+ SyncLogsService=SimpleNamespace(),
+ )
_install_module(
monkeypatch,
"api.db.services.task_service",
@@ -154,6 +166,7 @@ def _load_dataset_module(monkeypatch):
PAGERANK_FLD="pagerank_fea",
RetCode=SimpleNamespace(NOT_EFFECTIVE=590),
StatusEnum=SimpleNamespace(VALID=SimpleNamespace(value="1")),
+ TaskStatus=SimpleNamespace(SCHEDULE="schedule", RUNNING="running", CANCEL="cancel"),
)
_install_module(
monkeypatch,
diff --git a/test/unit_test/api/db/services/test_connector_doc_id.py b/test/unit_test/api/db/services/test_connector_doc_id.py
new file mode 100644
index 0000000000..c29159e4d4
--- /dev/null
+++ b/test/unit_test/api/db/services/test_connector_doc_id.py
@@ -0,0 +1,88 @@
+#
+# Copyright 2026 The InfiniFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+import pytest
+
+from api.db.services.connector_service import connector_doc_id_candidates, resolve_connector_doc_id
+from api.utils.common import hash128
+
+CONNECTOR_ID = "conn-1"
+EXTERNAL_ID = "JIRA-42"
+KB_A = "kb-a"
+KB_B = "kb-b"
+
+V0_ID = hash128(EXTERNAL_ID)
+LEGACY_ID = hash128(f"{CONNECTOR_ID}:{EXTERNAL_ID}")
+SCOPED_A = hash128(f"{KB_A}:{CONNECTOR_ID}:{EXTERNAL_ID}")
+SCOPED_B = hash128(f"{KB_B}:{CONNECTOR_ID}:{EXTERNAL_ID}")
+
+
+def test_candidates_cover_every_historical_scheme():
+ assert connector_doc_id_candidates(KB_A, CONNECTOR_ID, EXTERNAL_ID) == (V0_ID, LEGACY_ID, SCOPED_A)
+
+
+def test_first_sync_uses_kb_scoped_id():
+ assert resolve_connector_doc_id(KB_A, CONNECTOR_ID, EXTERNAL_ID, set()) == SCOPED_A
+
+
[email protected]("owned_id", [LEGACY_ID, V0_ID])
+def test_kb_agnostic_id_is_reused_when_this_kb_owns_it(owned_id):
+ """Re-syncing an upgraded instance updates its rows instead of duplicating them."""
+ assert resolve_connector_doc_id(KB_A, CONNECTOR_ID, EXTERNAL_ID, {owned_id}) == owned_id
+
+
[email protected]("foreign_id", [LEGACY_ID, V0_ID])
+def test_kb_agnostic_id_owned_elsewhere_is_never_reused(foreign_id):
+ """Regression test for #18116.
+
+ ``owned_doc_ids`` lists only what this knowledge base already holds, so an
+ id absent from it may well exist under a different knowledge base. Handing
+ that id to ``FileService.upload_document`` is what produced "Existing
+ document id collision with another knowledge base; skipping update." and
+ dropped the document. KB B must route around it.
+ """
+ kb_a_owns = {foreign_id}
+ kb_b_owns = set()
+
+ assert resolve_connector_doc_id(KB_A, CONNECTOR_ID, EXTERNAL_ID, kb_a_owns) == foreign_id
+ assert resolve_connector_doc_id(KB_B, CONNECTOR_ID, EXTERNAL_ID, kb_b_owns) == SCOPED_B
+ assert SCOPED_B != foreign_id
+
+
+def test_oldest_owned_id_wins_when_several_upgrades_left_rows_behind():
+ """An instance carrying rows from more than one upgrade settles, not drifts.
+
+ Preferring the newest owned id would re-key the document on every upgrade
+ and strand the older row again each time.
+ """
+ assert resolve_connector_doc_id(KB_A, CONNECTOR_ID, EXTERNAL_ID, {V0_ID, LEGACY_ID}) == V0_ID
+
+
+def test_two_kbs_on_one_connector_never_share_an_id():
+ assert resolve_connector_doc_id(KB_A, CONNECTOR_ID, EXTERNAL_ID, set()) != resolve_connector_doc_id(KB_B, CONNECTOR_ID, EXTERNAL_ID, set())
+
+
+def test_resolution_is_stable_across_runs():
+ """The second run sees the id the first run wrote and settles on it."""
+ first = resolve_connector_doc_id(KB_A, CONNECTOR_ID, EXTERNAL_ID, set())
+ second = resolve_connector_doc_id(KB_A, CONNECTOR_ID, EXTERNAL_ID, {first})
+ assert first == second == SCOPED_A
diff --git a/test/unit_test/api/db/services/test_file_service_upload_document.py b/test/unit_test/api/db/services/test_file_service_upload_document.py
index de8c65d4ea..655f1f1476 100644
--- a/test/unit_test/api/db/services/test_file_service_upload_document.py
+++ b/test/unit_test/api/db/services/test_file_service_upload_document.py
@@ -70,21 +70,40 @@ from api.db.services.file_service import FileService # noqa: E402
class _DummyUploadFile:
- def __init__(self, filename, doc_id):
+ def __init__(self, filename, doc_id, blob=None):
self.filename = filename
self.id = doc_id
+ self._blob = blob
def read(self):
- raise AssertionError("read() should not be called for cross-KB collision path")
+ if self._blob is None:
+ raise AssertionError("read() should not be called for cross-KB collision path")
+ return self._blob
+
+
+class _DummyStorage:
+ """Storage double recording writes without touching MinIO."""
+
+ def __init__(self):
+ self.written = {}
+ self.removed = []
+
+ def obj_exist(self, _bucket, _location):
+ return False
+
+ def put(self, bucket, location, blob, *_args):
+ self.written[(bucket, location)] = blob
+
+ def rm(self, bucket, location):
+ self.removed.append((bucket, location))
def _unwrapped_upload_document():
return FileService.upload_document.__func__.__wrapped__
-def test_upload_document_skips_cross_kb_document_id_collision(monkeypatch):
- kb = SimpleNamespace(
+def _target_kb():
+ return SimpleNamespace(
id="kb-target",
tenant_id="tenant-1",
name="Target KB",
@@ -92,14 +111,9 @@ def test_upload_document_skips_cross_kb_document_id_collision(monkeypatch):
pipeline_id=None,
parser_config={},
)
- existing_doc = SimpleNamespace(
- id="doc-1",
- kb_id="kb-other",
- location="old-location.txt",
- content_hash="old-hash",
- to_dict=lambda: {"id": "doc-1"},
- )
+
+def _stub_folder_lookups(monkeypatch):
monkeypatch.setattr(FileService, "get_root_folder", classmethod(lambda cls, _uid: {"id": "root"}))
monkeypatch.setattr(FileService, "init_knowledgebase_docs", classmethod(lambda cls, _pf_id, _uid: None))
monkeypatch.setattr(FileService, "get_kb_folder", classmethod(lambda cls, _uid: {"id": "kb-root"}))
@@ -108,7 +122,26 @@ def test_upload_document_skips_cross_kb_document_id_collision(monkeypatch):
"new_a_file_from_kb",
classmethod(lambda cls, _tenant_id, _name, _parent_id: {"id": "kb-folder"}),
)
+
+
+def test_upload_document_skips_cross_kb_document_id_collision(monkeypatch):
+ kb = _target_kb()
+ existing_doc = SimpleNamespace(
+ id="doc-1",
+ kb_id="kb-other",
+ location="old-location.txt",
+ content_hash="old-hash",
+ to_dict=lambda: {"id": "doc-1"},
+ )
+
+ _stub_folder_lookups(monkeypatch)
monkeypatch.setattr(file_service_module.DocumentService, "get_by_id", lambda _doc_id: (True, existing_doc))
+ monkeypatch.setattr(
+ file_service_module.KnowledgebaseService,
+ "get_or_none",
+ classmethod(lambda cls, **_kwargs: SimpleNamespace(id="kb-other")),
+ )
err, files = _unwrapped_upload_document()(
FileService,
@@ -120,7 +153,98 @@ def test_upload_document_skips_cross_kb_document_id_collision(monkeypatch):
assert files == []
assert len(err) == 1
assert err[0].startswith("collision.txt: ")
- assert "Existing document id collision with another knowledge base; skipping update." in err[0]
+ # The owning knowledge base is named so the conflict can actually be found.
+ assert "Existing document id collision with knowledge base 'kb-other'; skipping update." in err[0]
+
+
+def test_upload_document_reclaims_document_stranded_by_deleted_kb(monkeypatch):
+ """A row whose knowledge base is gone must not block ingestion forever.
+
+ Regression test for #18116: the stranded row is unreachable through the UI
+ and the API, so treating it as a live conflict permanently blocked the
+ document from syncing anywhere.
+ """
+ kb = _target_kb()
+ stranded_doc = SimpleNamespace(
+ id="doc-1",
+ kb_id="kb-deleted",
+ location="old-location.txt",
+ content_hash="old-hash",
+ to_dict=lambda: {"id": "doc-1"},
+ )
+ storage = _DummyStorage()
+ discarded = []
+ inserted = []
+ tasks_deleted = []
+ files_deleted = []
+ mappings_deleted = []
+
+ _stub_folder_lookups(monkeypatch)
+ monkeypatch.setattr(file_service_module.DocumentService, "get_by_id", lambda _doc_id: (True, stranded_doc))
+ monkeypatch.setattr(
+ file_service_module.KnowledgebaseService,
+ "get_or_none",
+ classmethod(lambda cls, **kwargs: None if kwargs.get("id") == "kb-deleted" else SimpleNamespace(id=kwargs.get("id"))),
+ )
+ monkeypatch.setattr(
+ file_service_module.DocumentService,
+ "delete_by_id",
+ classmethod(lambda cls, doc_id: discarded.append(doc_id)),
+ )
+ monkeypatch.setattr(
+ file_service_module.File2DocumentService,
+ "delete_by_document_id",
+ classmethod(lambda cls, doc_id: mappings_deleted.append(doc_id)),
+ )
+ monkeypatch.setattr(
+ file_service_module.File2DocumentService,
+ "get_storage_address",
+ classmethod(lambda cls, **_kwargs: ("kb-deleted", "old-location.txt")),
+ )
+ monkeypatch.setattr(
+ file_service_module.File2DocumentService,
+ "get_by_document_id",
+ classmethod(lambda cls, _doc_id: [SimpleNamespace(file_id="file-1")]),
+ )
+ monkeypatch.setattr(
+ file_service_module.TaskService,
+ "filter_delete",
+ classmethod(lambda cls, _filters: tasks_deleted.append("doc-1")),
+ )
+ monkeypatch.setattr(
+ FileService,
+ "filter_delete",
+ classmethod(lambda cls, _filters: files_deleted.append("file-1") or 1),
+ )
+ monkeypatch.setattr(file_service_module.DocumentService, "check_doc_health", classmethod(lambda cls, _tid, _name: None))
+ monkeypatch.setattr(file_service_module.DocumentService, "query", classmethod(lambda cls, **_kwargs: []))
+ monkeypatch.setattr(file_service_module.DocumentService, "insert", classmethod(lambda cls, doc: inserted.append(doc)))
+ monkeypatch.setattr(FileService, "add_file_from_kb", classmethod(lambda cls, _doc, _folder_id, _tenant_id: None))
+ monkeypatch.setattr(file_service_module, "thumbnail_img", lambda _name, _blob: None)
+ monkeypatch.setattr(file_service_module.settings, "STORAGE_IMPL", storage)
+
+ err, files = _unwrapped_upload_document()(
+ FileService,
+ kb,
+ [_DummyUploadFile(filename="collision.txt", doc_id="doc-1", blob=b"payload")],
+ "user-1",
+ )
+
+ assert err == []
+ assert discarded == ["doc-1"]
+ # The stranded row's debris goes with it, matching what delete_docs would
+ # have removed had the dataset deletion reached this document.
+ assert tasks_deleted == ["doc-1"]
+ assert files_deleted == ["file-1"]
+ assert mappings_deleted == ["doc-1"]
+ assert storage.removed == [("kb-deleted", "old-location.txt")]
+ assert len(inserted) == 1
+ assert inserted[0]["id"] == "doc-1"
+ assert inserted[0]["kb_id"] == "kb-target"
+ assert len(files) == 1
+ assert files[0][1] == b"payload"
+ assert storage.written[("kb-target", "collision.txt")] == b"payload"
# ---------------------------------------------------------------------------