RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.
Go agent-harnessagent-native-databaseagentic-aiagentic-retrievalagentic-searchaiai-agentscontext-enginecontext-engineeringcontext-managementharness-engineeringknowledge-compilationragretrieval-augmented-generationretrieval-harnesssearch-harness
Fix: cross-KB document id collision in data source sync (#18116) (#18244)
Mpider-Sanaa57022bfdb1
diff --git a/api/apps/services/dataset_api_service.py b/api/apps/services/dataset_api_service.pyindex 914b21cb54..aa6ba965e1 100644--- a/api/apps/services/dataset_api_service.py+++ b/api/apps/services/dataset_api_service.py@@ -18,9 +18,9 @@ import loggingimport osimport re-from api.db.db_models import File+from api.db.db_models import Connector2Kb, Document, File, SyncLogsfrom api.db.joint_services.tenant_model_service import get_composite_model_name_by_ids, resolve_model_config, resolve_model_id-from api.db.services.connector_service import Connector2KbService+from api.db.services.connector_service import Connector2KbService, SyncLogsServicefrom api.db.services.document_service import DocumentService, queue_raptor_o_graphrag_tasksfrom api.db.services.file2document_service import File2DocumentServicefrom api.db.services.file_service import FileService@@ -30,7 +30,7 @@ from api.db.services.tenant_model_service import TenantModelServicefrom api.db.services.user_service import TenantService, UserService, UserTenantServicefrom api.utils.api_utils import deep_merge, get_parser_config, remap_dictionary_keys, verify_embedding_availabilityfrom common import settings-from common.constants import PAGERANK_FLD, FileSource, LLMType, RetCode, StatusEnum+from common.constants import PAGERANK_FLD, FileSource, LLMType, RetCode, StatusEnum, TaskStatusfrom common.misc_utils import thread_pool_exec, thread_pool_exec_long_timefrom rag.advanced_rag.knowlege_compile.wiki import WIKI_PAGE_COMPILE_KWD@@ -171,6 +171,15 @@ def _delete_datasets_sync(tenant_id: str, ids: list = None, delete_all: bool = Ferrors = []success_count = 0for kb_id, kb in kb_id_instance_pairs:+ # Cancel this dataset's queued syncs before touching its documents.+ # Tasks are only picked up while they are SCHEDULE, so cancelling stops+ # every run that has not started yet; a sync already in flight is not+ # interruptible, which is what the stranded-row sweep below covers.+ SyncLogsService.filter_update(+ [SyncLogs.kb_id == kb_id, SyncLogs.status.in_([TaskStatus.SCHEDULE, TaskStatus.RUNNING])],+ {"status": TaskStatus.CANCEL},+ )+for doc in DocumentService.query(kb_id=kb_id):if not DocumentService.remove_document(doc, tenant_id):errors.append(f"Remove document '{doc.id}' error for dataset '{kb_id}'")@@ -207,6 +216,22 @@ def _delete_datasets_sync(tenant_id: str, ids: list = None, delete_all: bool = Fif not KnowledgebaseService.delete_by_id(kb_id):errors.append(f"Delete dataset error for {kb_id}")continue++ # Unwire the data sources only once the dataset is really gone, so a+ # failed deletion above leaves a dataset that is still linked and still+ # syncable. Left behind, these rows keep the connector scheduler queueing+ # syncs against a kb_id that no longer resolves, and any document such a+ # run writes outlives its dataset -- an invisible row that later reports+ # a cross-KB id collision against whatever dataset is linked next.+ Connector2KbService.filter_delete([Connector2Kb.kb_id == kb_id])+ SyncLogsService.filter_delete([SyncLogs.kb_id == kb_id])++ # Sweep anything the per-document loop could not see, including rows+ # written by a sync that was already in flight when deletion started.+ stranded = DocumentService.filter_delete([Document.kb_id == kb_id])+ if stranded:+ logging.warning("delete_datasets: removed %s stranded document rows for dataset %s", stranded, kb_id)+success_count += 1if not errors:diff --git a/api/db/services/connector_service.py b/api/db/services/connector_service.pyindex b940e6e663..8d6e10edbb 100644--- a/api/db/services/connector_service.py+++ b/api/db/services/connector_service.py@@ -54,6 +54,47 @@ def _append_text_expr(field, suffix: str):return fn.COALESCE(field + suffix, suffix)+def connector_doc_id_candidates(kb_id: str, connector_id: str, external_id: str) -> tuple[str, str, str]:+ """Every document id a connector-sourced document may legitimately carry.++ A synced document's primary key is derived from its external id so that+ re-running a sync updates the existing row instead of duplicating it. Two+ historical derivations left ``kb_id`` out, which made the key identical for+ every knowledge base linked to the same data source:++ <= v0.25.1 hash128(external_id)+ legacy hash128(f"{connector_id}:{external_id}")+ current hash128(f"{kb_id}:{connector_id}:{external_id}")++ Ordered oldest first, so a knowledge base holding rows from more than one+ upgrade settles on the earliest id it owns instead of migrating forward+ again on every sync.+ """+ return (+ hash128(external_id),+ hash128(f"{connector_id}:{external_id}"),+ hash128(f"{kb_id}:{connector_id}:{external_id}"),+ )+++def resolve_connector_doc_id(kb_id: str, connector_id: str, external_id: str, owned_doc_ids) -> str:+ """Pick the document id to sync ``external_id`` into ``kb_id`` under.++ ``owned_doc_ids`` must only contain ids this knowledge base already owns+ (see ``DocumentService.list_doc_headers_by_kb_and_source_type``). A+ KB-agnostic id is reused only when it resolves to one of those rows; when it+ does not, the row it would hit belongs to some other knowledge base and+ ``FileService.upload_document`` would reject the write as a cross-KB+ collision and drop the document. Everything else gets the KB-scoped id,+ which cannot collide by construction.+ """+ *kb_agnostic_doc_ids, scoped_doc_id = connector_doc_id_candidates(kb_id, connector_id, external_id)+ for candidate in kb_agnostic_doc_ids:+ if candidate in owned_doc_ids:+ return candidate+ return scoped_doc_id++class ConnectorService(CommonService):model = Connector@@ -186,7 +227,7 @@ class ConnectorService(CommonService):return 0, []source_type = f"{conn.source}/{conn.id}"- retain_doc_ids = {doc_id for file in file_list for doc_id in (hash128(f"{connector_id}:{file.id}"), hash128(f"{kb_id}:{connector_id}:{file.id}"))}+ retain_doc_ids = {doc_id for file in file_list for doc_id in connector_doc_id_candidates(kb_id, connector_id, file.id)}existing_docs = DocumentService.list_doc_headers_by_kb_and_source_type(kb_id,source_type,diff --git a/api/db/services/file_service.py b/api/db/services/file_service.pyindex 5af5b04f1c..e92b505131 100644--- a/api/db/services/file_service.py+++ b/api/db/services/file_service.py@@ -537,6 +537,43 @@ class FileService(CommonService):logger.exception("move_file")raise RuntimeError("Database error (File move)!")+ @classmethod+ def _discard_orphaned_document(cls, doc) -> bool:+ """Drop a document stranded by a deleted knowledge base, and its debris.++ Connector syncs derive document ids from the external document, so a row+ stranded this way keeps answering ``get_by_id`` and blocks that document+ from ever being ingested again -- while being invisible to the user,+ because the knowledge base it names is gone. Returns whether it was+ removed.++ Mirrors the teardown ``delete_docs`` performs, minus the chunk work:+ the chunks went with the index dropped at dataset deletion, and the+ document's tenant is no longer resolvable through its knowledge base,+ so there is no index left to address. Storage and row cleanup are+ best-effort -- the point is to unblock ingestion, so debris that cannot+ be reached must not resurrect the collision.+ """+ if KnowledgebaseService.get_or_none(id=doc.kb_id) is not None:+ return False++ logger.warning("Discarding orphaned document %s: its kb_id=%s no longer exists.", doc.id, doc.kb_id)+ try:+ bucket, location = File2DocumentService.get_storage_address(doc_id=doc.id)+ TaskService.filter_delete([Task.doc_id == doc.id])+ f2d = File2DocumentService.get_by_document_id(doc.id)+ deleted_file_count = 0+ if f2d:+ deleted_file_count = cls.filter_delete([File.source_type == FileSource.KNOWLEDGEBASE, File.id == f2d[0].file_id])+ File2DocumentService.delete_by_document_id(doc.id)+ if deleted_file_count > 0:+ settings.STORAGE_IMPL.rm(bucket, location)+ except Exception:+ logger.exception("Failed to fully clean up orphaned document %s; removing the row anyway", doc.id)++ DocumentService.delete_by_id(doc.id)+ return True+@classmethod@DB.connection_context()def upload_document(self, kb, file_objs, user_id, src="local", parent_path: str | None = None, parser_config_override: dict | None = None):@@ -559,18 +596,21 @@ class FileService(CommonService):for file in file_objs:doc_id = file.id if hasattr(file, "id") else get_uuid()e, doc = DocumentService.get_by_id(doc_id)+ if e and str(doc.kb_id) != str(kb.id):+ if not self._discard_orphaned_document(doc):+ logger.warning(+ "Existing document id collision detected for %s: belongs to kb_id=%s, incoming kb_id=%s. Skipping update to avoid cross-KB overwrite.",+ doc_id,+ doc.kb_id,+ kb.id,+ )+ user_msg = f"Existing document id collision with knowledge base '{doc.kb_id}'; skipping update."+ err.append(file.filename + ": " + user_msg)+ continue+ # The stranded row is gone; ingest as a fresh document.+ e, doc = False, Noneif e:try:- if str(doc.kb_id) != str(kb.id):- logger.warning(- "Existing document id collision detected for %s: belongs to kb_id=%s, incoming kb_id=%s. Skipping update to avoid cross-KB overwrite.",- doc_id,- doc.kb_id,- kb.id,- )- user_msg = "Existing document id collision with another knowledge base; skipping update."- err.append(file.filename + ": " + user_msg)- continueblob = file.read()# Connector-supplied fingerprint (e.g. xxhash128(S3 ETag))# takes precedence: for connector-sourced docs the bypassdiff --git a/rag/svr/sync_data_source.py b/rag/svr/sync_data_source.pyindex 5f13ae4ae1..a7a847e934 100644--- a/rag/svr/sync_data_source.py+++ b/rag/svr/sync_data_source.py@@ -37,8 +37,7 @@ from typing import Anyfrom flask import json-from api.utils.common import hash128-from api.db.services.connector_service import ConnectorService, SyncLogsService+from api.db.services.connector_service import ConnectorService, SyncLogsService, resolve_connector_doc_idfrom api.db.services.document_service import DocumentServicefrom api.db.services.knowledgebase_service import KnowledgebaseServicefrom common import settings@@ -232,10 +231,8 @@ class SyncBase:docs = []for doc in document_batch:- legacy_doc_id = hash128(f"{task['connector_id']}:{doc.id}")- new_doc_id = hash128(f"{task['kb_id']}:{task['connector_id']}:{doc.id}")d = {- "id": legacy_doc_id if legacy_doc_id in existing_doc_ids else new_doc_id,+ "id": resolve_connector_doc_id(task["kb_id"], task["connector_id"], doc.id, existing_doc_ids),"connector_id": task["connector_id"],"source": self.SOURCE_NAME,"semantic_identifier": doc.semantic_identifier,@@ -392,9 +389,8 @@ class _BlobLikeBase(SyncBase):if key_record.deleted:continue- legacy_doc_id = hash128(f"{task['connector_id']}:{key_record.key}")- new_doc_id = hash128(f"{task['kb_id']}:{task['connector_id']}:{key_record.key}")- stored = existing_fingerprints.get(legacy_doc_id, "") or existing_fingerprints.get(new_doc_id, "")+ doc_id = resolve_connector_doc_id(task["kb_id"], task["connector_id"], key_record.key, existing_fingerprints)+ stored = existing_fingerprints.get(doc_id, "")if key_record.fingerprint and stored and key_record.fingerprint == stored:bypass_count += 1continuediff --git a/test/unit_test/api/apps/services/test_dataset_api_service_list_datasets.py b/test/unit_test/api/apps/services/test_dataset_api_service_list_datasets.pyindex f30d20aac1..52c5d4d197 100644--- a/test/unit_test/api/apps/services/test_dataset_api_service_list_datasets.py+++ b/test/unit_test/api/apps/services/test_dataset_api_service_list_datasets.py@@ -96,6 +96,7 @@ def _load_list_datasets_module(monkeypatch, *, kbs, parsing_status_by_kb):FileSource=SimpleNamespace(KNOWLEDGEBASE="knowledgebase"),PipelineTaskType=SimpleNamespace(),StatusEnum=SimpleNamespace(),+ TaskStatus=SimpleNamespace(SCHEDULE="schedule", RUNNING="running", CANCEL="cancel"),RetCode=SimpleNamespace(),ModelTypeBinary=_StubModelTypeBinary,)@@ -107,7 +108,10 @@ def _load_list_datasets_module(monkeypatch, *, kbs, parsing_status_by_kb):_stub(monkeypatch,"api.db.db_models",+ Connector2Kb=SimpleNamespace(kb_id="kb_id"),+ Document=SimpleNamespace(kb_id="kb_id"),File=SimpleNamespace(),+ SyncLogs=SimpleNamespace(kb_id="kb_id"),)_stub(monkeypatch,@@ -140,6 +144,7 @@ def _load_list_datasets_module(monkeypatch, *, kbs, parsing_status_by_kb):monkeypatch,"api.db.services.connector_service",Connector2KbService=SimpleNamespace(),+ SyncLogsService=SimpleNamespace(),)_stub(monkeypatch,diff --git a/test/unit_test/api/apps/services/test_delete_datasets.py b/test/unit_test/api/apps/services/test_delete_datasets.pyindex 12862fcc5a..0ba30ce741 100644--- a/test/unit_test/api/apps/services/test_delete_datasets.py+++ b/test/unit_test/api/apps/services/test_delete_datasets.py@@ -50,8 +50,16 @@ def _stub(monkeypatch, name, **attrs):return mod-def _load_delete_datasets_module(monkeypatch, *, f2d_rows, file_filter_delete):+def _load_delete_datasets_module(monkeypatch, *, f2d_rows, file_filter_delete, stranded_doc_rows=0):f2d_delete = MagicMock()+ calls = []+ cleanup = SimpleNamespace(+ calls=calls,+ doc_filter_delete=MagicMock(side_effect=lambda *_a, **_k: (calls.append("sweep_documents"), stranded_doc_rows)[1]),+ connector2kb_filter_delete=MagicMock(side_effect=lambda *_a, **_k: calls.append("unlink_connectors")),+ sync_logs_filter_delete=MagicMock(side_effect=lambda *_a, **_k: calls.append("delete_sync_logs")),+ sync_logs_filter_update=MagicMock(side_effect=lambda *_a, **_k: calls.append("cancel_running_syncs")),+ )kb = SimpleNamespace(id="kb-1", tenant_id="tenant-1", name="test-kb")doc = SimpleNamespace(id="doc-1")@@ -61,6 +69,7 @@ def _load_delete_datasets_module(monkeypatch, *, f2d_rows, file_filter_delete):DocumentService=SimpleNamespace(query=lambda kb_id: [doc],remove_document=lambda doc, tenant_id: True,+ filter_delete=cleanup.doc_filter_delete,),queue_raptor_o_graphrag_tasks=MagicMock(),)@@ -81,8 +90,8 @@ def _load_delete_datasets_module(monkeypatch, *, f2d_rows, file_filter_delete):monkeypatch,"api.db.services.knowledgebase_service",KnowledgebaseService=SimpleNamespace(- get_or_none=lambda id, tenant_id: kb,- delete_by_id=lambda kb_id: True,+ get_or_none=lambda **_kwargs: kb,+ delete_by_id=lambda kb_id: calls.append("delete_dataset") is None,query=lambda **kwargs: [],),validate_dataset_embedding_models=lambda kbs: None,@@ -90,7 +99,11 @@ def _load_delete_datasets_module(monkeypatch, *, f2d_rows, file_filter_delete):_stub(monkeypatch,"api.db.services.connector_service",- Connector2KbService=SimpleNamespace(),+ Connector2KbService=SimpleNamespace(filter_delete=cleanup.connector2kb_filter_delete),+ SyncLogsService=SimpleNamespace(+ filter_delete=cleanup.sync_logs_filter_delete,+ filter_update=cleanup.sync_logs_filter_update,+ ),)_stub(monkeypatch,@@ -136,7 +149,10 @@ def _load_delete_datasets_module(monkeypatch, *, f2d_rows, file_filter_delete):"api.db.db_models",DB=SimpleNamespace(connection_context=lambda: lambda func: func),TenantModel=SimpleNamespace(),+ Connector2Kb=SimpleNamespace(kb_id="kb_id"),+ Document=SimpleNamespace(kb_id="kb_id"),File=SimpleNamespace(source_type="source_type", id="id", type="type", name="name"),+ SyncLogs=SimpleNamespace(kb_id="kb_id", status=SimpleNamespace(in_=lambda _values: None)),)_stub(monkeypatch,@@ -156,6 +172,7 @@ def _load_delete_datasets_module(monkeypatch, *, f2d_rows, file_filter_delete):StatusEnum=SimpleNamespace(),LLMType=SimpleNamespace(),RetCode=SimpleNamespace(),+ TaskStatus=SimpleNamespace(SCHEDULE="schedule", RUNNING="running", CANCEL="cancel"),ModelTypeBinary=_StubModelTypeBinary,)_stub(monkeypatch, "rag.advanced_rag", __path__=[])@@ -178,14 +195,14 @@ def _load_delete_datasets_module(monkeypatch, *, f2d_rows, file_filter_delete):module = importlib.util.module_from_spec(spec)monkeypatch.setitem(sys.modules, "test_delete_datasets_module", module)spec.loader.exec_module(module)- return module, f2d_delete+ return module, f2d_delete, cleanup@pytest.mark.asyncioasync def test_delete_datasets_skips_file_delete_when_no_file2document(monkeypatch):"""Documents without a File2Document row must not crash dataset deletion."""file_filter_delete = MagicMock(return_value=0)- module, f2d_delete = _load_delete_datasets_module(+ module, f2d_delete, _cleanup = _load_delete_datasets_module(monkeypatch,f2d_rows=[],file_filter_delete=file_filter_delete,@@ -203,7 +220,7 @@ async def test_delete_datasets_skips_file_delete_when_no_file2document(monkeypatasync def test_delete_datasets_deletes_linked_file_when_file2document_exists(monkeypatch):f2d_row = SimpleNamespace(file_id="file-1")file_filter_delete = MagicMock(side_effect=[1, 0])- module, _f2d_delete = _load_delete_datasets_module(+ module, _f2d_delete, _cleanup = _load_delete_datasets_module(monkeypatch,f2d_rows=[f2d_row],file_filter_delete=file_filter_delete,@@ -214,3 +231,40 @@ async def test_delete_datasets_deletes_linked_file_when_file2document_exists(monassert ok is Trueassert result == {"success_count": 1}assert file_filter_delete.call_count == 2+++async def test_delete_datasets_unwires_connectors_and_sweeps_stranded_documents(monkeypatch):+ """Deleting a dataset must not leave connector state pointing at it.++ Regression test for #18116. Surviving Connector2Kb/SyncLogs rows keep the+ connector scheduler queueing syncs against a kb_id that no longer resolves,+ and a surviving document row is invisible to the user while still able to+ trip the cross-KB id collision guard.+ """+ module, _f2d_delete, cleanup = _load_delete_datasets_module(+ monkeypatch,+ f2d_rows=[],+ file_filter_delete=MagicMock(return_value=0),+ stranded_doc_rows=2,+ )++ ok, result = await module.delete_datasets("tenant-1", ids=["kb-1"])++ assert ok is True+ assert result == {"success_count": 1}+ cleanup.connector2kb_filter_delete.assert_called_once()+ cleanup.sync_logs_filter_delete.assert_called_once()+ cleanup.doc_filter_delete.assert_called_once()+ cleanup.sync_logs_filter_update.assert_called_once()++ # Queued syncs are cancelled while the connector mapping still exists, the+ # mapping is dropped only once the dataset is really gone, and the sweep for+ # rows an in-flight sync may have written runs last.+ assert cleanup.calls == [+ "cancel_running_syncs",+ "delete_dataset",+ "unlink_connectors",+ "delete_sync_logs",+ "sweep_documents",+ ]diff --git a/test/unit_test/api/apps/services/test_gaussdb_dataset_embedding.py b/test/unit_test/api/apps/services/test_gaussdb_dataset_embedding.pyindex 21cc664ad1..73fef442f8 100644--- a/test/unit_test/api/apps/services/test_gaussdb_dataset_embedding.py+++ b/test/unit_test/api/apps/services/test_gaussdb_dataset_embedding.py@@ -108,7 +108,14 @@ def _load_dataset_module(monkeypatch):resolve_model_config=lambda *_args, **_kwargs: {"llm_name": "new-embd"},resolve_model_id=lambda *_args, **_kwargs: "new-embd",)- _install_module(monkeypatch, "api.db.db_models", File=SimpleNamespace())+ _install_module(+ monkeypatch,+ "api.db.db_models",+ Connector2Kb=SimpleNamespace(kb_id="kb_id"),+ Document=SimpleNamespace(kb_id="kb_id"),+ File=SimpleNamespace(),+ SyncLogs=SimpleNamespace(kb_id="kb_id", status=SimpleNamespace(in_=lambda _values: None)),+ )_install_module(monkeypatch,"api.db.services.document_service",@@ -126,7 +133,12 @@ def _load_dataset_module(monkeypatch):),validate_dataset_embedding_models=lambda *_args, **_kwargs: None,)- _install_module(monkeypatch, "api.db.services.connector_service", Connector2KbService=SimpleNamespace())+ _install_module(+ monkeypatch,+ "api.db.services.connector_service",+ Connector2KbService=SimpleNamespace(),+ SyncLogsService=SimpleNamespace(),+ )_install_module(monkeypatch,"api.db.services.task_service",@@ -154,6 +166,7 @@ def _load_dataset_module(monkeypatch):PAGERANK_FLD="pagerank_fea",RetCode=SimpleNamespace(NOT_EFFECTIVE=590),StatusEnum=SimpleNamespace(VALID=SimpleNamespace(value="1")),+ TaskStatus=SimpleNamespace(SCHEDULE="schedule", RUNNING="running", CANCEL="cancel"),)_install_module(monkeypatch,diff --git a/test/unit_test/api/db/services/test_connector_doc_id.py b/test/unit_test/api/db/services/test_connector_doc_id.pynew file mode 100644index 0000000000..c29159e4d4--- /dev/null+++ b/test/unit_test/api/db/services/test_connector_doc_id.py@@ -0,0 +1,88 @@+#+# Copyright 2026 The InfiniFlow Authors. All Rights Reserved.+#+# Licensed under the Apache License, Version 2.0 (the "License");+# you may not use this file except in compliance with the License.+# You may obtain a copy of the License at+#+# http://www.apache.org/licenses/LICENSE-2.0+#+# Unless required by applicable law or agreed to in writing, software+# distributed under the License is distributed on an "AS IS" BASIS,+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+# See the License for the specific language governing permissions and+# limitations under the License.+#+import pytest++from api.db.services.connector_service import connector_doc_id_candidates, resolve_connector_doc_id+from api.utils.common import hash128++CONNECTOR_ID = "conn-1"+EXTERNAL_ID = "JIRA-42"+KB_A = "kb-a"+KB_B = "kb-b"++V0_ID = hash128(EXTERNAL_ID)+LEGACY_ID = hash128(f"{CONNECTOR_ID}:{EXTERNAL_ID}")+SCOPED_A = hash128(f"{KB_A}:{CONNECTOR_ID}:{EXTERNAL_ID}")+SCOPED_B = hash128(f"{KB_B}:{CONNECTOR_ID}:{EXTERNAL_ID}")+++def test_candidates_cover_every_historical_scheme():+ assert connector_doc_id_candidates(KB_A, CONNECTOR_ID, EXTERNAL_ID) == (V0_ID, LEGACY_ID, SCOPED_A)+++def test_first_sync_uses_kb_scoped_id():+ assert resolve_connector_doc_id(KB_A, CONNECTOR_ID, EXTERNAL_ID, set()) == SCOPED_A++[email protected]("owned_id", [LEGACY_ID, V0_ID])+def test_kb_agnostic_id_is_reused_when_this_kb_owns_it(owned_id):+ """Re-syncing an upgraded instance updates its rows instead of duplicating them."""+ assert resolve_connector_doc_id(KB_A, CONNECTOR_ID, EXTERNAL_ID, {owned_id}) == owned_id++[email protected]("foreign_id", [LEGACY_ID, V0_ID])+def test_kb_agnostic_id_owned_elsewhere_is_never_reused(foreign_id):+ """Regression test for #18116.++ ``owned_doc_ids`` lists only what this knowledge base already holds, so an+ id absent from it may well exist under a different knowledge base. Handing+ that id to ``FileService.upload_document`` is what produced "Existing+ document id collision with another knowledge base; skipping update." and+ dropped the document. KB B must route around it.+ """+ kb_a_owns = {foreign_id}+ kb_b_owns = set()++ assert resolve_connector_doc_id(KB_A, CONNECTOR_ID, EXTERNAL_ID, kb_a_owns) == foreign_id+ assert resolve_connector_doc_id(KB_B, CONNECTOR_ID, EXTERNAL_ID, kb_b_owns) == SCOPED_B+ assert SCOPED_B != foreign_id+++def test_oldest_owned_id_wins_when_several_upgrades_left_rows_behind():+ """An instance carrying rows from more than one upgrade settles, not drifts.++ Preferring the newest owned id would re-key the document on every upgrade+ and strand the older row again each time.+ """+ assert resolve_connector_doc_id(KB_A, CONNECTOR_ID, EXTERNAL_ID, {V0_ID, LEGACY_ID}) == V0_ID+++def test_two_kbs_on_one_connector_never_share_an_id():+ assert resolve_connector_doc_id(KB_A, CONNECTOR_ID, EXTERNAL_ID, set()) != resolve_connector_doc_id(KB_B, CONNECTOR_ID, EXTERNAL_ID, set())+++def test_resolution_is_stable_across_runs():+ """The second run sees the id the first run wrote and settles on it."""+ first = resolve_connector_doc_id(KB_A, CONNECTOR_ID, EXTERNAL_ID, set())+ second = resolve_connector_doc_id(KB_A, CONNECTOR_ID, EXTERNAL_ID, {first})+ assert first == second == SCOPED_Adiff --git a/test/unit_test/api/db/services/test_file_service_upload_document.py b/test/unit_test/api/db/services/test_file_service_upload_document.pyindex de8c65d4ea..655f1f1476 100644--- a/test/unit_test/api/db/services/test_file_service_upload_document.py+++ b/test/unit_test/api/db/services/test_file_service_upload_document.py@@ -70,21 +70,40 @@ from api.db.services.file_service import FileService # noqa: E402class _DummyUploadFile:- def __init__(self, filename, doc_id):+ def __init__(self, filename, doc_id, blob=None):self.filename = filenameself.id = doc_id+ self._blob = blobdef read(self):- raise AssertionError("read() should not be called for cross-KB collision path")+ if self._blob is None:+ raise AssertionError("read() should not be called for cross-KB collision path")+ return self._blob+++class _DummyStorage:+ """Storage double recording writes without touching MinIO."""++ def __init__(self):+ self.written = {}+ self.removed = []++ def obj_exist(self, _bucket, _location):+ return False++ def put(self, bucket, location, blob, *_args):+ self.written[(bucket, location)] = blob++ def rm(self, bucket, location):+ self.removed.append((bucket, location))def _unwrapped_upload_document():return FileService.upload_document.__func__.__wrapped__-def test_upload_document_skips_cross_kb_document_id_collision(monkeypatch):- kb = SimpleNamespace(+def _target_kb():+ return SimpleNamespace(id="kb-target",tenant_id="tenant-1",name="Target KB",@@ -92,14 +111,9 @@ def test_upload_document_skips_cross_kb_document_id_collision(monkeypatch):pipeline_id=None,parser_config={},)- existing_doc = SimpleNamespace(- id="doc-1",- kb_id="kb-other",- location="old-location.txt",- content_hash="old-hash",- to_dict=lambda: {"id": "doc-1"},- )++def _stub_folder_lookups(monkeypatch):monkeypatch.setattr(FileService, "get_root_folder", classmethod(lambda cls, _uid: {"id": "root"}))monkeypatch.setattr(FileService, "init_knowledgebase_docs", classmethod(lambda cls, _pf_id, _uid: None))monkeypatch.setattr(FileService, "get_kb_folder", classmethod(lambda cls, _uid: {"id": "kb-root"}))@@ -108,7 +122,26 @@ def test_upload_document_skips_cross_kb_document_id_collision(monkeypatch):"new_a_file_from_kb",classmethod(lambda cls, _tenant_id, _name, _parent_id: {"id": "kb-folder"}),)+++def test_upload_document_skips_cross_kb_document_id_collision(monkeypatch):+ kb = _target_kb()+ existing_doc = SimpleNamespace(+ id="doc-1",+ kb_id="kb-other",+ location="old-location.txt",+ content_hash="old-hash",+ to_dict=lambda: {"id": "doc-1"},+ )++ _stub_folder_lookups(monkeypatch)monkeypatch.setattr(file_service_module.DocumentService, "get_by_id", lambda _doc_id: (True, existing_doc))+ monkeypatch.setattr(+ file_service_module.KnowledgebaseService,+ "get_or_none",+ classmethod(lambda cls, **_kwargs: SimpleNamespace(id="kb-other")),+ )err, files = _unwrapped_upload_document()(FileService,@@ -120,7 +153,98 @@ def test_upload_document_skips_cross_kb_document_id_collision(monkeypatch):assert files == []assert len(err) == 1assert err[0].startswith("collision.txt: ")- assert "Existing document id collision with another knowledge base; skipping update." in err[0]+ # The owning knowledge base is named so the conflict can actually be found.+ assert "Existing document id collision with knowledge base 'kb-other'; skipping update." in err[0]+++def test_upload_document_reclaims_document_stranded_by_deleted_kb(monkeypatch):+ """A row whose knowledge base is gone must not block ingestion forever.++ Regression test for #18116: the stranded row is unreachable through the UI+ and the API, so treating it as a live conflict permanently blocked the+ document from syncing anywhere.+ """+ kb = _target_kb()+ stranded_doc = SimpleNamespace(+ id="doc-1",+ kb_id="kb-deleted",+ location="old-location.txt",+ content_hash="old-hash",+ to_dict=lambda: {"id": "doc-1"},+ )+ storage = _DummyStorage()+ discarded = []+ inserted = []+ tasks_deleted = []+ files_deleted = []+ mappings_deleted = []++ _stub_folder_lookups(monkeypatch)+ monkeypatch.setattr(file_service_module.DocumentService, "get_by_id", lambda _doc_id: (True, stranded_doc))+ monkeypatch.setattr(+ file_service_module.KnowledgebaseService,+ "get_or_none",+ classmethod(lambda cls, **kwargs: None if kwargs.get("id") == "kb-deleted" else SimpleNamespace(id=kwargs.get("id"))),+ )+ monkeypatch.setattr(+ file_service_module.DocumentService,+ "delete_by_id",+ classmethod(lambda cls, doc_id: discarded.append(doc_id)),+ )+ monkeypatch.setattr(+ file_service_module.File2DocumentService,+ "delete_by_document_id",+ classmethod(lambda cls, doc_id: mappings_deleted.append(doc_id)),+ )+ monkeypatch.setattr(+ file_service_module.File2DocumentService,+ "get_storage_address",+ classmethod(lambda cls, **_kwargs: ("kb-deleted", "old-location.txt")),+ )+ monkeypatch.setattr(+ file_service_module.File2DocumentService,+ "get_by_document_id",+ classmethod(lambda cls, _doc_id: [SimpleNamespace(file_id="file-1")]),+ )+ monkeypatch.setattr(+ file_service_module.TaskService,+ "filter_delete",+ classmethod(lambda cls, _filters: tasks_deleted.append("doc-1")),+ )+ monkeypatch.setattr(+ FileService,+ "filter_delete",+ classmethod(lambda cls, _filters: files_deleted.append("file-1") or 1),+ )+ monkeypatch.setattr(file_service_module.DocumentService, "check_doc_health", classmethod(lambda cls, _tid, _name: None))+ monkeypatch.setattr(file_service_module.DocumentService, "query", classmethod(lambda cls, **_kwargs: []))+ monkeypatch.setattr(file_service_module.DocumentService, "insert", classmethod(lambda cls, doc: inserted.append(doc)))+ monkeypatch.setattr(FileService, "add_file_from_kb", classmethod(lambda cls, _doc, _folder_id, _tenant_id: None))+ monkeypatch.setattr(file_service_module, "thumbnail_img", lambda _name, _blob: None)+ monkeypatch.setattr(file_service_module.settings, "STORAGE_IMPL", storage)++ err, files = _unwrapped_upload_document()(+ FileService,+ kb,+ [_DummyUploadFile(filename="collision.txt", doc_id="doc-1", blob=b"payload")],+ "user-1",+ )++ assert err == []+ assert discarded == ["doc-1"]+ # The stranded row's debris goes with it, matching what delete_docs would+ # have removed had the dataset deletion reached this document.+ assert tasks_deleted == ["doc-1"]+ assert files_deleted == ["file-1"]+ assert mappings_deleted == ["doc-1"]+ assert storage.removed == [("kb-deleted", "old-location.txt")]+ assert len(inserted) == 1+ assert inserted[0]["id"] == "doc-1"+ assert inserted[0]["kb_id"] == "kb-target"+ assert len(files) == 1+ assert files[0][1] == b"payload"+ assert storage.written[("kb-target", "collision.txt")] == b"payload"# ---------------------------------------------------------------------------