OpenRAG is a comprehensive, single package Retrieval-Augmented Generation platform built on Langflow, Docling, and Opensearch.
Python
fix: make a duplicate check answerable from the server logs (#2461)
Rico Furtado367fe318e485
diff --git a/src/api/connectors.py b/src/api/connectors.pyindex 9330de8ae..a982e6a53 100644--- a/src/api/connectors.py+++ b/src/api/connectors.py@@ -1073,6 +1073,7 @@ async def _expand_selected_connector_files(async def _classify_connector_duplicates(connector,+ connector_type: str,selected_files_raw: list[Any],session_manager,user_id: str,@@ -1081,6 +1082,14 @@ async def _classify_connector_duplicates("""Expand connector selections and split them into duplicate/non-duplicate files."""expanded_files_info = await _expand_selected_connector_files(connector, selected_files_raw)if not expanded_files_info:+ # The selection named only folders that expanded to nothing, so there is+ # nothing to compare. Counted against what the caller asked for, which+ # separates an empty expansion from an empty request.+ logger.info(+ "Duplicate check found nothing to check",+ connector_type=connector_type,+ selected=len(selected_files_raw),+ )return {"duplicate_names": [],"duplicate_files": [],@@ -1101,6 +1110,13 @@ async def _classify_connector_duplicates(all_candidates.update(aliases)if not all_candidates:+ # Files expanded but none yielded a comparable name, so the index was+ # never queried: every file is reported new without having been checked.+ logger.info(+ "Duplicate check skipped: no comparable filenames",+ connector_type=connector_type,+ listed=len(cleaned_files),+ )return {"duplicate_names": [],"duplicate_files": [],@@ -1131,6 +1147,16 @@ async def _classify_connector_duplicates(else:non_duplicate_files.append(file_info)+ # See the bucket classifier: logged so a "no dialog appeared" report can be+ # answered from the server side.+ logger.info(+ "Duplicate check complete",+ connector_type=connector_type,+ selected=len(cleaned_files),+ duplicates=len(duplicate_files),+ new_files=len(non_duplicate_files),+ )+return {"duplicate_names": list(dict.fromkeys(duplicate_names)),"duplicate_files": duplicate_files,@@ -1335,6 +1361,11 @@ async def _classify_bucket_connector_duplicates(breakif not all_files:+ logger.info(+ "Duplicate check found nothing to check",+ connector_type=connector_type,+ buckets=bucket_filter,+ )return {"duplicate_names": [],"duplicate_files": [],@@ -1366,18 +1397,38 @@ async def _classify_bucket_connector_duplicates(duplicate_names: list[str] = []non_duplicate_files: list[dict[str, Any]] = []total = 0+ already_synced = 0for f in all_files:fid = f.get("id")if not fid:continuetotal += 1response_file = _connector_file_response(f, cleaned_name=_cleaned_blob_filename(f))- if fid in existing_set or fid in name_taken_ids:+ id_match = fid in existing_set+ if id_match:+ already_synced += 1+ if id_match or fid in name_taken_ids:duplicate_files.append(response_file)duplicate_names.append(response_file["name"])else:non_duplicate_files.append(response_file)+ # Both halves of the answer, so a "the dialog never appeared" report can be+ # settled from the logs: whether the listing came back, whether the names+ # were found in the index, and what the caller was told. Without it the only+ # way to tell a check that found nothing from one that never ran is the+ # browser's network tab.+ logger.info(+ "Duplicate check complete",+ connector_type=connector_type,+ buckets=bucket_filter,+ listed=total,+ already_synced_here=already_synced,+ name_taken_elsewhere=len(set(name_taken_ids) - existing_set),+ duplicates=len(duplicate_files),+ new_files=len(non_duplicate_files),+ )+return {"duplicate_names": list(dict.fromkeys(duplicate_names)),"duplicate_files": duplicate_files,@@ -1402,7 +1453,19 @@ async def connector_check_duplicates(selected_files_raw = body.selected_filesif not selected_files_raw and not body.bucket_filter:- return JSONResponse({"duplicate_names": []})+ # Returning an empty duplicate list here reads to the caller as "checked,+ # found nothing", and the UI goes on to ingest without ever offering the+ # overwrite choice — the same thing a broken check looks like. The+ # request asked about nothing, so say that instead of answering it.+ logger.warning(+ "Duplicate check asked about nothing",+ connector_type=connector_type,+ connection_id=body.connection_id,+ )+ return JSONResponse(+ {"error": "Specify selected_files or bucket_filter to check for duplicates"},+ status_code=400,+ )try:jwt_token = user.jwt_token@@ -1458,6 +1521,7 @@ async def connector_check_duplicates(return JSONResponse(await _classify_connector_duplicates(connector=connector,+ connector_type=connector_type,selected_files_raw=selected_files_raw,session_manager=session_manager,user_id=user.user_id,@@ -1786,6 +1850,7 @@ async def connector_sync(connector=await connector_service.get_connector(working_connection.connection_id),+ connector_type=connector_type,selected_files_raw=file_infos,session_manager=session_manager,user_id=user.user_id,diff --git a/tests/unit/connectors/test_connector_file_type_validation.py b/tests/unit/connectors/test_connector_file_type_validation.pyindex 7dccbe00c..68ecc82fe 100644--- a/tests/unit/connectors/test_connector_file_type_validation.py+++ b/tests/unit/connectors/test_connector_file_type_validation.py@@ -474,3 +474,231 @@ async def test_connector_sync_does_not_report_all_duplicates_when_expansion_is_ekwargs = connector_service.sync_specific_files.await_args.kwargsassert args[2] == ["folder-id"]assert kwargs["file_infos"] == [{"id": "folder-id", "name": "Folder", "isFolder": True}]+++async def test_check_duplicates_rejects_a_request_that_names_nothing():+ """A check with neither selected_files nor bucket_filter used to answer with+ an empty duplicate list, which reads to the caller as "checked, found+ nothing" — the UI then ingests without ever offering the overwrite choice,+ indistinguishable from a check that genuinely found none. Say the request+ asked about nothing instead.+ """+ import json++ from fastapi.responses import JSONResponse++ from api.connectors import ConnectorCheckDuplicatesBody, connector_check_duplicates++ response = await connector_check_duplicates(+ connector_type="ibm_cos",+ body=ConnectorCheckDuplicatesBody(connection_id="conn-id"),+ request=MagicMock(),+ connector_service=MagicMock(),+ session_manager=MagicMock(),+ user=MagicMock(user_id="u1", jwt_token="t"),+ )++ assert isinstance(response, JSONResponse)+ assert response.status_code == 400+ data = json.loads(response.body.decode())+ assert "duplicate_names" not in data+ assert "bucket_filter" in data["error"]+++async def test_bucket_duplicate_check_logs_its_outcome(monkeypatch):+ """The counts behind the answer are logged, so "the dialog never appeared"+ can be settled from the server rather than from a browser network tab."""+ from api import connectors as connectors_api++ monkeypatch.setattr(connectors_api, "get_index_name", lambda: "idx")+ monkeypatch.setattr(+ connectors_api,+ "get_synced_file_ids_for_connector",+ AsyncMock(return_value=(["b::synced.pdf"], [], "connector_file_id")),+ )++ indexed = {"taken.pdf"}++ async def search(*, index, body):+ terms = body["query"].get("terms")+ if terms is not None:+ asked = terms["filename"]+ return {+ "aggregations": {+ "filenames": {"buckets": [{"key": n} for n in asked if n in indexed]}+ }+ }+ return {+ "aggregations": {+ "connector_file_ids": {"buckets": []},+ "document_ids": {"buckets": []},+ }+ }++ client = AsyncMock()+ client.search = AsyncMock(side_effect=search)+ session_manager = MagicMock()+ session_manager.get_user_opensearch_client = MagicMock(return_value=client)++ connector = MagicMock()+ connector.bucket_names = None+ connector.list_files = AsyncMock(+ return_value={+ "files": [+ {"id": "b::synced.pdf", "name": "synced.pdf"},+ {"id": "b::taken.pdf", "name": "taken.pdf"},+ {"id": "b::fresh.pdf", "name": "fresh.pdf"},+ ],+ "next_page_token": None,+ }+ )++ # The project logger writes straight to stderr rather than propagating to the+ # stdlib root logger, so assert on the call instead of on caplog.+ logged = []+ monkeypatch.setattr(connectors_api.logger, "info", lambda msg, **kw: logged.append((msg, kw)))++ await connectors_api._classify_bucket_connector_duplicates(+ connector=connector,+ connector_type="ibm_cos",+ bucket_filter=["b"],+ session_manager=session_manager,+ user_id="u1",+ jwt_token="t",+ )++ summary = next(kw for msg, kw in logged if msg == "Duplicate check complete")+ assert summary["listed"] == 3+ assert summary["already_synced_here"] == 1+ assert summary["name_taken_elsewhere"] == 1+ assert summary["duplicates"] == 2+ assert summary["new_files"] == 1+++async def test_bucket_duplicate_check_logs_an_empty_listing(monkeypatch):+ """Zero duplicates because the bucket listing came back empty is a different+ fact from zero duplicates among files that were listed."""+ from api import connectors as connectors_api++ connector = MagicMock()+ connector.bucket_names = None+ connector.list_files = AsyncMock(return_value={"files": [], "next_page_token": None})++ logged = []+ monkeypatch.setattr(connectors_api.logger, "info", lambda msg, **kw: logged.append((msg, kw)))++ result = await connectors_api._classify_bucket_connector_duplicates(+ connector=connector,+ connector_type="ibm_cos",+ bucket_filter=["b"],+ session_manager=MagicMock(),+ user_id="u1",+ jwt_token="t",+ )++ assert result["duplicate_count"] == 0+ assert any(msg == "Duplicate check found nothing to check" for msg, _ in logged)+++async def test_selected_files_check_logs_an_empty_expansion(monkeypatch):+ """Folders that expand to nothing answer "no duplicates" without ever+ comparing anything. Logged against what the caller selected, so the OAuth+ connectors are as diagnosable as the bucket ones."""+ from api import connectors as connectors_api++ connector = MagicMock()+ connector.cfg = MagicMock()+ connector.list_selected_files = AsyncMock(return_value={"files": []})++ logged = []+ monkeypatch.setattr(connectors_api.logger, "info", lambda msg, **kw: logged.append((msg, kw)))++ result = await connectors_api._classify_connector_duplicates(+ connector=connector,+ connector_type="onedrive",+ selected_files_raw=[+ {"id": "folder-1", "name": "Empty 1", "isFolder": True},+ {"id": "folder-2", "name": "Empty 2", "isFolder": True},+ ],+ session_manager=MagicMock(),+ user_id="u1",+ jwt_token="t",+ )++ assert result["duplicate_count"] == 0+ summary = next(kw for msg, kw in logged if msg == "Duplicate check found nothing to check")+ assert summary["connector_type"] == "onedrive"+ assert summary["selected"] == 2+++async def test_selected_files_check_logs_when_no_name_is_comparable(monkeypatch):+ """Files expanded but none produced a name to compare, so the index was+ never queried — every file is reported new without having been checked."""+ from api import connectors as connectors_api++ connector = MagicMock()+ connector.cfg = None # not a cfg-backed connector: use the raw selection++ logged = []+ monkeypatch.setattr(connectors_api.logger, "info", lambda msg, **kw: logged.append((msg, kw)))++ result = await connectors_api._classify_connector_duplicates(+ connector=connector,+ connector_type="google_drive",+ selected_files_raw=[{"id": "f1", "name": "", "mimeType": ""}],+ session_manager=MagicMock(),+ user_id="u1",+ jwt_token="t",+ )++ assert result["duplicate_count"] == 0+ summary = next(+ kw for msg, kw in logged if msg == "Duplicate check skipped: no comparable filenames"+ )+ assert summary["connector_type"] == "google_drive"+ assert summary["listed"] == 1+++async def test_selected_files_check_names_its_connector_in_the_summary(monkeypatch):+ """One grep covers both connector families; connector_type says which path+ the line came from."""+ from api import connectors as connectors_api++ monkeypatch.setattr(connectors_api, "get_index_name", lambda: "idx")++ client = AsyncMock()+ client.search = AsyncMock(+ return_value={"aggregations": {"filenames": {"buckets": [{"key": "taken.pdf"}]}}}+ )+ session_manager = MagicMock()+ session_manager.get_user_opensearch_client = MagicMock(return_value=client)++ connector = MagicMock()+ connector.cfg = None++ logged = []+ monkeypatch.setattr(connectors_api.logger, "info", lambda msg, **kw: logged.append((msg, kw)))++ await connectors_api._classify_connector_duplicates(+ connector=connector,+ connector_type="sharepoint",+ selected_files_raw=[+ {"id": "f1", "name": "taken.pdf", "mimeType": "application/pdf"},+ {"id": "f2", "name": "fresh.pdf", "mimeType": "application/pdf"},+ ],+ session_manager=session_manager,+ user_id="u1",+ jwt_token="t",+ )++ summary = next(kw for msg, kw in logged if msg == "Duplicate check complete")+ assert summary["connector_type"] == "sharepoint"+ assert summary["selected"] == 2+ assert summary["duplicates"] == 1+ assert summary["new_files"] == 1