Skip to content

OpenRAG is a comprehensive, single package Retrieval-Augmented Generation platform built on Langflow, Docling, and Opensearch.

Python

fix: make a duplicate check answerable from the server logs (#2461)

Rico Furtado367fe318e485
diff --git a/src/api/connectors.py b/src/api/connectors.py
index 9330de8ae..a982e6a53 100644
--- a/src/api/connectors.py
+++ b/src/api/connectors.py
@@ -1073,6 +1073,7 @@ async def _expand_selected_connector_files(
async def _classify_connector_duplicates(
connector,
+ connector_type: str,
selected_files_raw: list[Any],
session_manager,
user_id: str,
@@ -1081,6 +1082,14 @@ async def _classify_connector_duplicates(
"""Expand connector selections and split them into duplicate/non-duplicate files."""
expanded_files_info = await _expand_selected_connector_files(connector, selected_files_raw)
if not expanded_files_info:
+ # The selection named only folders that expanded to nothing, so there is
+ # nothing to compare. Counted against what the caller asked for, which
+ # separates an empty expansion from an empty request.
+ logger.info(
+ "Duplicate check found nothing to check",
+ connector_type=connector_type,
+ selected=len(selected_files_raw),
+ )
return {
"duplicate_names": [],
"duplicate_files": [],
@@ -1101,6 +1110,13 @@ async def _classify_connector_duplicates(
all_candidates.update(aliases)
if not all_candidates:
+ # Files expanded but none yielded a comparable name, so the index was
+ # never queried: every file is reported new without having been checked.
+ logger.info(
+ "Duplicate check skipped: no comparable filenames",
+ connector_type=connector_type,
+ listed=len(cleaned_files),
+ )
return {
"duplicate_names": [],
"duplicate_files": [],
@@ -1131,6 +1147,16 @@ async def _classify_connector_duplicates(
else:
non_duplicate_files.append(file_info)
+ # See the bucket classifier: logged so a "no dialog appeared" report can be
+ # answered from the server side.
+ logger.info(
+ "Duplicate check complete",
+ connector_type=connector_type,
+ selected=len(cleaned_files),
+ duplicates=len(duplicate_files),
+ new_files=len(non_duplicate_files),
+ )
+
return {
"duplicate_names": list(dict.fromkeys(duplicate_names)),
"duplicate_files": duplicate_files,
@@ -1335,6 +1361,11 @@ async def _classify_bucket_connector_duplicates(
break
if not all_files:
+ logger.info(
+ "Duplicate check found nothing to check",
+ connector_type=connector_type,
+ buckets=bucket_filter,
+ )
return {
"duplicate_names": [],
"duplicate_files": [],
@@ -1366,18 +1397,38 @@ async def _classify_bucket_connector_duplicates(
duplicate_names: list[str] = []
non_duplicate_files: list[dict[str, Any]] = []
total = 0
+ already_synced = 0
for f in all_files:
fid = f.get("id")
if not fid:
continue
total += 1
response_file = _connector_file_response(f, cleaned_name=_cleaned_blob_filename(f))
- if fid in existing_set or fid in name_taken_ids:
+ id_match = fid in existing_set
+ if id_match:
+ already_synced += 1
+ if id_match or fid in name_taken_ids:
duplicate_files.append(response_file)
duplicate_names.append(response_file["name"])
else:
non_duplicate_files.append(response_file)
+ # Both halves of the answer, so a "the dialog never appeared" report can be
+ # settled from the logs: whether the listing came back, whether the names
+ # were found in the index, and what the caller was told. Without it the only
+ # way to tell a check that found nothing from one that never ran is the
+ # browser's network tab.
+ logger.info(
+ "Duplicate check complete",
+ connector_type=connector_type,
+ buckets=bucket_filter,
+ listed=total,
+ already_synced_here=already_synced,
+ name_taken_elsewhere=len(set(name_taken_ids) - existing_set),
+ duplicates=len(duplicate_files),
+ new_files=len(non_duplicate_files),
+ )
+
return {
"duplicate_names": list(dict.fromkeys(duplicate_names)),
"duplicate_files": duplicate_files,
@@ -1402,7 +1453,19 @@ async def connector_check_duplicates(
selected_files_raw = body.selected_files
if not selected_files_raw and not body.bucket_filter:
- return JSONResponse({"duplicate_names": []})
+ # Returning an empty duplicate list here reads to the caller as "checked,
+ # found nothing", and the UI goes on to ingest without ever offering the
+ # overwrite choice — the same thing a broken check looks like. The
+ # request asked about nothing, so say that instead of answering it.
+ logger.warning(
+ "Duplicate check asked about nothing",
+ connector_type=connector_type,
+ connection_id=body.connection_id,
+ )
+ return JSONResponse(
+ {"error": "Specify selected_files or bucket_filter to check for duplicates"},
+ status_code=400,
+ )
try:
jwt_token = user.jwt_token
@@ -1458,6 +1521,7 @@ async def connector_check_duplicates(
return JSONResponse(
await _classify_connector_duplicates(
connector=connector,
+ connector_type=connector_type,
selected_files_raw=selected_files_raw,
session_manager=session_manager,
user_id=user.user_id,
@@ -1786,6 +1850,7 @@ async def connector_sync(
connector=await connector_service.get_connector(
working_connection.connection_id
),
+ connector_type=connector_type,
selected_files_raw=file_infos,
session_manager=session_manager,
user_id=user.user_id,
diff --git a/tests/unit/connectors/test_connector_file_type_validation.py b/tests/unit/connectors/test_connector_file_type_validation.py
index 7dccbe00c..68ecc82fe 100644
--- a/tests/unit/connectors/test_connector_file_type_validation.py
+++ b/tests/unit/connectors/test_connector_file_type_validation.py
@@ -474,3 +474,231 @@ async def test_connector_sync_does_not_report_all_duplicates_when_expansion_is_e
kwargs = connector_service.sync_specific_files.await_args.kwargs
assert args[2] == ["folder-id"]
assert kwargs["file_infos"] == [{"id": "folder-id", "name": "Folder", "isFolder": True}]
+
+
+async def test_check_duplicates_rejects_a_request_that_names_nothing():
+ """A check with neither selected_files nor bucket_filter used to answer with
+ an empty duplicate list, which reads to the caller as "checked, found
+ nothing" — the UI then ingests without ever offering the overwrite choice,
+ indistinguishable from a check that genuinely found none. Say the request
+ asked about nothing instead.
+ """
+ import json
+
+ from fastapi.responses import JSONResponse
+
+ from api.connectors import ConnectorCheckDuplicatesBody, connector_check_duplicates
+
+ response = await connector_check_duplicates(
+ connector_type="ibm_cos",
+ body=ConnectorCheckDuplicatesBody(connection_id="conn-id"),
+ request=MagicMock(),
+ connector_service=MagicMock(),
+ session_manager=MagicMock(),
+ user=MagicMock(user_id="u1", jwt_token="t"),
+ )
+
+ assert isinstance(response, JSONResponse)
+ assert response.status_code == 400
+ data = json.loads(response.body.decode())
+ assert "duplicate_names" not in data
+ assert "bucket_filter" in data["error"]
+
+
+async def test_bucket_duplicate_check_logs_its_outcome(monkeypatch):
+ """The counts behind the answer are logged, so "the dialog never appeared"
+ can be settled from the server rather than from a browser network tab."""
+ from api import connectors as connectors_api
+
+ monkeypatch.setattr(connectors_api, "get_index_name", lambda: "idx")
+ monkeypatch.setattr(
+ connectors_api,
+ "get_synced_file_ids_for_connector",
+ AsyncMock(return_value=(["b::synced.pdf"], [], "connector_file_id")),
+ )
+
+ indexed = {"taken.pdf"}
+
+ async def search(*, index, body):
+ terms = body["query"].get("terms")
+ if terms is not None:
+ asked = terms["filename"]
+ return {
+ "aggregations": {
+ "filenames": {"buckets": [{"key": n} for n in asked if n in indexed]}
+ }
+ }
+ return {
+ "aggregations": {
+ "connector_file_ids": {"buckets": []},
+ "document_ids": {"buckets": []},
+ }
+ }
+
+ client = AsyncMock()
+ client.search = AsyncMock(side_effect=search)
+ session_manager = MagicMock()
+ session_manager.get_user_opensearch_client = MagicMock(return_value=client)
+
+ connector = MagicMock()
+ connector.bucket_names = None
+ connector.list_files = AsyncMock(
+ return_value={
+ "files": [
+ {"id": "b::synced.pdf", "name": "synced.pdf"},
+ {"id": "b::taken.pdf", "name": "taken.pdf"},
+ {"id": "b::fresh.pdf", "name": "fresh.pdf"},
+ ],
+ "next_page_token": None,
+ }
+ )
+
+ # The project logger writes straight to stderr rather than propagating to the
+ # stdlib root logger, so assert on the call instead of on caplog.
+ logged = []
+ monkeypatch.setattr(connectors_api.logger, "info", lambda msg, **kw: logged.append((msg, kw)))
+
+ await connectors_api._classify_bucket_connector_duplicates(
+ connector=connector,
+ connector_type="ibm_cos",
+ bucket_filter=["b"],
+ session_manager=session_manager,
+ user_id="u1",
+ jwt_token="t",
+ )
+
+ summary = next(kw for msg, kw in logged if msg == "Duplicate check complete")
+ assert summary["listed"] == 3
+ assert summary["already_synced_here"] == 1
+ assert summary["name_taken_elsewhere"] == 1
+ assert summary["duplicates"] == 2
+ assert summary["new_files"] == 1
+
+
+async def test_bucket_duplicate_check_logs_an_empty_listing(monkeypatch):
+ """Zero duplicates because the bucket listing came back empty is a different
+ fact from zero duplicates among files that were listed."""
+ from api import connectors as connectors_api
+
+ connector = MagicMock()
+ connector.bucket_names = None
+ connector.list_files = AsyncMock(return_value={"files": [], "next_page_token": None})
+
+ logged = []
+ monkeypatch.setattr(connectors_api.logger, "info", lambda msg, **kw: logged.append((msg, kw)))
+
+ result = await connectors_api._classify_bucket_connector_duplicates(
+ connector=connector,
+ connector_type="ibm_cos",
+ bucket_filter=["b"],
+ session_manager=MagicMock(),
+ user_id="u1",
+ jwt_token="t",
+ )
+
+ assert result["duplicate_count"] == 0
+ assert any(msg == "Duplicate check found nothing to check" for msg, _ in logged)
+
+
+async def test_selected_files_check_logs_an_empty_expansion(monkeypatch):
+ """Folders that expand to nothing answer "no duplicates" without ever
+ comparing anything. Logged against what the caller selected, so the OAuth
+ connectors are as diagnosable as the bucket ones."""
+ from api import connectors as connectors_api
+
+ connector = MagicMock()
+ connector.cfg = MagicMock()
+ connector.list_selected_files = AsyncMock(return_value={"files": []})
+
+ logged = []
+ monkeypatch.setattr(connectors_api.logger, "info", lambda msg, **kw: logged.append((msg, kw)))
+
+ result = await connectors_api._classify_connector_duplicates(
+ connector=connector,
+ connector_type="onedrive",
+ selected_files_raw=[
+ {"id": "folder-1", "name": "Empty 1", "isFolder": True},
+ {"id": "folder-2", "name": "Empty 2", "isFolder": True},
+ ],
+ session_manager=MagicMock(),
+ user_id="u1",
+ jwt_token="t",
+ )
+
+ assert result["duplicate_count"] == 0
+ summary = next(kw for msg, kw in logged if msg == "Duplicate check found nothing to check")
+ assert summary["connector_type"] == "onedrive"
+ assert summary["selected"] == 2
+
+
+async def test_selected_files_check_logs_when_no_name_is_comparable(monkeypatch):
+ """Files expanded but none produced a name to compare, so the index was
+ never queried — every file is reported new without having been checked."""
+ from api import connectors as connectors_api
+
+ connector = MagicMock()
+ connector.cfg = None # not a cfg-backed connector: use the raw selection
+
+ logged = []
+ monkeypatch.setattr(connectors_api.logger, "info", lambda msg, **kw: logged.append((msg, kw)))
+
+ result = await connectors_api._classify_connector_duplicates(
+ connector=connector,
+ connector_type="google_drive",
+ selected_files_raw=[{"id": "f1", "name": "", "mimeType": ""}],
+ session_manager=MagicMock(),
+ user_id="u1",
+ jwt_token="t",
+ )
+
+ assert result["duplicate_count"] == 0
+ summary = next(
+ kw for msg, kw in logged if msg == "Duplicate check skipped: no comparable filenames"
+ )
+ assert summary["connector_type"] == "google_drive"
+ assert summary["listed"] == 1
+
+
+async def test_selected_files_check_names_its_connector_in_the_summary(monkeypatch):
+ """One grep covers both connector families; connector_type says which path
+ the line came from."""
+ from api import connectors as connectors_api
+
+ monkeypatch.setattr(connectors_api, "get_index_name", lambda: "idx")
+
+ client = AsyncMock()
+ client.search = AsyncMock(
+ return_value={"aggregations": {"filenames": {"buckets": [{"key": "taken.pdf"}]}}}
+ )
+ session_manager = MagicMock()
+ session_manager.get_user_opensearch_client = MagicMock(return_value=client)
+
+ connector = MagicMock()
+ connector.cfg = None
+
+ logged = []
+ monkeypatch.setattr(connectors_api.logger, "info", lambda msg, **kw: logged.append((msg, kw)))
+
+ await connectors_api._classify_connector_duplicates(
+ connector=connector,
+ connector_type="sharepoint",
+ selected_files_raw=[
+ {"id": "f1", "name": "taken.pdf", "mimeType": "application/pdf"},
+ {"id": "f2", "name": "fresh.pdf", "mimeType": "application/pdf"},
+ ],
+ session_manager=session_manager,
+ user_id="u1",
+ jwt_token="t",
+ )
+
+ summary = next(kw for msg, kw in logged if msg == "Duplicate check complete")
+ assert summary["connector_type"] == "sharepoint"
+ assert summary["selected"] == 2
+ assert summary["duplicates"] == 1
+ assert summary["new_files"] == 1