Skip to content

OpenRAG is a comprehensive, single package Retrieval-Augmented Generation platform built on Langflow, Docling, and Opensearch.

Python

fix: include connector_file_id-only files in orphan filename map (#2375)

Nathan Bakaredbb6f9e442fe
diff --git a/src/api/connectors.py b/src/api/connectors.py
index 287b5bd3c..9afe02f56 100644
--- a/src/api/connectors.py
+++ b/src/api/connectors.py
@@ -230,10 +230,15 @@ async def get_synced_id_to_filename_map(
session_manager,
jwt_token: str | None = None,
) -> dict[str, str]:
- """Return a {document_id: filename} map for files ingested under this connector_type.
+ """Return an {id: filename} map for files ingested under this connector_type.
- Uses a sub-aggregation so each document_id is paired with its top filename in
- a single OpenSearch round trip.
+ Keys cover both ingest layouts (mirrors ``get_synced_file_ids_for_connector``):
+ the non-Langflow path stores the connector id in ``connector_file_id`` (where
+ ``document_id`` is a content hash), while the Langflow path stores it in
+ ``document_id``. ``connector_file_id`` wins when both are present for the same id.
+
+ Uses a sub-aggregation so each id is paired with its top filename in a single
+ OpenSearch round trip.
"""
try:
opensearch_client = session_manager.get_user_opensearch_client(user_id, jwt_token)
@@ -242,31 +247,53 @@ async def get_synced_id_to_filename_map(
"size": 0,
"query": {"term": {"connector_type": connector_type}},
"aggs": {
+ "by_connector_file_id": {
+ "terms": {"field": "connector_file_id", "size": OPENSEARCH_TERMS_AGG_LIMIT},
+ "aggs": {
+ "top_filename": {"terms": {"field": "filename", "size": 1}},
+ },
+ },
"by_document_id": {
"terms": {"field": "document_id", "size": OPENSEARCH_TERMS_AGG_LIMIT},
"aggs": {
"top_filename": {"terms": {"field": "filename", "size": 1}},
},
- }
+ },
},
}
- result = await opensearch_client.search(index=get_index_name(), body=query_body)
- buckets = result.get("aggregations", {}).get("by_document_id", {}).get("buckets", [])
- if len(buckets) == OPENSEARCH_TERMS_AGG_LIMIT:
- logger.warning(
- "Document ID to filename mapping hit 10k limit - results may be truncated",
- connector_type=connector_type,
- returned_count=len(buckets),
+ try:
+ result = await opensearch_client.search(index=get_index_name(), body=query_body)
+ except Exception as agg_err:
+ if not _is_unmapped_keyword_agg_error(agg_err):
+ raise
+ # See get_synced_file_ids_for_connector: some indices predate the
+ # explicit keyword mapping for connector_file_id.
+ query_body["aggs"]["by_connector_file_id"]["terms"]["field"] = (
+ "connector_file_id.keyword"
)
+ result = await opensearch_client.search(index=get_index_name(), body=query_body)
+
+ aggs = result.get("aggregations", {})
mapping: dict[str, str] = {}
- for bucket in buckets:
- doc_id = bucket.get("key")
- if not doc_id:
- continue
- fn_buckets = bucket.get("top_filename", {}).get("buckets", [])
- mapping[doc_id] = fn_buckets[0]["key"] if fn_buckets else ""
+ # document_id first; connector_file_id overlays it (connector_file_id wins),
+ # matching get_synced_id_to_modified_time_map's precedence.
+ for agg_name in ("by_document_id", "by_connector_file_id"):
+ buckets = aggs.get(agg_name, {}).get("buckets", [])
+ if len(buckets) == OPENSEARCH_TERMS_AGG_LIMIT:
+ logger.warning(
+ "ID to filename mapping hit 10k limit - results may be truncated",
+ connector_type=connector_type,
+ agg=agg_name,
+ returned_count=len(buckets),
+ )
+ for bucket in buckets:
+ key = bucket.get("key")
+ if not key:
+ continue
+ fn_buckets = bucket.get("top_filename", {}).get("buckets", [])
+ mapping[key] = fn_buckets[0]["key"] if fn_buckets else ""
return mapping
except Exception as e:
logger.error(
diff --git a/tests/unit/api/test_connector_change_detection.py b/tests/unit/api/test_connector_change_detection.py
index 52e0d4ba1..653d45c74 100644
--- a/tests/unit/api/test_connector_change_detection.py
+++ b/tests/unit/api/test_connector_change_detection.py
@@ -464,6 +464,163 @@ async def test_state_map_returns_empty_on_error(monkeypatch):
assert result == {}
+# ---------------------------------------------------------------------------
+# get_synced_id_to_filename_map
+# ---------------------------------------------------------------------------
+
+
+async def test_filename_map_includes_connector_file_id_only_buckets(monkeypatch):
+ """Regression test: filenames for ids that only exist under connector_file_id
+ (the standard ConnectorFileProcessor path used by OAuth connectors, S3, etc.)
+ must not be dropped just because document_id holds an unrelated content hash."""
+ from api import connectors as connectors_api
+
+ monkeypatch.setattr(connectors_api, "get_index_name", lambda: "idx")
+
+ opensearch_client = AsyncMock()
+ opensearch_client.search = AsyncMock(
+ return_value={
+ "aggregations": {
+ "by_connector_file_id": {
+ "buckets": [
+ {
+ "key": "gdrive-file-guid",
+ "top_filename": {"buckets": [{"key": "report.pdf"}]},
+ },
+ ]
+ },
+ "by_document_id": {
+ "buckets": [
+ # Content-hash document_id from the standard ingest path —
+ # unrelated to any enumerated source id.
+ {
+ "key": "content-hash-abc",
+ "top_filename": {"buckets": [{"key": "report.pdf"}]},
+ },
+ ]
+ },
+ }
+ }
+ )
+ sm = MagicMock()
+ sm.get_user_opensearch_client = MagicMock(return_value=opensearch_client)
+
+ result = await connectors_api.get_synced_id_to_filename_map(
+ connector_type="google_drive",
+ user_id="alice",
+ session_manager=sm,
+ jwt_token=None,
+ )
+
+ assert result["gdrive-file-guid"] == "report.pdf"
+ assert result["content-hash-abc"] == "report.pdf"
+
+
+async def test_filename_map_prefers_connector_file_id_over_document_id(monkeypatch):
+ from api import connectors as connectors_api
+
+ monkeypatch.setattr(connectors_api, "get_index_name", lambda: "idx")
+
+ opensearch_client = AsyncMock()
+ opensearch_client.search = AsyncMock(
+ return_value={
+ "aggregations": {
+ "by_connector_file_id": {
+ "buckets": [
+ {"key": "c::a", "top_filename": {"buckets": [{"key": "real-name.txt"}]}},
+ ]
+ },
+ "by_document_id": {
+ "buckets": [
+ # Same key from the content-hash side, stale/wrong filename —
+ # connector_file_id must win.
+ {"key": "c::a", "top_filename": {"buckets": [{"key": "stale-name.txt"}]}},
+ ]
+ },
+ }
+ }
+ )
+ sm = MagicMock()
+ sm.get_user_opensearch_client = MagicMock(return_value=opensearch_client)
+
+ result = await connectors_api.get_synced_id_to_filename_map(
+ connector_type="azure_blob",
+ user_id="alice",
+ session_manager=sm,
+ jwt_token=None,
+ )
+
+ assert result["c::a"] == "real-name.txt"
+
+
+async def test_filename_map_falls_back_to_keyword_subfield_on_text_field_error(
+ monkeypatch,
+):
+ from api import connectors as connectors_api
+
+ monkeypatch.setattr(connectors_api, "get_index_name", lambda: "idx")
+
+ opensearch_client = AsyncMock()
+ called_fields = []
+
+ async def fake_search(*, index, body):
+ called_fields.append(body["aggs"]["by_connector_file_id"]["terms"]["field"])
+ if len(called_fields) == 1:
+ raise Exception(
+ "RequestError(400, 'search_phase_execution_exception', 'Text fields "
+ "are not optimised for operations that require per-document field "
+ "data like aggregations and sorting, so these operations are "
+ "disabled by default. Please use a keyword field instead. "
+ "Alternatively, set fielddata=true on [connector_file_id]...')"
+ )
+ return {
+ "aggregations": {
+ "by_connector_file_id": {
+ "buckets": [
+ {"key": "c::a", "top_filename": {"buckets": [{"key": "real-name.txt"}]}},
+ ]
+ },
+ "by_document_id": {"buckets": []},
+ }
+ }
+
+ opensearch_client.search = AsyncMock(side_effect=fake_search)
+ sm = MagicMock()
+ sm.get_user_opensearch_client = MagicMock(return_value=opensearch_client)
+
+ result = await connectors_api.get_synced_id_to_filename_map(
+ connector_type="azure_blob",
+ user_id="alice",
+ session_manager=sm,
+ jwt_token=None,
+ )
+
+ assert result == {"c::a": "real-name.txt"}
+ assert called_fields == ["connector_file_id", "connector_file_id.keyword"]
+
+
+async def test_filename_map_returns_empty_on_error(monkeypatch):
+ from api import connectors as connectors_api
+
+ monkeypatch.setattr(connectors_api, "get_index_name", lambda: "idx")
+ opensearch_client = AsyncMock()
+ opensearch_client.search = AsyncMock(side_effect=RuntimeError("boom"))
+ sm = MagicMock()
+ sm.get_user_opensearch_client = MagicMock(return_value=opensearch_client)
+
+ result = await connectors_api.get_synced_id_to_filename_map(
+ connector_type="azure_blob",
+ user_id="alice",
+ session_manager=sm,
+ jwt_token=None,
+ )
+ assert result == {}
+
+
# ---------------------------------------------------------------------------
# get_synced_file_ids_for_connector
# ---------------------------------------------------------------------------