OpenRAG is a comprehensive, single package Retrieval-Augmented Generation platform built on Langflow, Docling, and Opensearch.
Python
fix: include connector_file_id-only files in orphan filename map (#2375)
Nathan Bakaredbb6f9e442fe
diff --git a/src/api/connectors.py b/src/api/connectors.pyindex 287b5bd3c..9afe02f56 100644--- a/src/api/connectors.py+++ b/src/api/connectors.py@@ -230,10 +230,15 @@ async def get_synced_id_to_filename_map(session_manager,jwt_token: str | None = None,) -> dict[str, str]:- """Return a {document_id: filename} map for files ingested under this connector_type.+ """Return an {id: filename} map for files ingested under this connector_type.- Uses a sub-aggregation so each document_id is paired with its top filename in- a single OpenSearch round trip.+ Keys cover both ingest layouts (mirrors ``get_synced_file_ids_for_connector``):+ the non-Langflow path stores the connector id in ``connector_file_id`` (where+ ``document_id`` is a content hash), while the Langflow path stores it in+ ``document_id``. ``connector_file_id`` wins when both are present for the same id.++ Uses a sub-aggregation so each id is paired with its top filename in a single+ OpenSearch round trip."""try:opensearch_client = session_manager.get_user_opensearch_client(user_id, jwt_token)@@ -242,31 +247,53 @@ async def get_synced_id_to_filename_map("size": 0,"query": {"term": {"connector_type": connector_type}},"aggs": {+ "by_connector_file_id": {+ "terms": {"field": "connector_file_id", "size": OPENSEARCH_TERMS_AGG_LIMIT},+ "aggs": {+ "top_filename": {"terms": {"field": "filename", "size": 1}},+ },+ },"by_document_id": {"terms": {"field": "document_id", "size": OPENSEARCH_TERMS_AGG_LIMIT},"aggs": {"top_filename": {"terms": {"field": "filename", "size": 1}},},- }+ },},}- result = await opensearch_client.search(index=get_index_name(), body=query_body)- buckets = result.get("aggregations", {}).get("by_document_id", {}).get("buckets", [])- if len(buckets) == OPENSEARCH_TERMS_AGG_LIMIT:- logger.warning(- "Document ID to filename mapping hit 10k limit - results may be truncated",- connector_type=connector_type,- returned_count=len(buckets),+ try:+ result = await opensearch_client.search(index=get_index_name(), body=query_body)+ except Exception as agg_err:+ if not _is_unmapped_keyword_agg_error(agg_err):+ raise+ # See get_synced_file_ids_for_connector: some indices predate the+ # explicit keyword mapping for connector_file_id.+ query_body["aggs"]["by_connector_file_id"]["terms"]["field"] = (+ "connector_file_id.keyword")+ result = await opensearch_client.search(index=get_index_name(), body=query_body)++ aggs = result.get("aggregations", {})mapping: dict[str, str] = {}- for bucket in buckets:- doc_id = bucket.get("key")- if not doc_id:- continue- fn_buckets = bucket.get("top_filename", {}).get("buckets", [])- mapping[doc_id] = fn_buckets[0]["key"] if fn_buckets else ""+ # document_id first; connector_file_id overlays it (connector_file_id wins),+ # matching get_synced_id_to_modified_time_map's precedence.+ for agg_name in ("by_document_id", "by_connector_file_id"):+ buckets = aggs.get(agg_name, {}).get("buckets", [])+ if len(buckets) == OPENSEARCH_TERMS_AGG_LIMIT:+ logger.warning(+ "ID to filename mapping hit 10k limit - results may be truncated",+ connector_type=connector_type,+ agg=agg_name,+ returned_count=len(buckets),+ )+ for bucket in buckets:+ key = bucket.get("key")+ if not key:+ continue+ fn_buckets = bucket.get("top_filename", {}).get("buckets", [])+ mapping[key] = fn_buckets[0]["key"] if fn_buckets else ""return mappingexcept Exception as e:logger.error(diff --git a/tests/unit/api/test_connector_change_detection.py b/tests/unit/api/test_connector_change_detection.pyindex 52e0d4ba1..653d45c74 100644--- a/tests/unit/api/test_connector_change_detection.py+++ b/tests/unit/api/test_connector_change_detection.py@@ -464,6 +464,163 @@ async def test_state_map_returns_empty_on_error(monkeypatch):assert result == {}+# ---------------------------------------------------------------------------+# get_synced_id_to_filename_map+# ---------------------------------------------------------------------------+++async def test_filename_map_includes_connector_file_id_only_buckets(monkeypatch):+ """Regression test: filenames for ids that only exist under connector_file_id+ (the standard ConnectorFileProcessor path used by OAuth connectors, S3, etc.)+ must not be dropped just because document_id holds an unrelated content hash."""+ from api import connectors as connectors_api++ monkeypatch.setattr(connectors_api, "get_index_name", lambda: "idx")++ opensearch_client = AsyncMock()+ opensearch_client.search = AsyncMock(+ return_value={+ "aggregations": {+ "by_connector_file_id": {+ "buckets": [+ {+ "key": "gdrive-file-guid",+ "top_filename": {"buckets": [{"key": "report.pdf"}]},+ },+ ]+ },+ "by_document_id": {+ "buckets": [+ # Content-hash document_id from the standard ingest path —+ # unrelated to any enumerated source id.+ {+ "key": "content-hash-abc",+ "top_filename": {"buckets": [{"key": "report.pdf"}]},+ },+ ]+ },+ }+ }+ )+ sm = MagicMock()+ sm.get_user_opensearch_client = MagicMock(return_value=opensearch_client)++ result = await connectors_api.get_synced_id_to_filename_map(+ connector_type="google_drive",+ user_id="alice",+ session_manager=sm,+ jwt_token=None,+ )++ assert result["gdrive-file-guid"] == "report.pdf"+ assert result["content-hash-abc"] == "report.pdf"+++async def test_filename_map_prefers_connector_file_id_over_document_id(monkeypatch):+ from api import connectors as connectors_api++ monkeypatch.setattr(connectors_api, "get_index_name", lambda: "idx")++ opensearch_client = AsyncMock()+ opensearch_client.search = AsyncMock(+ return_value={+ "aggregations": {+ "by_connector_file_id": {+ "buckets": [+ {"key": "c::a", "top_filename": {"buckets": [{"key": "real-name.txt"}]}},+ ]+ },+ "by_document_id": {+ "buckets": [+ # Same key from the content-hash side, stale/wrong filename —+ # connector_file_id must win.+ {"key": "c::a", "top_filename": {"buckets": [{"key": "stale-name.txt"}]}},+ ]+ },+ }+ }+ )+ sm = MagicMock()+ sm.get_user_opensearch_client = MagicMock(return_value=opensearch_client)++ result = await connectors_api.get_synced_id_to_filename_map(+ connector_type="azure_blob",+ user_id="alice",+ session_manager=sm,+ jwt_token=None,+ )++ assert result["c::a"] == "real-name.txt"+++async def test_filename_map_falls_back_to_keyword_subfield_on_text_field_error(+ monkeypatch,+):+ from api import connectors as connectors_api++ monkeypatch.setattr(connectors_api, "get_index_name", lambda: "idx")++ opensearch_client = AsyncMock()+ called_fields = []++ async def fake_search(*, index, body):+ called_fields.append(body["aggs"]["by_connector_file_id"]["terms"]["field"])+ if len(called_fields) == 1:+ raise Exception(+ "RequestError(400, 'search_phase_execution_exception', 'Text fields "+ "are not optimised for operations that require per-document field "+ "data like aggregations and sorting, so these operations are "+ "disabled by default. Please use a keyword field instead. "+ "Alternatively, set fielddata=true on [connector_file_id]...')"+ )+ return {+ "aggregations": {+ "by_connector_file_id": {+ "buckets": [+ {"key": "c::a", "top_filename": {"buckets": [{"key": "real-name.txt"}]}},+ ]+ },+ "by_document_id": {"buckets": []},+ }+ }++ opensearch_client.search = AsyncMock(side_effect=fake_search)+ sm = MagicMock()+ sm.get_user_opensearch_client = MagicMock(return_value=opensearch_client)++ result = await connectors_api.get_synced_id_to_filename_map(+ connector_type="azure_blob",+ user_id="alice",+ session_manager=sm,+ jwt_token=None,+ )++ assert result == {"c::a": "real-name.txt"}+ assert called_fields == ["connector_file_id", "connector_file_id.keyword"]+++async def test_filename_map_returns_empty_on_error(monkeypatch):+ from api import connectors as connectors_api++ monkeypatch.setattr(connectors_api, "get_index_name", lambda: "idx")+ opensearch_client = AsyncMock()+ opensearch_client.search = AsyncMock(side_effect=RuntimeError("boom"))+ sm = MagicMock()+ sm.get_user_opensearch_client = MagicMock(return_value=opensearch_client)++ result = await connectors_api.get_synced_id_to_filename_map(+ connector_type="azure_blob",+ user_id="alice",+ session_manager=sm,+ jwt_token=None,+ )+ assert result == {}++# ---------------------------------------------------------------------------# get_synced_file_ids_for_connector# ---------------------------------------------------------------------------