OpenRAG is a comprehensive, single package Retrieval-Augmented Generation platform built on Langflow, Docling, and Opensearch.
Python
fix: add warning logs for 10k aggregation limit (#2067)
yogichipalkatti9b536d0f3f3b
diff --git a/src/api/connectors.py b/src/api/connectors.pyindex 7b35f02c7..949fc06ed 100644--- a/src/api/connectors.py+++ b/src/api/connectors.py@@ -33,6 +33,12 @@ from utils.telemetry import Category, MessageId, TelemetryClientlogger = get_logger(__name__)+# Maximum number of buckets returned by a single OpenSearch terms aggregation.+# Results are silently truncated when the true cardinality exceeds this value.+# See get_synced_file_ids_for_connector / get_synced_id_to_filename_map.+# TODO: replace with composite aggregation + pagination to handle arbitrary cardinality.+OPENSEARCH_TERMS_AGG_LIMIT = 10_000+async def _connector_access_denied(request: Request,@@ -98,10 +104,14 @@ async def get_synced_file_ids_for_connector("query": {"term": {"connector_type": connector_type}},"aggs": {"unique_connector_file_ids": {- "terms": {"field": "connector_file_id", "size": 10000}+ "terms": {"field": "connector_file_id", "size": OPENSEARCH_TERMS_AGG_LIMIT}+ },+ "unique_document_ids": {+ "terms": {"field": "document_id", "size": OPENSEARCH_TERMS_AGG_LIMIT}+ },+ "unique_filenames": {+ "terms": {"field": "filename", "size": OPENSEARCH_TERMS_AGG_LIMIT}},- "unique_document_ids": {"terms": {"field": "document_id", "size": 10000}},- "unique_filenames": {"terms": {"field": "filename", "size": 10000}},},}@@ -113,7 +123,12 @@ async def get_synced_file_ids_for_connector(result.get("aggregations", {}).get("unique_connector_file_ids", {}).get("buckets", []))connector_file_ids = [b["key"] for b in connector_file_id_buckets if b["key"]]-+ if len(connector_file_id_buckets) == OPENSEARCH_TERMS_AGG_LIMIT:+ logger.warning(+ "Connector file ID aggregation hit 10k limit - results may be truncated",+ connector_type=connector_type,+ returned_count=len(connector_file_ids),+ )if connector_file_ids:file_ids = connector_file_idsid_field = "connector_file_id"@@ -123,13 +138,24 @@ async def get_synced_file_ids_for_connector(result.get("aggregations", {}).get("unique_document_ids", {}).get("buckets", []))file_ids = [b["key"] for b in doc_id_buckets if b["key"]]+ if len(doc_id_buckets) == OPENSEARCH_TERMS_AGG_LIMIT:+ logger.warning(+ "Document ID aggregation hit 10k limit - results may be truncated",+ connector_type=connector_type,+ returned_count=len(file_ids),+ )id_field = "document_id"filename_buckets = (result.get("aggregations", {}).get("unique_filenames", {}).get("buckets", []))filenames = [b["key"] for b in filename_buckets if b["key"]]-+ if len(filename_buckets) == OPENSEARCH_TERMS_AGG_LIMIT:+ logger.warning(+ "Filename aggregation hit 10k limit - results may be truncated",+ connector_type=connector_type,+ returned_count=len(filenames),+ )logger.debug("Found synced files for connector",connector_type=connector_type,@@ -168,7 +194,7 @@ async def get_synced_id_to_filename_map("query": {"term": {"connector_type": connector_type}},"aggs": {"by_document_id": {- "terms": {"field": "document_id", "size": 10000},+ "terms": {"field": "document_id", "size": OPENSEARCH_TERMS_AGG_LIMIT},"aggs": {"top_filename": {"terms": {"field": "filename", "size": 1}},},@@ -178,6 +204,12 @@ async def get_synced_id_to_filename_map(result = await opensearch_client.search(index=get_index_name(), body=query_body)buckets = result.get("aggregations", {}).get("by_document_id", {}).get("buckets", [])+ if len(buckets) == OPENSEARCH_TERMS_AGG_LIMIT:+ logger.warning(+ "Document ID to filename mapping hit 10k limit - results may be truncated",+ connector_type=connector_type,+ returned_count=len(buckets),+ )mapping: dict[str, str] = {}for bucket in buckets: