Skip to content

OpenRAG is a comprehensive, single package Retrieval-Augmented Generation platform built on Langflow, Docling, and Opensearch.

Python

fix: add warning logs for 10k aggregation limit (#2067)

yogichipalkatti9b536d0f3f3b
diff --git a/src/api/connectors.py b/src/api/connectors.py
index 7b35f02c7..949fc06ed 100644
--- a/src/api/connectors.py
+++ b/src/api/connectors.py
@@ -33,6 +33,12 @@ from utils.telemetry import Category, MessageId, TelemetryClient
logger = get_logger(__name__)
+# Maximum number of buckets returned by a single OpenSearch terms aggregation.
+# Results are silently truncated when the true cardinality exceeds this value.
+# See get_synced_file_ids_for_connector / get_synced_id_to_filename_map.
+# TODO: replace with composite aggregation + pagination to handle arbitrary cardinality.
+OPENSEARCH_TERMS_AGG_LIMIT = 10_000
+
async def _connector_access_denied(
request: Request,
@@ -98,10 +104,14 @@ async def get_synced_file_ids_for_connector(
"query": {"term": {"connector_type": connector_type}},
"aggs": {
"unique_connector_file_ids": {
- "terms": {"field": "connector_file_id", "size": 10000}
+ "terms": {"field": "connector_file_id", "size": OPENSEARCH_TERMS_AGG_LIMIT}
+ },
+ "unique_document_ids": {
+ "terms": {"field": "document_id", "size": OPENSEARCH_TERMS_AGG_LIMIT}
+ },
+ "unique_filenames": {
+ "terms": {"field": "filename", "size": OPENSEARCH_TERMS_AGG_LIMIT}
},
- "unique_document_ids": {"terms": {"field": "document_id", "size": 10000}},
- "unique_filenames": {"terms": {"field": "filename", "size": 10000}},
},
}
@@ -113,7 +123,12 @@ async def get_synced_file_ids_for_connector(
result.get("aggregations", {}).get("unique_connector_file_ids", {}).get("buckets", [])
)
connector_file_ids = [b["key"] for b in connector_file_id_buckets if b["key"]]
-
+ if len(connector_file_id_buckets) == OPENSEARCH_TERMS_AGG_LIMIT:
+ logger.warning(
+ "Connector file ID aggregation hit 10k limit - results may be truncated",
+ connector_type=connector_type,
+ returned_count=len(connector_file_ids),
+ )
if connector_file_ids:
file_ids = connector_file_ids
id_field = "connector_file_id"
@@ -123,13 +138,24 @@ async def get_synced_file_ids_for_connector(
result.get("aggregations", {}).get("unique_document_ids", {}).get("buckets", [])
)
file_ids = [b["key"] for b in doc_id_buckets if b["key"]]
+ if len(doc_id_buckets) == OPENSEARCH_TERMS_AGG_LIMIT:
+ logger.warning(
+ "Document ID aggregation hit 10k limit - results may be truncated",
+ connector_type=connector_type,
+ returned_count=len(file_ids),
+ )
id_field = "document_id"
filename_buckets = (
result.get("aggregations", {}).get("unique_filenames", {}).get("buckets", [])
)
filenames = [b["key"] for b in filename_buckets if b["key"]]
-
+ if len(filename_buckets) == OPENSEARCH_TERMS_AGG_LIMIT:
+ logger.warning(
+ "Filename aggregation hit 10k limit - results may be truncated",
+ connector_type=connector_type,
+ returned_count=len(filenames),
+ )
logger.debug(
"Found synced files for connector",
connector_type=connector_type,
@@ -168,7 +194,7 @@ async def get_synced_id_to_filename_map(
"query": {"term": {"connector_type": connector_type}},
"aggs": {
"by_document_id": {
- "terms": {"field": "document_id", "size": 10000},
+ "terms": {"field": "document_id", "size": OPENSEARCH_TERMS_AGG_LIMIT},
"aggs": {
"top_filename": {"terms": {"field": "filename", "size": 1}},
},
@@ -178,6 +204,12 @@ async def get_synced_id_to_filename_map(
result = await opensearch_client.search(index=get_index_name(), body=query_body)
buckets = result.get("aggregations", {}).get("by_document_id", {}).get("buckets", [])
+ if len(buckets) == OPENSEARCH_TERMS_AGG_LIMIT:
+ logger.warning(
+ "Document ID to filename mapping hit 10k limit - results may be truncated",
+ connector_type=connector_type,
+ returned_count=len(buckets),
+ )
mapping: dict[str, str] = {}
for bucket in buckets: