Skip to content

OpenRAG is a comprehensive, single package Retrieval-Augmented Generation platform built on Langflow, Docling, and Opensearch.

Python

fix: add limit for max tokens in a batch for watsonx (#2119)

Lucas Oliveira47a22acec69d
diff --git a/src/models/processors.py b/src/models/processors.py
index a45171705..5f9970912 100644
--- a/src/models/processors.py
+++ b/src/models/processors.py
@@ -437,9 +437,15 @@ class TaskProcessor:
# Use provided embedding model or configured model.
# get_embedding_model() returns empty string when Langflow ingest is enabled,
# but OpenRAG processors still need a concrete embedding model.
- configured_embedding_model = get_openrag_config().knowledge.embedding_model
+ config = get_openrag_config()
+ configured_embedding_model = config.knowledge.embedding_model
embedding_model = embedding_model or configured_embedding_model or get_embedding_model()
+ if chunk_size is None:
+ chunk_size = config.knowledge.chunk_size
+ if chunk_overlap is None:
+ chunk_overlap = config.knowledge.chunk_overlap
+
# Get user's OpenSearch client with JWT for OIDC auth
opensearch_client = self.document_service.session_manager.get_user_opensearch_client(
owner_user_id, jwt_token
@@ -508,12 +514,13 @@ class TaskProcessor:
else embedding_model
)
- # Split into batches to avoid token limits (8191 limit, use 8000 with buffer or 2000 if it's ollama)
- max_tokens = (
- 2000
- if litellm_embedding_model and "ollama" in litellm_embedding_model.lower()
- else 8000
- )
+ litellm_model_lower = litellm_embedding_model.lower() if litellm_embedding_model else ""
+ if "watsonx" in litellm_model_lower:
+ max_tokens = 500
+ elif "ollama" in litellm_model_lower:
+ max_tokens = 2000
+ else:
+ max_tokens = 8000
# Split any chunks that exceed max_tokens before embedding, ensuring chunks and embeddings align 1-to-1.
slim_doc["chunks"] = split_chunks_by_max_tokens(
diff --git a/tests/integration/core/test_non_langflow_ingestion.py b/tests/integration/core/test_non_langflow_ingestion.py
index 7f5fd6fc4..dacfd5968 100644
--- a/tests/integration/core/test_non_langflow_ingestion.py
+++ b/tests/integration/core/test_non_langflow_ingestion.py
@@ -193,9 +193,9 @@ async def test_non_langflow_csv_ingestion_with_splitting(tmp_path: Path):
)
hits = opensearch_resp.get("hits", {}).get("hits", [])
- # Since the text is ~12,000 tokens, and max_tokens=8000,
- # it should be split into 2 chunks, resulting in 2 documents in OpenSearch.
- assert len(hits) == 2, f"Expected 2 indexed chunks, but got {len(hits)}"
+ # Since the text is ~12,000 tokens, and max_tokens=1000,
+ # it should be split into 10 chunks, resulting in 10 documents in OpenSearch.
+ assert len(hits) == 10, f"Expected 10 indexed chunks, but got {len(hits)}"
for hit in hits:
source = hit.get("_source", {})
assert source.get("document_id") == expected_document_id