Skip to content

OpenRAG is a comprehensive, single package Retrieval-Augmented Generation platform built on Langflow, Docling, and Opensearch.

Python

fix: support big csv files on langflow-less ingestion (#1968)

Lucas Oliveira6beb53841f04
diff --git a/src/models/processors.py b/src/models/processors.py
index 71021ae94..0db7e77cc 100644
--- a/src/models/processors.py
+++ b/src/models/processors.py
@@ -9,6 +9,7 @@ from utils.document_processing import (
extract_relevant,
process_text_file,
resplit_chunks_character_windows,
+ split_chunks_by_max_tokens,
)
from utils.file_utils import (
auto_cleanup_tempfile,
@@ -396,7 +397,6 @@ class TaskProcessor:
# This ensures the length of chunks matches the length of the embeddings array,
# since chunk_texts_for_embeddings also drops empty texts.
slim_doc["chunks"] = [c for c in slim_doc["chunks"] if c.get("text") and c["text"].strip()]
- texts = [c["text"] for c in slim_doc["chunks"]]
litellm_embedding_model = (
await self.models_service.get_litellm_model_name(embedding_model)
@@ -405,10 +405,21 @@ class TaskProcessor:
)
# Split into batches to avoid token limits (8191 limit, use 8000 with buffer or 2000 if it's ollama)
- if "ollama" in litellm_embedding_model:
- text_batches = chunk_texts_for_embeddings(texts, max_tokens=2000)
- else:
- text_batches = chunk_texts_for_embeddings(texts, max_tokens=8000)
+ max_tokens = (
+ 2000
+ if litellm_embedding_model and "ollama" in litellm_embedding_model.lower()
+ else 8000
+ )
+
+ # Split any chunks that exceed max_tokens before embedding, ensuring chunks and embeddings align 1-to-1.
+ slim_doc["chunks"] = split_chunks_by_max_tokens(
+ slim_doc["chunks"], max_tokens, litellm_embedding_model
+ )
+ # Re-filter out chunks with empty or whitespace-only text that may have resulted from splitting
+ slim_doc["chunks"] = [c for c in slim_doc["chunks"] if c.get("text") and c["text"].strip()]
+ texts = [c["text"] for c in slim_doc["chunks"]]
+
+ text_batches = chunk_texts_for_embeddings(texts, max_tokens=max_tokens)
embeddings = []
for batch in text_batches:
diff --git a/src/utils/document_processing.py b/src/utils/document_processing.py
index 6fff1c129..e35b8a353 100644
--- a/src/utils/document_processing.py
+++ b/src/utils/document_processing.py
@@ -108,11 +108,13 @@ def extract_relevant(doc_dict: dict) -> dict:
# group cells by their row index
rows = defaultdict(list)
- for cell in table.get("data").get("table_cells", []):
- r = cell.get("start_row_offset_idx")
- c = cell.get("start_col_offset_idx")
- text = cell.get("text", "").strip()
- rows[r].append((c, text))
+ table_data = table.get("data")
+ if table_data:
+ for cell in table_data.get("table_cells", []):
+ r = cell.get("start_row_offset_idx")
+ c = cell.get("start_col_offset_idx")
+ text = cell.get("text", "").strip()
+ rows[r].append((c, text))
# build a tab‑separated line for each row, in order
flat_rows = []
@@ -174,3 +176,45 @@ def resplit_chunks_character_windows(
break
start += stride
return out
+
+
+def split_chunks_by_max_tokens(
+ chunks: list[dict], max_tokens: int, model: str | None = None
+) -> list[dict]:
+ """Split any chunks whose token count exceeds max_tokens.
+
+ Ensures that when chunk texts are sent to the embedding model, they don't get
+ further split by token limits in the batching helper, allowing the final
+ embeddings and chunks to align 1-to-1.
+ """
+ import tiktoken
+
+ from config.settings import get_embedding_model
+
+ model = model or get_embedding_model()
+ try:
+ encoding = tiktoken.encoding_for_model(model)
+ except KeyError:
+ encoding = tiktoken.get_encoding("cl100k_base")
+
+ new_chunks = []
+ for chunk in chunks:
+ text = chunk.get("text", "")
+ if not text:
+ new_chunks.append(chunk)
+ continue
+
+ tokens = encoding.encode(text)
+ if len(tokens) <= max_tokens:
+ new_chunks.append(chunk)
+ continue
+
+ # Split tokens into segments of at most max_tokens size
+ for i in range(0, len(tokens), max_tokens):
+ chunk_tokens = tokens[i : i + max_tokens]
+ chunk_text = encoding.decode(chunk_tokens)
+ new_chunk = dict(chunk)
+ new_chunk["text"] = chunk_text
+ new_chunks.append(new_chunk)
+
+ return new_chunks
diff --git a/tests/integration/core/test_non_langflow_ingestion.py b/tests/integration/core/test_non_langflow_ingestion.py
new file mode 100644
index 000000000..2df654d36
--- /dev/null
+++ b/tests/integration/core/test_non_langflow_ingestion.py
@@ -0,0 +1,197 @@
+import asyncio
+import os
+from pathlib import Path
+from unittest.mock import AsyncMock, patch
+
+import httpx
+import pytest
+
+
+# Pop cached modules so they reload with modified env vars
+def _purge_modules():
+ import sys
+
+ for mod in [
+ "api.router",
+ "api.connector_router",
+ "config.settings",
+ "auth_middleware",
+ "main",
+ "api",
+ "services",
+ "services.search_service",
+ "services.default_docs_service",
+ "services.startup_orchestrator",
+ "app.routes.internal",
+ "app.routes",
+ "app.container",
+ "app.factory",
+ "app.lifespan",
+ "dependencies",
+ "utils.opensearch_init",
+ ]:
+ sys.modules.pop(mod, None)
+
+
+async def wait_for_service_ready(client: httpx.AsyncClient, timeout_s: float = 30.0):
+ deadline = asyncio.get_event_loop().time() + timeout_s
+ last_err = None
+ while asyncio.get_event_loop().time() < deadline:
+ try:
+ r1 = await client.get("/auth/me")
+ if r1.status_code != 200:
+ await asyncio.sleep(0.5)
+ continue
+ r2 = await client.post("/search", json={"query": "*", "limit": 0})
+ if r2.status_code == 200:
+ return
+ last_err = r2.text
+ except Exception as e:
+ last_err = str(e)
+ await asyncio.sleep(0.5)
+ raise AssertionError(f"Service not ready in time: {last_err}")
+
+
+async def wait_for_task_completion(
+ client: httpx.AsyncClient, task_id: str, timeout_s: float = 60.0
+) -> dict:
+ deadline = asyncio.get_event_loop().time() + timeout_s
+ last_payload = None
+ while asyncio.get_event_loop().time() < deadline:
+ resp = await client.get(f"/tasks/{task_id}")
+ if resp.status_code == 200:
+ try:
+ data = resp.json()
+ except Exception:
+ last_payload = resp.text
+ else:
+ status = (data.get("status") or "").lower()
+ if status == "completed":
+ return data
+ if status == "failed":
+ raise AssertionError(f"Task {task_id} failed: {data}")
+ last_payload = data
+ await asyncio.sleep(1.0)
+ raise AssertionError(f"Task {task_id} did not complete in time. Last payload: {last_payload}")
+
+
+async def test_non_langflow_csv_ingestion_with_splitting(tmp_path: Path):
+ """Validate standard CSV ingestion using standard non-Langflow processing pipeline.
+
+ Simulates parsing of a CSV file yielding a large table chunk (exceeding standard 8,000 token limit),
+ verifying it gets split correctly and indexed into OpenSearch.
+ """
+ os.environ["DISABLE_INGEST_WITH_LANGFLOW"] = "true"
+ os.environ["DISABLE_STARTUP_INGEST"] = "true"
+ os.environ["EMBEDDING_MODEL"] = "text-embedding-3-small"
+ os.environ["EMBEDDING_PROVIDER"] = "openai"
+ os.environ["GOOGLE_OAUTH_CLIENT_ID"] = ""
+ os.environ["GOOGLE_OAUTH_CLIENT_SECRET"] = ""
+
+ _purge_modules()
+
+ from config.settings import clients, get_index_name
+ from main import create_app, startup_tasks
+
+ await clients.initialize()
+ try:
+ await clients.opensearch.indices.delete(index=get_index_name())
+ await asyncio.sleep(1)
+ except Exception:
+ pass
+
+ app = await create_app()
+ await startup_tasks(app.state.services)
+
+ from main import _ensure_opensearch_index
+
+ await _ensure_opensearch_index()
+
+ # Mock the DoclingService conversion result
+ # We want a table chunk whose token count exceeds 8,000 tokens (so it gets split)
+ # "testlargephrase " is 3 tokens in cl100k_base. Repeating it 4000 times gives ~12,000 tokens.
+ large_table_text = "testlargephrase " * 4000
+
+ mock_docling_result = {
+ "origin": {
+ "binary_hash": "sha-csv-integration-123",
+ "filename": "correlation_report.csv",
+ "mimetype": "text/csv",
+ },
+ "texts": [],
+ "tables": [
+ {
+ "prov": [{"page_no": 1}],
+ "data": {
+ "table_cells": [
+ {
+ "start_row_offset_idx": 0,
+ "start_col_offset_idx": 0,
+ "text": large_table_text,
+ }
+ ]
+ },
+ }
+ ],
+ }
+
+ # Patch convert_file method on DoclingService class
+ from services.docling_service import DoclingService
+
+ with patch.object(DoclingService, "convert_file", AsyncMock(return_value=mock_docling_result)):
+ transport = httpx.ASGITransport(app=app)
+ async with httpx.AsyncClient(transport=transport, base_url="http://testserver") as client:
+ await wait_for_service_ready(client)
+
+ # Create mock CSV
+ csv_path = tmp_path / "correlation_report.csv"
+ csv_path.write_text("header1,header2\nvalue1,value2")
+
+ files = {
+ "file": (
+ csv_path.name,
+ csv_path.read_bytes(),
+ "text/csv",
+ )
+ }
+
+ resp = await client.post("/router/upload_ingest", files=files)
+ assert resp.status_code == 202, resp.text
+
+ data = resp.json()
+ task_id = data.get("task_id")
+ assert task_id is not None
+
+ # Wait for processing to complete
+ await wait_for_task_completion(client, task_id)
+
+ # Wait for search indices to refresh
+ await asyncio.sleep(1)
+
+ # Retrieve results from OpenSearch using search endpoint
+ search_resp = await client.post(
+ "/search",
+ json={"query": "testlargephrase", "limit": 10},
+ )
+ assert search_resp.status_code == 200, search_resp.text
+
+ results = search_resp.json().get("results", [])
+ assert len(results) > 0, "No search results returned from indexed chunks"
+
+ # Check direct OpenSearch doc count for this document ID
+ opensearch_resp = await clients.opensearch.search(
+ index=get_index_name(),
+ body={"query": {"term": {"document_id": "sha-csv-integration-123"}}},
+ )
+ hits = opensearch_resp.get("hits", {}).get("hits", [])
+
+ # Since the text is ~12,000 tokens, and max_tokens=8000,
+ # it should be split into 2 chunks, resulting in 2 documents in OpenSearch.
+ assert len(hits) == 2, f"Expected 2 indexed chunks, but got {len(hits)}"
+ for hit in hits:
+ source = hit.get("_source", {})
+ assert source.get("document_id") == "sha-csv-integration-123"
+ assert source.get("filename") == "correlation_report.csv"
+ assert source.get("mimetype") == "text/csv"
+ assert "testlargephrase" in source.get("text", "")
diff --git a/tests/unit/test_document_processing_resplit.py b/tests/unit/test_document_processing_resplit.py
index 8b097b5ee..919fa5a3e 100644
--- a/tests/unit/test_document_processing_resplit.py
+++ b/tests/unit/test_document_processing_resplit.py
@@ -64,3 +64,71 @@ def test_extract_relevant_with_missing_page_no():
assert result["chunks"][0]["page"] == 1
assert "Hello Asciidoc World!" in result["chunks"][0]["text"]
assert "Second paragraph of text." in result["chunks"][0]["text"]
+
+
+def test_split_chunks_by_max_tokens_under_limit():
+ from src.utils.document_processing import split_chunks_by_max_tokens
+
+ chunks = [
+ {"page": 1, "type": "text", "text": "hello world"},
+ {"page": 2, "type": "table", "text": "some other text"},
+ ]
+ out = split_chunks_by_max_tokens(chunks, max_tokens=100)
+ assert len(out) == 2
+ assert out[0]["text"] == "hello world"
+ assert out[1]["text"] == "some other text"
+
+
+def test_split_chunks_by_max_tokens_exceeding():
+ from src.utils.document_processing import split_chunks_by_max_tokens
+
+ # "hello world" is 2 tokens in cl100k_base encoding. If we limit to 1 token per chunk:
+ chunks = [{"page": 5, "type": "table", "table_index": 0, "text": "hello world"}]
+ # We use cl100k_base (standard for gpt-4/text-embedding-3)
+ out = split_chunks_by_max_tokens(chunks, max_tokens=1, model="text-embedding-3-small")
+ assert len(out) == 2
+ assert out[0]["page"] == 5
+ assert out[0]["type"] == "table"
+ assert out[0]["table_index"] == 0
+ assert out[0]["text"] == "hello"
+ assert out[1]["page"] == 5
+ assert out[1]["type"] == "table"
+ assert out[1]["table_index"] == 0
+ assert out[1]["text"] == " world"
+
+
+def test_extract_relevant_safeguard_none_table_data():
+ from src.utils.document_processing import extract_relevant
+
+ doc_dict = {
+ "origin": {"binary_hash": "hash456", "filename": "test.csv", "mimetype": "text/csv"},
+ "texts": [],
+ "tables": [
+ {
+ "prov": [{"page_no": 1}],
+ "data": None, # Simulate missing table data / structure
+ }
+ ],
+ }
+ result = extract_relevant(doc_dict)
+ assert result["id"] == "hash456"
+ assert len(result["chunks"]) == 1
+ assert result["chunks"][0]["type"] == "table"
+ assert (
+ result["chunks"][0]["text"] == ""
+ ) # should handle None data and produce empty string or no cells
+
+
+def test_split_chunks_by_max_tokens_emoji_token_override():
+ from src.utils.document_processing import split_chunks_by_max_tokens
+
+ # "👨‍👩‍👧‍👦" has len() = 7 in Python, but is 18 tokens in cl100k_base.
+ # If max_tokens is 10, then character count (7) < max_tokens (10),
+ # but token count (18) > max_tokens (10). It must be split.
+ chunks = [{"page": 1, "type": "text", "text": "👨‍👩‍👧‍👦"}]
+ out = split_chunks_by_max_tokens(chunks, max_tokens=10, model="text-embedding-3-small")
+ # Must be split because 18 > 10.
+ assert len(out) > 1
+ # Check that chunks reconstruct to the original text
+ reconstructed = "".join([c["text"] for c in out])
+ assert reconstructed == "👨‍👩‍👧‍👦"