OpenRAG is a comprehensive, single package Retrieval-Augmented Generation platform built on Langflow, Docling, and Opensearch.
Python
fix: support big csv files on langflow-less ingestion (#1968)
Lucas Oliveira6beb53841f04
diff --git a/src/models/processors.py b/src/models/processors.pyindex 71021ae94..0db7e77cc 100644--- a/src/models/processors.py+++ b/src/models/processors.py@@ -9,6 +9,7 @@ from utils.document_processing import (extract_relevant,process_text_file,resplit_chunks_character_windows,+ split_chunks_by_max_tokens,)from utils.file_utils import (auto_cleanup_tempfile,@@ -396,7 +397,6 @@ class TaskProcessor:# This ensures the length of chunks matches the length of the embeddings array,# since chunk_texts_for_embeddings also drops empty texts.slim_doc["chunks"] = [c for c in slim_doc["chunks"] if c.get("text") and c["text"].strip()]- texts = [c["text"] for c in slim_doc["chunks"]]litellm_embedding_model = (await self.models_service.get_litellm_model_name(embedding_model)@@ -405,10 +405,21 @@ class TaskProcessor:)# Split into batches to avoid token limits (8191 limit, use 8000 with buffer or 2000 if it's ollama)- if "ollama" in litellm_embedding_model:- text_batches = chunk_texts_for_embeddings(texts, max_tokens=2000)- else:- text_batches = chunk_texts_for_embeddings(texts, max_tokens=8000)+ max_tokens = (+ 2000+ if litellm_embedding_model and "ollama" in litellm_embedding_model.lower()+ else 8000+ )++ # Split any chunks that exceed max_tokens before embedding, ensuring chunks and embeddings align 1-to-1.+ slim_doc["chunks"] = split_chunks_by_max_tokens(+ slim_doc["chunks"], max_tokens, litellm_embedding_model+ )+ # Re-filter out chunks with empty or whitespace-only text that may have resulted from splitting+ slim_doc["chunks"] = [c for c in slim_doc["chunks"] if c.get("text") and c["text"].strip()]+ texts = [c["text"] for c in slim_doc["chunks"]]++ text_batches = chunk_texts_for_embeddings(texts, max_tokens=max_tokens)embeddings = []for batch in text_batches:diff --git a/src/utils/document_processing.py b/src/utils/document_processing.pyindex 6fff1c129..e35b8a353 100644--- a/src/utils/document_processing.py+++ b/src/utils/document_processing.py@@ -108,11 +108,13 @@ def extract_relevant(doc_dict: dict) -> dict:# group cells by their row indexrows = defaultdict(list)- for cell in table.get("data").get("table_cells", []):- r = cell.get("start_row_offset_idx")- c = cell.get("start_col_offset_idx")- text = cell.get("text", "").strip()- rows[r].append((c, text))+ table_data = table.get("data")+ if table_data:+ for cell in table_data.get("table_cells", []):+ r = cell.get("start_row_offset_idx")+ c = cell.get("start_col_offset_idx")+ text = cell.get("text", "").strip()+ rows[r].append((c, text))# build a tab‑separated line for each row, in orderflat_rows = []@@ -174,3 +176,45 @@ def resplit_chunks_character_windows(breakstart += stridereturn out+++def split_chunks_by_max_tokens(+ chunks: list[dict], max_tokens: int, model: str | None = None+) -> list[dict]:+ """Split any chunks whose token count exceeds max_tokens.++ Ensures that when chunk texts are sent to the embedding model, they don't get+ further split by token limits in the batching helper, allowing the final+ embeddings and chunks to align 1-to-1.+ """+ import tiktoken++ from config.settings import get_embedding_model++ model = model or get_embedding_model()+ try:+ encoding = tiktoken.encoding_for_model(model)+ except KeyError:+ encoding = tiktoken.get_encoding("cl100k_base")++ new_chunks = []+ for chunk in chunks:+ text = chunk.get("text", "")+ if not text:+ new_chunks.append(chunk)+ continue++ tokens = encoding.encode(text)+ if len(tokens) <= max_tokens:+ new_chunks.append(chunk)+ continue++ # Split tokens into segments of at most max_tokens size+ for i in range(0, len(tokens), max_tokens):+ chunk_tokens = tokens[i : i + max_tokens]+ chunk_text = encoding.decode(chunk_tokens)+ new_chunk = dict(chunk)+ new_chunk["text"] = chunk_text+ new_chunks.append(new_chunk)++ return new_chunksdiff --git a/tests/integration/core/test_non_langflow_ingestion.py b/tests/integration/core/test_non_langflow_ingestion.pynew file mode 100644index 000000000..2df654d36--- /dev/null+++ b/tests/integration/core/test_non_langflow_ingestion.py@@ -0,0 +1,197 @@+import asyncio+import os+from pathlib import Path+from unittest.mock import AsyncMock, patch++import httpx+import pytest+++# Pop cached modules so they reload with modified env vars+def _purge_modules():+ import sys++ for mod in [+ "api.router",+ "api.connector_router",+ "config.settings",+ "auth_middleware",+ "main",+ "api",+ "services",+ "services.search_service",+ "services.default_docs_service",+ "services.startup_orchestrator",+ "app.routes.internal",+ "app.routes",+ "app.container",+ "app.factory",+ "app.lifespan",+ "dependencies",+ "utils.opensearch_init",+ ]:+ sys.modules.pop(mod, None)+++async def wait_for_service_ready(client: httpx.AsyncClient, timeout_s: float = 30.0):+ deadline = asyncio.get_event_loop().time() + timeout_s+ last_err = None+ while asyncio.get_event_loop().time() < deadline:+ try:+ r1 = await client.get("/auth/me")+ if r1.status_code != 200:+ await asyncio.sleep(0.5)+ continue+ r2 = await client.post("/search", json={"query": "*", "limit": 0})+ if r2.status_code == 200:+ return+ last_err = r2.text+ except Exception as e:+ last_err = str(e)+ await asyncio.sleep(0.5)+ raise AssertionError(f"Service not ready in time: {last_err}")+++async def wait_for_task_completion(+ client: httpx.AsyncClient, task_id: str, timeout_s: float = 60.0+) -> dict:+ deadline = asyncio.get_event_loop().time() + timeout_s+ last_payload = None+ while asyncio.get_event_loop().time() < deadline:+ resp = await client.get(f"/tasks/{task_id}")+ if resp.status_code == 200:+ try:+ data = resp.json()+ except Exception:+ last_payload = resp.text+ else:+ status = (data.get("status") or "").lower()+ if status == "completed":+ return data+ if status == "failed":+ raise AssertionError(f"Task {task_id} failed: {data}")+ last_payload = data+ await asyncio.sleep(1.0)+ raise AssertionError(f"Task {task_id} did not complete in time. Last payload: {last_payload}")+++async def test_non_langflow_csv_ingestion_with_splitting(tmp_path: Path):+ """Validate standard CSV ingestion using standard non-Langflow processing pipeline.++ Simulates parsing of a CSV file yielding a large table chunk (exceeding standard 8,000 token limit),+ verifying it gets split correctly and indexed into OpenSearch.+ """+ os.environ["DISABLE_INGEST_WITH_LANGFLOW"] = "true"+ os.environ["DISABLE_STARTUP_INGEST"] = "true"+ os.environ["EMBEDDING_MODEL"] = "text-embedding-3-small"+ os.environ["EMBEDDING_PROVIDER"] = "openai"+ os.environ["GOOGLE_OAUTH_CLIENT_ID"] = ""+ os.environ["GOOGLE_OAUTH_CLIENT_SECRET"] = ""++ _purge_modules()++ from config.settings import clients, get_index_name+ from main import create_app, startup_tasks++ await clients.initialize()+ try:+ await clients.opensearch.indices.delete(index=get_index_name())+ await asyncio.sleep(1)+ except Exception:+ pass++ app = await create_app()+ await startup_tasks(app.state.services)++ from main import _ensure_opensearch_index++ await _ensure_opensearch_index()++ # Mock the DoclingService conversion result+ # We want a table chunk whose token count exceeds 8,000 tokens (so it gets split)+ # "testlargephrase " is 3 tokens in cl100k_base. Repeating it 4000 times gives ~12,000 tokens.+ large_table_text = "testlargephrase " * 4000++ mock_docling_result = {+ "origin": {+ "binary_hash": "sha-csv-integration-123",+ "filename": "correlation_report.csv",+ "mimetype": "text/csv",+ },+ "texts": [],+ "tables": [+ {+ "prov": [{"page_no": 1}],+ "data": {+ "table_cells": [+ {+ "start_row_offset_idx": 0,+ "start_col_offset_idx": 0,+ "text": large_table_text,+ }+ ]+ },+ }+ ],+ }++ # Patch convert_file method on DoclingService class+ from services.docling_service import DoclingService++ with patch.object(DoclingService, "convert_file", AsyncMock(return_value=mock_docling_result)):+ transport = httpx.ASGITransport(app=app)+ async with httpx.AsyncClient(transport=transport, base_url="http://testserver") as client:+ await wait_for_service_ready(client)++ # Create mock CSV+ csv_path = tmp_path / "correlation_report.csv"+ csv_path.write_text("header1,header2\nvalue1,value2")++ files = {+ "file": (+ csv_path.name,+ csv_path.read_bytes(),+ "text/csv",+ )+ }++ resp = await client.post("/router/upload_ingest", files=files)+ assert resp.status_code == 202, resp.text++ data = resp.json()+ task_id = data.get("task_id")+ assert task_id is not None++ # Wait for processing to complete+ await wait_for_task_completion(client, task_id)++ # Wait for search indices to refresh+ await asyncio.sleep(1)++ # Retrieve results from OpenSearch using search endpoint+ search_resp = await client.post(+ "/search",+ json={"query": "testlargephrase", "limit": 10},+ )+ assert search_resp.status_code == 200, search_resp.text++ results = search_resp.json().get("results", [])+ assert len(results) > 0, "No search results returned from indexed chunks"++ # Check direct OpenSearch doc count for this document ID+ opensearch_resp = await clients.opensearch.search(+ index=get_index_name(),+ body={"query": {"term": {"document_id": "sha-csv-integration-123"}}},+ )+ hits = opensearch_resp.get("hits", {}).get("hits", [])++ # Since the text is ~12,000 tokens, and max_tokens=8000,+ # it should be split into 2 chunks, resulting in 2 documents in OpenSearch.+ assert len(hits) == 2, f"Expected 2 indexed chunks, but got {len(hits)}"+ for hit in hits:+ source = hit.get("_source", {})+ assert source.get("document_id") == "sha-csv-integration-123"+ assert source.get("filename") == "correlation_report.csv"+ assert source.get("mimetype") == "text/csv"+ assert "testlargephrase" in source.get("text", "")diff --git a/tests/unit/test_document_processing_resplit.py b/tests/unit/test_document_processing_resplit.pyindex 8b097b5ee..919fa5a3e 100644--- a/tests/unit/test_document_processing_resplit.py+++ b/tests/unit/test_document_processing_resplit.py@@ -64,3 +64,71 @@ def test_extract_relevant_with_missing_page_no():assert result["chunks"][0]["page"] == 1assert "Hello Asciidoc World!" in result["chunks"][0]["text"]assert "Second paragraph of text." in result["chunks"][0]["text"]+++def test_split_chunks_by_max_tokens_under_limit():+ from src.utils.document_processing import split_chunks_by_max_tokens++ chunks = [+ {"page": 1, "type": "text", "text": "hello world"},+ {"page": 2, "type": "table", "text": "some other text"},+ ]+ out = split_chunks_by_max_tokens(chunks, max_tokens=100)+ assert len(out) == 2+ assert out[0]["text"] == "hello world"+ assert out[1]["text"] == "some other text"+++def test_split_chunks_by_max_tokens_exceeding():+ from src.utils.document_processing import split_chunks_by_max_tokens++ # "hello world" is 2 tokens in cl100k_base encoding. If we limit to 1 token per chunk:+ chunks = [{"page": 5, "type": "table", "table_index": 0, "text": "hello world"}]+ # We use cl100k_base (standard for gpt-4/text-embedding-3)+ out = split_chunks_by_max_tokens(chunks, max_tokens=1, model="text-embedding-3-small")+ assert len(out) == 2+ assert out[0]["page"] == 5+ assert out[0]["type"] == "table"+ assert out[0]["table_index"] == 0+ assert out[0]["text"] == "hello"+ assert out[1]["page"] == 5+ assert out[1]["type"] == "table"+ assert out[1]["table_index"] == 0+ assert out[1]["text"] == " world"+++def test_extract_relevant_safeguard_none_table_data():+ from src.utils.document_processing import extract_relevant++ doc_dict = {+ "origin": {"binary_hash": "hash456", "filename": "test.csv", "mimetype": "text/csv"},+ "texts": [],+ "tables": [+ {+ "prov": [{"page_no": 1}],+ "data": None, # Simulate missing table data / structure+ }+ ],+ }+ result = extract_relevant(doc_dict)+ assert result["id"] == "hash456"+ assert len(result["chunks"]) == 1+ assert result["chunks"][0]["type"] == "table"+ assert (+ result["chunks"][0]["text"] == ""+ ) # should handle None data and produce empty string or no cells+++def test_split_chunks_by_max_tokens_emoji_token_override():+ from src.utils.document_processing import split_chunks_by_max_tokens++ # "👨👩👧👦" has len() = 7 in Python, but is 18 tokens in cl100k_base.+ # If max_tokens is 10, then character count (7) < max_tokens (10),+ # but token count (18) > max_tokens (10). It must be split.+ chunks = [{"page": 1, "type": "text", "text": "👨👩👧👦"}]+ out = split_chunks_by_max_tokens(chunks, max_tokens=10, model="text-embedding-3-small")+ # Must be split because 18 > 10.+ assert len(out) > 1+ # Check that chunks reconstruct to the original text+ reconstructed = "".join([c["text"] for c in out])+ assert reconstructed == "👨👩👧👦"