OpenRAG is a comprehensive, single package Retrieval-Augmented Generation platform built on Langflow, Docling, and Opensearch.
Python
fix: ingestion failing for dense text file when langflow ingestion disabled with wxai models (#2591)
Wallgauebbe9218e457
diff --git a/src/models/processors.py b/src/models/processors.pyindex 7d02a5447..e2ed4990f 100644--- a/src/models/processors.py+++ b/src/models/processors.py@@ -740,7 +740,9 @@ class TaskProcessor:token_cap.set(non_empty=non_empty, chunks_out=len(slim_doc["chunks"]))texts = [c["text"] for c in slim_doc["chunks"]]- text_batches = chunk_texts_for_embeddings(texts, max_tokens=max_tokens)+ text_batches = chunk_texts_for_embeddings(+ texts, max_tokens=max_tokens, model=litellm_embedding_model+ )embeddings = []# Every provider goes through the gateway, not just the aliased ones.# The gateway resolves credentials per call from config; the directdiff --git a/src/services/document_service.py b/src/services/document_service.pyindex 2766f43b4..87dd38527 100644--- a/src/services/document_service.py+++ b/src/services/document_service.py@@ -22,9 +22,12 @@ def get_token_count(text: str, model: str = None) -> int:def chunk_texts_for_embeddings(texts: list[str], max_tokens: int = None, model: str = None) -> list[list[str]]:- """- Split texts into batches that won't exceed token limits.- If max_tokens is None, returns texts as single batch (no splitting).+ """Pack texts into batches that stay within ``max_tokens``.++ Each input stays one embedding input. A text already over the limit is+ placed in its own batch; splitting it here would make the embedding list+ longer than the chunk list. If ``max_tokens`` is None, texts are returned+ as a single batch."""model = model or get_embedding_model()@@ -44,40 +47,19 @@ def chunk_texts_for_embeddings(for text in texts:text_tokens = get_token_count(text, model)- # If single text exceeds limit, split it further- if text_tokens > max_tokens:- # If we have current batch, save it first+ over_limit = text_tokens > max_tokens+ if over_limit or (current_batch and current_tokens + text_tokens > max_tokens):if current_batch:batches.append(current_batch)current_batch = []current_tokens = 0+ if over_limit:+ batches.append([text])+ continue- # Split the large text into smaller chunks- try:- encoding = tiktoken.encoding_for_model(model)- except KeyError:- encoding = tiktoken.get_encoding("cl100k_base")-- tokens = encoding.encode(text)-- for i in range(0, len(tokens), max_tokens):- chunk_tokens = tokens[i : i + max_tokens]- chunk_text = encoding.decode(chunk_tokens)- batches.append([chunk_text])-- # If adding this text would exceed limit, start new batch- elif current_tokens + text_tokens > max_tokens:- if current_batch: # Don't add empty batches- batches.append(current_batch)- current_batch = [text]- current_tokens = text_tokens-- # Add to current batch- else:- current_batch.append(text)- current_tokens += text_tokens+ current_batch.append(text)+ current_tokens += text_tokens- # Add final batch if not emptyif current_batch:batches.append(current_batch)diff --git a/src/utils/document_processing.py b/src/utils/document_processing.pyindex e4b002d4a..b306ab99d 100644--- a/src/utils/document_processing.py+++ b/src/utils/document_processing.py@@ -291,14 +291,76 @@ def resplit_chunks_character_windows(return out+def _prefix_token_count(encoding, text: str, start: int, length: int) -> int:+ return len(encoding.encode(text[start : start + length]))+++def _split_text_to_max_tokens(text: str, encoding, max_tokens: int) -> list[str]:+ """Cut ``text`` into pieces that each encode to at most ``max_tokens``.++ Slices the original string, so the pieces join back to ``text``. A probe+ covers only the next doubled window, not the whole remaining suffix: a+ 500-token Watsonx cut can re-encode to 501, but finding that boundary+ must not encode the rest of the document on every guess. A single code+ point that is already over the cap is kept as its own piece so the cursor+ still advances.+ """+ if _prefix_token_count(encoding, text, 0, len(text)) <= max_tokens:+ return [text]++ pieces: list[str] = []+ start = 0+ length = len(text)+ while start < length:+ fit = 0+ window = 1+ over = False+ while start + window <= length:+ if _prefix_token_count(encoding, text, start, window) <= max_tokens:+ fit = window+ if start + window == length:+ break+ grown = min(length - start, window * 2)+ if grown == window:+ break+ window = grown+ else:+ over = True+ break+ if fit == 0:+ pieces.append(text[start : start + 1])+ start += 1+ continue+ if not over:+ pieces.append(text[start:])+ break+ low = fit+ high = window+ best = fit+ while low <= high:+ mid = (low + high) // 2+ if _prefix_token_count(encoding, text, start, mid) <= max_tokens:+ best = mid+ low = mid + 1+ else:+ high = mid - 1+ # `fit` was measured under the cap. A non-monotone boundary cannot+ # push the piece back over it.+ if _prefix_token_count(encoding, text, start, best) > max_tokens:+ best = fit+ pieces.append(text[start : start + best])+ start += best+ return pieces++def split_chunks_by_max_tokens(chunks: list[dict], max_tokens: int, model: str | None = None) -> list[dict]:"""Split any chunks whose token count exceeds max_tokens.- Ensures that when chunk texts are sent to the embedding model, they don't get- further split by token limits in the batching helper, allowing the final- embeddings and chunks to align 1-to-1.+ Each resulting chunk encodes to at most ``max_tokens`` (unless a single+ code point is already over the cap), so the embedding batcher can keep one+ vector per chunk."""import tiktoken@@ -317,17 +379,9 @@ def split_chunks_by_max_tokens(new_chunks.append(chunk)continue- tokens = encoding.encode(text)- if len(tokens) <= max_tokens:- new_chunks.append(chunk)- continue-- # Split tokens into segments of at most max_tokens size- for i in range(0, len(tokens), max_tokens):- chunk_tokens = tokens[i : i + max_tokens]- chunk_text = encoding.decode(chunk_tokens)+ for piece in _split_text_to_max_tokens(text, encoding, max_tokens):new_chunk = dict(chunk)- new_chunk["text"] = chunk_text+ new_chunk["text"] = piecenew_chunks.append(new_chunk)return new_chunksdiff --git a/tests/unit/services/test_embedding_calls_use_the_gateway.py b/tests/unit/services/test_embedding_calls_use_the_gateway.pyindex 55fee5df2..1355e7143 100644--- a/tests/unit/services/test_embedding_calls_use_the_gateway.py+++ b/tests/unit/services/test_embedding_calls_use_the_gateway.py@@ -233,7 +233,7 @@ async def test_ingestion_embeds_through_the_gateway_for_every_provider()monkeypatch.setattr("services.document_service.chunk_texts_for_embeddings",- lambda texts, max_tokens: [texts] if texts else [],+ lambda texts, max_tokens, model=None: [texts] if texts else [],)await _processor(model).process_document_standard(diff --git a/tests/unit/test_document_processing_resplit.py b/tests/unit/test_document_processing_resplit.pyindex 999e6b80f..85bb6bfe1 100644--- a/tests/unit/test_document_processing_resplit.py+++ b/tests/unit/test_document_processing_resplit.py@@ -127,6 +127,54 @@ def test_extract_relevant_safeguard_none_table_data():) # should handle None data and produce empty string or no cells+# 501 cl100k tokens. A 500-token slice of this string decodes to text that+# encodes back to 501, which is the Watsonx cap in the non-Langflow ingest path.+_WATSONX_CAP_OVERFLOW = (+ "地理的には、ユーラシア大陸北西の半島部を包括し、ウラル山脈及びコーカサス山脈の分水嶺とウラル川・"+ "カスピ海・黒海、そして黒海とエーゲ海を繋ぐボスポラス海峡-マルマラ海-ダーダネルス海峡が、アジアと区分される東の境界となる。\n"+ "面積から見るとヨーロッパ州は世界で2番目に小さな大州であり、1018万kmは地球表面積の2%、陸地に限れば6.8%を占める。"+ "アジアに跨る領土を持つロシアは、ヨーロッパ50か国の中で面積および人口第1位の国家である。"+ "対照的に最も小さな国家はバチカン市国である。総人口はアジア・アフリカに次ぐ7億3300万である。これは地球総人口の11%である。\n"+ "ヨーロッパ、特に古代ギリシアは西洋文明発祥の地である。これは、16世紀以降の植民地主義の始まりとともに世界中に拡散し、"+ "支配的な役割を果たした。16世紀から20世紀の間、ヨーロッパの国々はアメリカ州、アフリカ、オセアニア、中東、アジアの大部分を支配下に置いた。"+ "2度の世界大戦はヨーロッパを戦火で覆い、20世紀中頃の西ヨーロ"+)+++def test_split_chunks_by_max_tokens_watsonx_window_stays_under_cap():+ import tiktoken++ from src.services.document_service import chunk_texts_for_embeddings+ from src.utils.document_processing import split_chunks_by_max_tokens++ encoding = tiktoken.get_encoding("cl100k_base")+ assert len(encoding.encode(_WATSONX_CAP_OVERFLOW)) == 501++ out = split_chunks_by_max_tokens(+ [{"page": 3, "type": "text", "text": _WATSONX_CAP_OVERFLOW}],+ max_tokens=500,+ model="text-embedding-3-small",+ )+ pieces = [chunk["text"] for chunk in out]+ assert "".join(pieces) == _WATSONX_CAP_OVERFLOW+ assert all(len(encoding.encode(piece)) <= 500 for piece in pieces)+ assert all(chunk["page"] == 3 for chunk in out)++ batches = chunk_texts_for_embeddings(pieces, max_tokens=500, model="text-embedding-3-small")+ assert [text for batch in batches for text in batch] == pieces+++def test_chunk_texts_keeps_one_input_when_text_exceeds_cap():+ from src.services.document_service import chunk_texts_for_embeddings++ batches = chunk_texts_for_embeddings(+ [_WATSONX_CAP_OVERFLOW],+ max_tokens=500,+ model="text-embedding-3-small",+ )+ assert batches == [[_WATSONX_CAP_OVERFLOW]]++def test_split_chunks_by_max_tokens_emoji_token_override():from src.utils.document_processing import split_chunks_by_max_tokensdiff --git a/tests/unit/test_processors_clear_stale_chunks.py b/tests/unit/test_processors_clear_stale_chunks.pyindex 9ebde8fab..f8c878b5a 100644--- a/tests/unit/test_processors_clear_stale_chunks.py+++ b/tests/unit/test_processors_clear_stale_chunks.py@@ -84,7 +84,7 @@ def _patch_embedding_pipeline(monkeypatch, chunk_count: int, write_client=None):monkeypatch.setattr(ds_mod,"chunk_texts_for_embeddings",- lambda texts, max_tokens=8000: [list(texts)],+ lambda texts, max_tokens=8000, model=None: [list(texts)],)# Embeddings go through the gateway for every provider, so stub it at itsdiff --git a/tests/unit/test_processors_image_only.py b/tests/unit/test_processors_image_only.pyindex 47b50ccc9..bc31080da 100644--- a/tests/unit/test_processors_image_only.py+++ b/tests/unit/test_processors_image_only.py@@ -43,7 +43,7 @@ async def test_image_only_placeholder_succeeds_but_empty_document_fails()monkeypatch.setattr("services.document_service.chunk_texts_for_embeddings",- lambda texts, max_tokens: [texts] if texts else [],+ lambda texts, max_tokens, model=None: [texts] if texts else [],)user_client = SimpleNamespace()