Skip to content

OpenRAG is a comprehensive, single package Retrieval-Augmented Generation platform built on Langflow, Docling, and Opensearch.

Python

fix: ingestion failing for dense text file when langflow ingestion disabled with wxai models (#2591)

Wallgauebbe9218e457
diff --git a/src/models/processors.py b/src/models/processors.py
index 7d02a5447..e2ed4990f 100644
--- a/src/models/processors.py
+++ b/src/models/processors.py
@@ -740,7 +740,9 @@ class TaskProcessor:
token_cap.set(non_empty=non_empty, chunks_out=len(slim_doc["chunks"]))
texts = [c["text"] for c in slim_doc["chunks"]]
- text_batches = chunk_texts_for_embeddings(texts, max_tokens=max_tokens)
+ text_batches = chunk_texts_for_embeddings(
+ texts, max_tokens=max_tokens, model=litellm_embedding_model
+ )
embeddings = []
# Every provider goes through the gateway, not just the aliased ones.
# The gateway resolves credentials per call from config; the direct
diff --git a/src/services/document_service.py b/src/services/document_service.py
index 2766f43b4..87dd38527 100644
--- a/src/services/document_service.py
+++ b/src/services/document_service.py
@@ -22,9 +22,12 @@ def get_token_count(text: str, model: str = None) -> int:
def chunk_texts_for_embeddings(
texts: list[str], max_tokens: int = None, model: str = None
) -> list[list[str]]:
- """
- Split texts into batches that won't exceed token limits.
- If max_tokens is None, returns texts as single batch (no splitting).
+ """Pack texts into batches that stay within ``max_tokens``.
+
+ Each input stays one embedding input. A text already over the limit is
+ placed in its own batch; splitting it here would make the embedding list
+ longer than the chunk list. If ``max_tokens`` is None, texts are returned
+ as a single batch.
"""
model = model or get_embedding_model()
@@ -44,40 +47,19 @@ def chunk_texts_for_embeddings(
for text in texts:
text_tokens = get_token_count(text, model)
- # If single text exceeds limit, split it further
- if text_tokens > max_tokens:
- # If we have current batch, save it first
+ over_limit = text_tokens > max_tokens
+ if over_limit or (current_batch and current_tokens + text_tokens > max_tokens):
if current_batch:
batches.append(current_batch)
current_batch = []
current_tokens = 0
+ if over_limit:
+ batches.append([text])
+ continue
- # Split the large text into smaller chunks
- try:
- encoding = tiktoken.encoding_for_model(model)
- except KeyError:
- encoding = tiktoken.get_encoding("cl100k_base")
-
- tokens = encoding.encode(text)
-
- for i in range(0, len(tokens), max_tokens):
- chunk_tokens = tokens[i : i + max_tokens]
- chunk_text = encoding.decode(chunk_tokens)
- batches.append([chunk_text])
-
- # If adding this text would exceed limit, start new batch
- elif current_tokens + text_tokens > max_tokens:
- if current_batch: # Don't add empty batches
- batches.append(current_batch)
- current_batch = [text]
- current_tokens = text_tokens
-
- # Add to current batch
- else:
- current_batch.append(text)
- current_tokens += text_tokens
+ current_batch.append(text)
+ current_tokens += text_tokens
- # Add final batch if not empty
if current_batch:
batches.append(current_batch)
diff --git a/src/utils/document_processing.py b/src/utils/document_processing.py
index e4b002d4a..b306ab99d 100644
--- a/src/utils/document_processing.py
+++ b/src/utils/document_processing.py
@@ -291,14 +291,76 @@ def resplit_chunks_character_windows(
return out
+def _prefix_token_count(encoding, text: str, start: int, length: int) -> int:
+ return len(encoding.encode(text[start : start + length]))
+
+
+def _split_text_to_max_tokens(text: str, encoding, max_tokens: int) -> list[str]:
+ """Cut ``text`` into pieces that each encode to at most ``max_tokens``.
+
+ Slices the original string, so the pieces join back to ``text``. A probe
+ covers only the next doubled window, not the whole remaining suffix: a
+ 500-token Watsonx cut can re-encode to 501, but finding that boundary
+ must not encode the rest of the document on every guess. A single code
+ point that is already over the cap is kept as its own piece so the cursor
+ still advances.
+ """
+ if _prefix_token_count(encoding, text, 0, len(text)) <= max_tokens:
+ return [text]
+
+ pieces: list[str] = []
+ start = 0
+ length = len(text)
+ while start < length:
+ fit = 0
+ window = 1
+ over = False
+ while start + window <= length:
+ if _prefix_token_count(encoding, text, start, window) <= max_tokens:
+ fit = window
+ if start + window == length:
+ break
+ grown = min(length - start, window * 2)
+ if grown == window:
+ break
+ window = grown
+ else:
+ over = True
+ break
+ if fit == 0:
+ pieces.append(text[start : start + 1])
+ start += 1
+ continue
+ if not over:
+ pieces.append(text[start:])
+ break
+ low = fit
+ high = window
+ best = fit
+ while low <= high:
+ mid = (low + high) // 2
+ if _prefix_token_count(encoding, text, start, mid) <= max_tokens:
+ best = mid
+ low = mid + 1
+ else:
+ high = mid - 1
+ # `fit` was measured under the cap. A non-monotone boundary cannot
+ # push the piece back over it.
+ if _prefix_token_count(encoding, text, start, best) > max_tokens:
+ best = fit
+ pieces.append(text[start : start + best])
+ start += best
+ return pieces
+
+
def split_chunks_by_max_tokens(
chunks: list[dict], max_tokens: int, model: str | None = None
) -> list[dict]:
"""Split any chunks whose token count exceeds max_tokens.
- Ensures that when chunk texts are sent to the embedding model, they don't get
- further split by token limits in the batching helper, allowing the final
- embeddings and chunks to align 1-to-1.
+ Each resulting chunk encodes to at most ``max_tokens`` (unless a single
+ code point is already over the cap), so the embedding batcher can keep one
+ vector per chunk.
"""
import tiktoken
@@ -317,17 +379,9 @@ def split_chunks_by_max_tokens(
new_chunks.append(chunk)
continue
- tokens = encoding.encode(text)
- if len(tokens) <= max_tokens:
- new_chunks.append(chunk)
- continue
-
- # Split tokens into segments of at most max_tokens size
- for i in range(0, len(tokens), max_tokens):
- chunk_tokens = tokens[i : i + max_tokens]
- chunk_text = encoding.decode(chunk_tokens)
+ for piece in _split_text_to_max_tokens(text, encoding, max_tokens):
new_chunk = dict(chunk)
- new_chunk["text"] = chunk_text
+ new_chunk["text"] = piece
new_chunks.append(new_chunk)
return new_chunks
diff --git a/tests/unit/services/test_embedding_calls_use_the_gateway.py b/tests/unit/services/test_embedding_calls_use_the_gateway.py
index 55fee5df2..1355e7143 100644
--- a/tests/unit/services/test_embedding_calls_use_the_gateway.py
+++ b/tests/unit/services/test_embedding_calls_use_the_gateway.py
@@ -233,7 +233,7 @@ async def test_ingestion_embeds_through_the_gateway_for_every_provider(
)
monkeypatch.setattr(
"services.document_service.chunk_texts_for_embeddings",
- lambda texts, max_tokens: [texts] if texts else [],
+ lambda texts, max_tokens, model=None: [texts] if texts else [],
)
await _processor(model).process_document_standard(
diff --git a/tests/unit/test_document_processing_resplit.py b/tests/unit/test_document_processing_resplit.py
index 999e6b80f..85bb6bfe1 100644
--- a/tests/unit/test_document_processing_resplit.py
+++ b/tests/unit/test_document_processing_resplit.py
@@ -127,6 +127,54 @@ def test_extract_relevant_safeguard_none_table_data():
) # should handle None data and produce empty string or no cells
+# 501 cl100k tokens. A 500-token slice of this string decodes to text that
+# encodes back to 501, which is the Watsonx cap in the non-Langflow ingest path.
+_WATSONX_CAP_OVERFLOW = (
+ "地理的には、ユーラシア大陸北西の半島部を包括し、ウラル山脈及びコーカサス山脈の分水嶺とウラル川・"
+ "カスピ海・黒海、そして黒海とエーゲ海を繋ぐボスポラス海峡-マルマラ海-ダーダネルス海峡が、アジアと区分される東の境界となる。\n"
+ "面積から見るとヨーロッパ州は世界で2番目に小さな大州であり、1018万kmは地球表面積の2%、陸地に限れば6.8%を占める。"
+ "アジアに跨る領土を持つロシアは、ヨーロッパ50か国の中で面積および人口第1位の国家である。"
+ "対照的に最も小さな国家はバチカン市国である。総人口はアジア・アフリカに次ぐ7億3300万である。これは地球総人口の11%である。\n"
+ "ヨーロッパ、特に古代ギリシアは西洋文明発祥の地である。これは、16世紀以降の植民地主義の始まりとともに世界中に拡散し、"
+ "支配的な役割を果たした。16世紀から20世紀の間、ヨーロッパの国々はアメリカ州、アフリカ、オセアニア、中東、アジアの大部分を支配下に置いた。"
+ "2度の世界大戦はヨーロッパを戦火で覆い、20世紀中頃の西ヨーロ"
+)
+
+
+def test_split_chunks_by_max_tokens_watsonx_window_stays_under_cap():
+ import tiktoken
+
+ from src.services.document_service import chunk_texts_for_embeddings
+ from src.utils.document_processing import split_chunks_by_max_tokens
+
+ encoding = tiktoken.get_encoding("cl100k_base")
+ assert len(encoding.encode(_WATSONX_CAP_OVERFLOW)) == 501
+
+ out = split_chunks_by_max_tokens(
+ [{"page": 3, "type": "text", "text": _WATSONX_CAP_OVERFLOW}],
+ max_tokens=500,
+ model="text-embedding-3-small",
+ )
+ pieces = [chunk["text"] for chunk in out]
+ assert "".join(pieces) == _WATSONX_CAP_OVERFLOW
+ assert all(len(encoding.encode(piece)) <= 500 for piece in pieces)
+ assert all(chunk["page"] == 3 for chunk in out)
+
+ batches = chunk_texts_for_embeddings(pieces, max_tokens=500, model="text-embedding-3-small")
+ assert [text for batch in batches for text in batch] == pieces
+
+
+def test_chunk_texts_keeps_one_input_when_text_exceeds_cap():
+ from src.services.document_service import chunk_texts_for_embeddings
+
+ batches = chunk_texts_for_embeddings(
+ [_WATSONX_CAP_OVERFLOW],
+ max_tokens=500,
+ model="text-embedding-3-small",
+ )
+ assert batches == [[_WATSONX_CAP_OVERFLOW]]
+
+
def test_split_chunks_by_max_tokens_emoji_token_override():
from src.utils.document_processing import split_chunks_by_max_tokens
diff --git a/tests/unit/test_processors_clear_stale_chunks.py b/tests/unit/test_processors_clear_stale_chunks.py
index 9ebde8fab..f8c878b5a 100644
--- a/tests/unit/test_processors_clear_stale_chunks.py
+++ b/tests/unit/test_processors_clear_stale_chunks.py
@@ -84,7 +84,7 @@ def _patch_embedding_pipeline(monkeypatch, chunk_count: int, write_client=None):
monkeypatch.setattr(
ds_mod,
"chunk_texts_for_embeddings",
- lambda texts, max_tokens=8000: [list(texts)],
+ lambda texts, max_tokens=8000, model=None: [list(texts)],
)
# Embeddings go through the gateway for every provider, so stub it at its
diff --git a/tests/unit/test_processors_image_only.py b/tests/unit/test_processors_image_only.py
index 47b50ccc9..bc31080da 100644
--- a/tests/unit/test_processors_image_only.py
+++ b/tests/unit/test_processors_image_only.py
@@ -43,7 +43,7 @@ async def test_image_only_placeholder_succeeds_but_empty_document_fails(
)
monkeypatch.setattr(
"services.document_service.chunk_texts_for_embeddings",
- lambda texts, max_tokens: [texts] if texts else [],
+ lambda texts, max_tokens, model=None: [texts] if texts else [],
)
user_client = SimpleNamespace()