OpenRAG is a comprehensive, single package Retrieval-Augmented Generation platform built on Langflow, Docling, and Opensearch.
Python
"""Tests for resplit_chunks_character_windows."""
from src.utils.document_processing import IMAGE_PLACEHOLDER, resplit_chunks_character_windows
def test_invalid_returns_original():
chunks = [{"page": 1, "text": "hello world" * 50}]
assert resplit_chunks_character_windows(chunks, 0, 0) is chunks
assert resplit_chunks_character_windows(chunks, 10, 10) is chunks
assert resplit_chunks_character_windows(chunks, 5, 6) is chunks
def test_non_dict_chunk_passthrough():
chunks = ["not-a-dict", {"page": 1, "text": "x" * 30}]
out = resplit_chunks_character_windows(chunks, 10, 2)
assert out[0] == "not-a-dict"
assert len(out) == 1 + 4 # one long dict chunk -> 4 windows
def test_short_text_unchanged_count():
chunks = [{"page": 2, "type": "text", "text": "short"}]
out = resplit_chunks_character_windows(chunks, 100, 20)
assert len(out) == 1
assert out[0]["text"] == "short"
assert out[0]["page"] == 2
def test_image_placeholder_is_not_split():
chunks = [{"page": 1, "type": "picture", "text": IMAGE_PLACEHOLDER}]
out = resplit_chunks_character_windows(chunks, 1, 0)
assert out == chunks
def test_long_text_windows_with_overlap():
# 22 chars, size 10, overlap 4 -> stride 6; third window reaches end
text = "abcdefghijklmnopqrstuv"
assert len(text) == 22
chunks = [{"page": 1, "text": text}]
out = resplit_chunks_character_windows(chunks, 10, 4)
assert len(out) == 3
assert out[0]["text"] == "abcdefghij"
assert out[1]["text"] == "ghijklmnop"
assert out[2]["text"] == "mnopqrstuv"
def test_extract_relevant_with_missing_page_no():
from src.utils.document_processing import extract_relevant
# Docling JSON representation where page_no is missing in prov (e.g. for asciidoc/html)
doc_dict = {
"origin": {"binary_hash": "hash123", "filename": "test.adoc", "mimetype": "text/asciidoc"},
"texts": [
{
"text": "Hello Asciidoc World!",
"prov": [], # empty prov
},
{
"text": "Second paragraph of text.",
"prov": [{"some_other_key": "val"}], # prov present but no page_no
},
],
"tables": [],
}
result = extract_relevant(doc_dict)
assert result["id"] == "hash123"
assert result["filename"] == "test.adoc"
assert result["mimetype"] == "text/asciidoc"
assert len(result["chunks"]) == 1
assert result["chunks"][0]["page"] == 1
assert "Hello Asciidoc World!" in result["chunks"][0]["text"]
assert "Second paragraph of text." in result["chunks"][0]["text"]
def test_split_chunks_by_max_tokens_under_limit():
from src.utils.document_processing import split_chunks_by_max_tokens
chunks = [
{"page": 1, "type": "text", "text": "hello world"},
{"page": 2, "type": "table", "text": "some other text"},
]
out = split_chunks_by_max_tokens(chunks, max_tokens=100)
assert len(out) == 2
assert out[0]["text"] == "hello world"
assert out[1]["text"] == "some other text"
def test_split_chunks_by_max_tokens_exceeding():
from src.utils.document_processing import split_chunks_by_max_tokens
# "hello world" is 2 tokens in cl100k_base encoding. If we limit to 1 token per chunk:
chunks = [{"page": 5, "type": "table", "table_index": 0, "text": "hello world"}]
# We use cl100k_base (standard for gpt-4/text-embedding-3)
out = split_chunks_by_max_tokens(chunks, max_tokens=1, model="text-embedding-3-small")
assert len(out) == 2
assert out[0]["page"] == 5
assert out[0]["type"] == "table"
assert out[0]["table_index"] == 0
assert out[0]["text"] == "hello"
assert out[1]["page"] == 5
assert out[1]["type"] == "table"
assert out[1]["table_index"] == 0
assert out[1]["text"] == " world"
def test_extract_relevant_safeguard_none_table_data():
from src.utils.document_processing import extract_relevant
doc_dict = {
"origin": {"binary_hash": "hash456", "filename": "test.csv", "mimetype": "text/csv"},
"texts": [],
"tables": [
{
"prov": [{"page_no": 1}],
"data": None, # Simulate missing table data / structure
}
],
}
result = extract_relevant(doc_dict)
assert result["id"] == "hash456"
assert len(result["chunks"]) == 1
assert result["chunks"][0]["type"] == "table"
assert (
result["chunks"][0]["text"] == ""
) # should handle None data and produce empty string or no cells
# 501 cl100k tokens. A 500-token slice of this string decodes to text that
# encodes back to 501, which is the Watsonx cap in the non-Langflow ingest path.
_WATSONX_CAP_OVERFLOW = (
"地理的には、ユーラシア大陸北西の半島部を包括し、ウラル山脈及びコーカサス山脈の分水嶺とウラル川・"
"カスピ海・黒海、そして黒海とエーゲ海を繋ぐボスポラス海峡-マルマラ海-ダーダネルス海峡が、アジアと区分される東の境界となる。\n"
"面積から見るとヨーロッパ州は世界で2番目に小さな大州であり、1018万kmは地球表面積の2%、陸地に限れば6.8%を占める。"
"アジアに跨る領土を持つロシアは、ヨーロッパ50か国の中で面積および人口第1位の国家である。"
"対照的に最も小さな国家はバチカン市国である。総人口はアジア・アフリカに次ぐ7億3300万である。これは地球総人口の11%である。\n"
"ヨーロッパ、特に古代ギリシアは西洋文明発祥の地である。これは、16世紀以降の植民地主義の始まりとともに世界中に拡散し、"
"支配的な役割を果たした。16世紀から20世紀の間、ヨーロッパの国々はアメリカ州、アフリカ、オセアニア、中東、アジアの大部分を支配下に置いた。"
"2度の世界大戦はヨーロッパを戦火で覆い、20世紀中頃の西ヨーロ"
)
def test_split_chunks_by_max_tokens_watsonx_window_stays_under_cap():
import tiktoken
from src.services.document_service import chunk_texts_for_embeddings
from src.utils.document_processing import split_chunks_by_max_tokens
encoding = tiktoken.get_encoding("cl100k_base")
assert len(encoding.encode(_WATSONX_CAP_OVERFLOW)) == 501
out = split_chunks_by_max_tokens(
[{"page": 3, "type": "text", "text": _WATSONX_CAP_OVERFLOW}],
max_tokens=500,
model="text-embedding-3-small",
)
pieces = [chunk["text"] for chunk in out]
assert "".join(pieces) == _WATSONX_CAP_OVERFLOW
assert all(len(encoding.encode(piece)) <= 500 for piece in pieces)
assert all(chunk["page"] == 3 for chunk in out)
batches = chunk_texts_for_embeddings(pieces, max_tokens=500, model="text-embedding-3-small")
assert [text for batch in batches for text in batch] == pieces
def test_chunk_texts_keeps_one_input_when_text_exceeds_cap():
from src.services.document_service import chunk_texts_for_embeddings
batches = chunk_texts_for_embeddings(
[_WATSONX_CAP_OVERFLOW],
max_tokens=500,
model="text-embedding-3-small",
)
assert batches == [[_WATSONX_CAP_OVERFLOW]]
def test_split_chunks_by_max_tokens_emoji_token_override():
from src.utils.document_processing import split_chunks_by_max_tokens
# "👨👩👧👦" has len() = 7 in Python, but is 18 tokens in cl100k_base.
# If max_tokens is 10, then character count (7) < max_tokens (10),
# but token count (18) > max_tokens (10). It must be split.
chunks = [{"page": 1, "type": "text", "text": "👨👩👧👦"}]
out = split_chunks_by_max_tokens(chunks, max_tokens=10, model="text-embedding-3-small")
# Must be split because 18 > 10.
assert len(out) > 1
# Check that chunks reconstruct to the original text
reconstructed = "".join([c["text"] for c in out])
assert reconstructed == "👨👩👧👦"