OpenRAG is a comprehensive, single package Retrieval-Augmented Generation platform built on Langflow, Docling, and Opensearch.
Python
import os
import re
import unicodedata
from collections import defaultdict
from utils.logging_config import get_logger
logger = get_logger(__name__)
IMAGE_PLACEHOLDER = "<!-- image -->"
# Some PDF fonts map common kanji (日, 月, 木, 金, 土, ...) to the look-alike
# Kangxi Radical code points in their /ToUnicode CMap, and Docling copies them
# through unchanged. Map just those blocks back to CJK Unified Ideographs;
# full NFKC would also rewrite full-width digits, ㈱, ①, half-width katakana.
_CJK_RADICAL_TABLE = {
cp: unicodedata.normalize("NFKC", chr(cp))
for cp in range(0x2E80, 0x2FE0) # CJK Radicals Supplement + Kangxi Radicals
if unicodedata.normalize("NFKC", chr(cp)) != chr(cp)
}
def normalize_cjk_radicals(text: str) -> str:
"""Replace Kangxi/CJK radical code points with their unified ideographs."""
return text.translate(_CJK_RADICAL_TABLE)
_SENTENCE_END = re.compile(r"(?<=[。.!?.!?])")
def split_to_size(
text: str, limit: int, levels: tuple[str, ...] = ("line", "sentence")
) -> list[str]:
"""Split text into pieces of at most ``limit`` characters.
Cuts on line breaks first, then after sentence-ending punctuation, and only
then at a fixed character count, so pieces end at natural boundaries when
the text has any.
"""
if len(text) <= limit:
return [text]
if not levels:
return [text[i : i + limit] for i in range(0, len(text), limit)]
if levels[0] == "line":
parts, joiner = text.split("\n"), "\n"
else:
parts, joiner = _SENTENCE_END.split(text), ""
pieces: list[str] = []
current = ""
for part in parts:
for sub in split_to_size(part, limit, levels[1:]):
candidate = f"{current}{joiner}{sub}" if current else sub
if len(candidate) <= limit:
current = candidate
else:
if current:
pieces.append(current)
current = sub
if current:
pieces.append(current)
return pieces
def process_text_file(file_path: str) -> dict:
"""
Process a plain text file without using docling.
Returns the same structure as extract_relevant() for consistency.
Args:
file_path: Path to the .txt file
Returns:
dict with keys: id, filename, mimetype, chunks
"""
from utils.hash_utils import hash_id
# Read the file
with open(file_path, encoding="utf-8", errors="replace") as f:
content = f.read()
# Compute hash
file_hash = hash_id(file_path)
filename = os.path.basename(file_path)
# Split content into chunks of ~1000 characters to match typical docling chunk sizes
# This ensures embeddings stay within reasonable token limits
chunk_size = 1000
chunks = []
# Split by paragraphs first (double newline)
paragraphs = content.split("\n\n")
current_chunk = ""
chunk_index = 0
# A file with no blank lines is one paragraph, so oversized paragraphs are
# cut on lines and sentences before packing.
pieces = (
piece.strip() for para in paragraphs for piece in split_to_size(para.strip(), chunk_size)
)
for para in pieces:
if not para:
continue
# If adding this paragraph would exceed chunk size, save current chunk
if len(current_chunk) + len(para) + 2 > chunk_size and current_chunk:
chunks.append(
{
"page": chunk_index + 1, # Use chunk_index + 1 as "page" number
"type": "text",
"text": current_chunk.strip(),
}
)
chunk_index += 1
current_chunk = para
else:
if current_chunk:
current_chunk += "\n\n" + para
else:
current_chunk = para
# Add the last chunk if any
if current_chunk.strip():
chunks.append({"page": chunk_index + 1, "type": "text", "text": current_chunk.strip()})
# If no chunks were created (empty file), create a single empty chunk
if not chunks:
chunks.append({"page": 1, "type": "text", "text": ""})
return {
"id": file_hash,
"filename": filename,
"mimetype": "text/plain",
"chunks": chunks,
}
def extract_relevant(doc_dict: dict) -> dict:
"""
Given the full export_to_dict() result:
- Grabs origin metadata (hash, filename, mimetype)
- Finds every text fragment in `texts`, groups them by page_no
- Flattens tables in `tables` into tab-separated text, grouping by row
- Concatenates each page's fragments and each table into its own chunk
- Emits a picture chunk for each picture with description annotations
- Emits one image placeholder only when pictures exist and no other chunks were produced
Returns a slimmed dict ready for indexing, with each chunk under "text".
"""
origin = doc_dict.get("origin", {})
chunks = []
# 1) process free-text fragments
page_texts = defaultdict(list)
for txt in doc_dict.get("texts", []):
prov = txt.get("prov", [])
page_no = prov[0].get("page_no") if prov else None
if page_no is None:
page_no = 1
page_texts[page_no].append(txt.get("text", "").strip())
for page in sorted(page_texts):
chunks.append({"page": page, "type": "text", "text": "\n".join(page_texts[page])})
# 2) process tables
for t_idx, table in enumerate(doc_dict.get("tables", [])):
prov = table.get("prov", [])
page_no = prov[0].get("page_no") if prov else None
if page_no is None:
page_no = 1
# group cells by their row index
rows = defaultdict(list)
table_data = table.get("data")
if table_data:
for cell in table_data.get("table_cells", []):
r = cell.get("start_row_offset_idx")
c = cell.get("start_col_offset_idx")
text = cell.get("text", "").strip()
rows[r].append((c, text))
# build a tab‑separated line for each row, in order
flat_rows = []
for r in sorted(rows):
cells = [txt for _, txt in sorted(rows[r], key=lambda x: x[0])]
flat_rows.append("\t".join(cells))
chunks.append(
{
"page": page_no,
"type": "table",
"table_index": t_idx,
"text": "\n".join(flat_rows),
}
)
# 3) process picture descriptions (VLM-generated annotations)
# Docling stores these under pictures[].annotations with kind == "description".
# Without this, image-only documents (no text layer) yield zero chunks on the
# non-Langflow path, which surfaces as a misleading "corrupted or invalid" error.
# The Langflow path already captures them via Docling's export_to_markdown.
pictures = doc_dict.get("pictures", [])
for p_idx, picture in enumerate(pictures):
prov = picture.get("prov", [])
page_no = prov[0].get("page_no") if prov else None
if page_no is None:
page_no = 1
descriptions = [
ann.get("text", "").strip()
for ann in picture.get("annotations", [])
if ann.get("kind") == "description" and ann.get("text", "").strip()
]
if descriptions:
chunks.append(
{
"page": page_no,
"type": "picture",
"picture_index": p_idx,
"text": "\n".join(descriptions),
}
)
# Preserve a non-empty signal for image-only documents without descriptions,
# but avoid embedding one identical placeholder for every uncaptioned picture.
if not chunks and pictures:
first_picture = pictures[0]
prov = first_picture.get("prov", [])
page_no = prov[0].get("page_no") if prov else None
chunks.append(
{
"page": page_no if page_no is not None else 1,
"type": "picture",
"picture_index": 0,
"text": IMAGE_PLACEHOLDER,
}
)
for chunk in chunks:
chunk["text"] = normalize_cjk_radicals(chunk["text"])
return {
"id": origin.get("binary_hash"),
"filename": origin.get("filename"),
"mimetype": origin.get("mimetype"),
"chunks": chunks,
}
def resplit_chunks_character_windows(
chunks: list,
chunk_size: int,
chunk_overlap: int,
) -> list:
"""Split long chunk texts into fixed-size character windows with overlap.
Used by the non-Langflow ingestion path when the UI supplies ``chunkSize`` /
``chunkOverlap``. Invalid combinations (non-positive size or overlap >= size)
return ``chunks`` unchanged.
"""
if chunk_size <= 0 or chunk_overlap >= chunk_size:
return chunks
stride = chunk_size - chunk_overlap
if stride <= 0:
return chunks
out: list = []
for ch in chunks:
text = ch.get("text") if isinstance(ch, dict) else None
if not isinstance(text, str):
out.append(dict(ch) if isinstance(ch, dict) else ch)
continue
# This marker is a semantic signal that Docling detected an image even
# though it extracted no text. Keep it intact so image-only documents
# remain ingestible and display the same placeholder as the Langflow path.
if text.strip() == IMAGE_PLACEHOLDER:
out.append(dict(ch))
continue
if len(text) <= chunk_size:
out.append(dict(ch))
continue
start = 0
while start < len(text):
end = min(start + chunk_size, len(text))
piece = text[start:end]
new_ch = dict(ch)
new_ch["text"] = piece
out.append(new_ch)
if end >= len(text):
break
start += stride
return out
def _prefix_token_count(encoding, text: str, start: int, length: int) -> int:
return len(encoding.encode(text[start : start + length]))
def _split_text_to_max_tokens(text: str, encoding, max_tokens: int) -> list[str]:
"""Cut ``text`` into pieces that each encode to at most ``max_tokens``.
Slices the original string, so the pieces join back to ``text``. A probe
covers only the next doubled window, not the whole remaining suffix: a
500-token Watsonx cut can re-encode to 501, but finding that boundary
must not encode the rest of the document on every guess. A single code
point that is already over the cap is kept as its own piece so the cursor
still advances.
"""
if _prefix_token_count(encoding, text, 0, len(text)) <= max_tokens:
return [text]
pieces: list[str] = []
start = 0
length = len(text)
while start < length:
fit = 0
window = 1
over = False
while start + window <= length:
if _prefix_token_count(encoding, text, start, window) <= max_tokens:
fit = window
if start + window == length:
break
grown = min(length - start, window * 2)
if grown == window:
break
window = grown
else:
over = True
break
if fit == 0:
pieces.append(text[start : start + 1])
start += 1
continue
if not over:
pieces.append(text[start:])
break
low = fit
high = window
best = fit
while low <= high:
mid = (low + high) // 2
if _prefix_token_count(encoding, text, start, mid) <= max_tokens:
best = mid
low = mid + 1
else:
high = mid - 1
# `fit` was measured under the cap. A non-monotone boundary cannot
# push the piece back over it.
if _prefix_token_count(encoding, text, start, best) > max_tokens:
best = fit
pieces.append(text[start : start + best])
start += best
return pieces
def split_chunks_by_max_tokens(
chunks: list[dict], max_tokens: int, model: str | None = None
) -> list[dict]:
"""Split any chunks whose token count exceeds max_tokens.
Each resulting chunk encodes to at most ``max_tokens`` (unless a single
code point is already over the cap), so the embedding batcher can keep one
vector per chunk.
"""
import tiktoken
from config.settings import get_embedding_model
model = model or get_embedding_model()
try:
encoding = tiktoken.encoding_for_model(model)
except KeyError:
encoding = tiktoken.get_encoding("cl100k_base")
new_chunks = []
for chunk in chunks:
text = chunk.get("text", "")
if not text:
new_chunks.append(chunk)
continue
for piece in _split_text_to_max_tokens(text, encoding, max_tokens):
new_chunk = dict(chunk)
new_chunk["text"] = piece
new_chunks.append(new_chunk)
return new_chunks