OpenRAG is a comprehensive, single package Retrieval-Augmented Generation platform built on Langflow, Docling, and Opensearch.
Python
fix: fix ingestion without page number (#1857)
Lucas Oliveira5468a9f85314
diff --git a/src/utils/document_processing.py b/src/utils/document_processing.pyindex 0ca02bfa4..6fff1c129 100644--- a/src/utils/document_processing.py+++ b/src/utils/document_processing.py@@ -1,6 +1,6 @@-import hashlibimport osfrom collections import defaultdict+from utils.logging_config import get_loggerlogger = get_logger(__name__)@@ -17,11 +17,10 @@ def process_text_file(file_path: str) -> dict:Returns:dict with keys: id, filename, mimetype, chunks"""- import osfrom utils.hash_utils import hash_id# Read the file- with open(file_path, 'r', encoding='utf-8', errors='replace') as f:+ with open(file_path, encoding="utf-8", errors="replace") as f:content = f.read()# Compute hash@@ -34,7 +33,7 @@ def process_text_file(file_path: str) -> dict:chunks = []# Split by paragraphs first (double newline)- paragraphs = content.split('\n\n')+ paragraphs = content.split("\n\n")current_chunk = ""chunk_index = 0@@ -45,11 +44,13 @@ def process_text_file(file_path: str) -> dict:# If adding this paragraph would exceed chunk size, save current chunkif len(current_chunk) + len(para) + 2 > chunk_size and current_chunk:- chunks.append({- "page": chunk_index + 1, # Use chunk_index + 1 as "page" number- "type": "text",- "text": current_chunk.strip()- })+ chunks.append(+ {+ "page": chunk_index + 1, # Use chunk_index + 1 as "page" number+ "type": "text",+ "text": current_chunk.strip(),+ }+ )chunk_index += 1current_chunk = paraelse:@@ -60,19 +61,11 @@ def process_text_file(file_path: str) -> dict:# Add the last chunk if anyif current_chunk.strip():- chunks.append({- "page": chunk_index + 1,- "type": "text",- "text": current_chunk.strip()- })+ chunks.append({"page": chunk_index + 1, "type": "text", "text": current_chunk.strip()})# If no chunks were created (empty file), create a single empty chunkif not chunks:- chunks.append({- "page": 1,- "type": "text",- "text": ""- })+ chunks.append({"page": 1, "type": "text", "text": ""})return {"id": file_hash,@@ -99,18 +92,19 @@ def extract_relevant(doc_dict: dict) -> dict:for txt in doc_dict.get("texts", []):prov = txt.get("prov", [])page_no = prov[0].get("page_no") if prov else None- if page_no is not None:- page_texts[page_no].append(txt.get("text", "").strip())+ if page_no is None:+ page_no = 1+ page_texts[page_no].append(txt.get("text", "").strip())for page in sorted(page_texts):- chunks.append(- {"page": page, "type": "text", "text": "\n".join(page_texts[page])}- )+ chunks.append({"page": page, "type": "text", "text": "\n".join(page_texts[page])})# 2) process tablesfor t_idx, table in enumerate(doc_dict.get("tables", [])):prov = table.get("prov", [])page_no = prov[0].get("page_no") if prov else None+ if page_no is None:+ page_no = 1# group cells by their row indexrows = defaultdict(list)diff --git a/tests/unit/test_document_processing_resplit.py b/tests/unit/test_document_processing_resplit.pyindex bfb6b3254..8b097b5ee 100644--- a/tests/unit/test_document_processing_resplit.py+++ b/tests/unit/test_document_processing_resplit.py@@ -35,3 +35,32 @@ def test_long_text_windows_with_overlap():assert out[0]["text"] == "abcdefghij"assert out[1]["text"] == "ghijklmnop"assert out[2]["text"] == "mnopqrstuv"+++def test_extract_relevant_with_missing_page_no():+ from src.utils.document_processing import extract_relevant++ # Docling JSON representation where page_no is missing in prov (e.g. for asciidoc/html)+ doc_dict = {+ "origin": {"binary_hash": "hash123", "filename": "test.adoc", "mimetype": "text/asciidoc"},+ "texts": [+ {+ "text": "Hello Asciidoc World!",+ "prov": [], # empty prov+ },+ {+ "text": "Second paragraph of text.",+ "prov": [{"some_other_key": "val"}], # prov present but no page_no+ },+ ],+ "tables": [],+ }++ result = extract_relevant(doc_dict)+ assert result["id"] == "hash123"+ assert result["filename"] == "test.adoc"+ assert result["mimetype"] == "text/asciidoc"+ assert len(result["chunks"]) == 1+ assert result["chunks"][0]["page"] == 1+ assert "Hello Asciidoc World!" in result["chunks"][0]["text"]+ assert "Second paragraph of text." in result["chunks"][0]["text"]