Skip to content

RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.

fix(parser): handle large sparse spreadsheets, JSON BOM, and XLSX QA chunking (#19237)

Sourav Rajviec0ff067c198
diff --git a/deepdoc/parser/excel_parser.py b/deepdoc/parser/excel_parser.py
index 5a9c908f40..bfa688b26f 100644
--- a/deepdoc/parser/excel_parser.py
+++ b/deepdoc/parser/excel_parser.py
@@ -172,37 +172,23 @@ class RAGFlowExcelParser:
max_col = min(ws.max_column or 1, 50)
- def row_has_data(row_idx):
- for col_idx in range(1, max_col + 1):
- cell = ws.cell(row=row_idx, column=col_idx)
- if cell.value is not None and str(cell.value).strip():
- return True
- return False
-
- if not any(row_has_data(i) for i in range(1, min(101, max_row + 1))):
- return 0
-
- left, right = 1, max_row
- last_data_row = 1
-
- while left <= right:
- mid = (left + right) // 2
- found = False
- for r in range(mid, min(mid + 10, max_row + 1)):
- if row_has_data(r):
- found = True
- last_data_row = max(last_data_row, r)
- break
- if found:
- left = mid + 1
- else:
- right = mid - 1
-
- for r in range(last_data_row, min(last_data_row + 500, max_row + 1)):
- if row_has_data(r):
- last_data_row = r
-
- return last_data_row
+ # max_row is often inflated by styling far below real data. Scan only
+ # materialized cells so we do not call ws.cell() on every empty row.
+ highest = 0
+ for (row_idx, col_idx), cell in ws._cells.items():
+ if col_idx > max_col:
+ continue
+ if cell.value is not None and str(cell.value).strip():
+ highest = max(highest, row_idx)
+
+ if highest:
+ logging.debug(
+ "Excel row scan: max_row=%s max_col=%s detected_highest_data_row=%s",
+ max_row,
+ max_col,
+ highest,
+ )
+ return highest
@staticmethod
def _get_rows_limited(ws):
diff --git a/internal/ingestion/component/chunker/qa_test.go b/internal/ingestion/component/chunker/qa_test.go
index 61bc0b1195..a3af4c2cb6 100644
--- a/internal/ingestion/component/chunker/qa_test.go
+++ b/internal/ingestion/component/chunker/qa_test.go
@@ -357,3 +357,40 @@ func TestQAChunker_MarkdownRendersHTML(t *testing.T) {
t.Fatalf("markdown not rendered to HTML: %q", cww)
}
}
+
+func TestQAChunker_XLSXJSONRegression(t *testing.T) {
+ comp, err := NewQAChunker(map[string]any{"lang": "english"})
+ if err != nil {
+ t.Fatal(err)
+ }
+ inputs := map[string]any{
+ "name": "qa.xlsx",
+ "output_format": "json",
+ "json": []map[string]any{
+ {
+ "text": "<table><caption>Sheet1</caption><tr><th>question</th><th>answer</th></tr><tr><td>What is RAGFlow?</td><td>A RAG engine.</td></tr><tr><td>Where are the docs?</td><td>On the website.</td></tr></table>",
+ "doc_type_kwd": "table",
+ },
+ },
+ }
+ out, err := comp.Invoke(t.Context(), nil, inputs)
+ if err != nil {
+ t.Fatalf("Invoke failed: %v", err)
+ }
+ chunks, _ := out["chunks"].([]map[string]any)
+ if len(chunks) != 3 {
+ t.Fatalf("expected 3 chunks, got %d", len(chunks))
+ }
+ expected := []string{
+ "Question: question\tAnswer: answer",
+ // rmQAPrefix strips a leading "A " answer marker from "A RAG engine."
+ "Question: What is RAGFlow?\tAnswer: RAG engine.",
+ "Question: Where are the docs?\tAnswer: On the website.",
+ }
+ for i, want := range expected {
+ cww, _ := chunks[i]["text"].(string)
+ if cww != want {
+ t.Fatalf("chunk[%d] text = %q, want %q", i, cww, want)
+ }
+ }
+}
diff --git a/test/unit_test/deepdoc/parser/test_excel_parser.py b/test/unit_test/deepdoc/parser/test_excel_parser.py
index c3768ec4f4..71880e6413 100644
--- a/test/unit_test/deepdoc/parser/test_excel_parser.py
+++ b/test/unit_test/deepdoc/parser/test_excel_parser.py
@@ -224,3 +224,68 @@ def test_call_emits_zero_based_sheet_index():
assert sheets == {0, 1}
second = next(pos for _, pos in lines if pos[0] == 1)
assert second == (1, 2, 2, 1, 1)
+
+
+def _make_large_used_range_xlsx(first_data_row, last_data_row, inflate_row=12000):
+ from openpyxl import Workbook
+ from openpyxl.styles import Font
+
+ wb = Workbook()
+ ws = wb.active
+ ws.cell(row=first_data_row, column=1, value="question")
+ ws.cell(row=first_data_row, column=2, value="answer")
+ for r in range(first_data_row + 1, last_data_row + 1):
+ ws.cell(row=r, column=1, value=f"Q{r}")
+ ws.cell(row=r, column=2, value=f"A{r}")
+ ws.cell(row=inflate_row, column=1).font = Font(bold=True)
+ buf = BytesIO()
+ wb.save(buf)
+ buf.seek(0)
+ return buf.read()
+
+
+def _make_gap_xlsx(gap_rows, inflate_row=12000):
+ from openpyxl import Workbook
+
+ wb = Workbook()
+ ws = wb.active
+ ws["A1"] = "Name"
+ ws["B1"] = "City"
+ for r in range(2, 202):
+ ws.cell(row=r, column=1, value=f"n{r}")
+ ws.cell(row=r, column=2, value=f"c{r}")
+ second_block_start = 202 + gap_rows
+ for r in range(second_block_start, second_block_start + 200):
+ ws.cell(row=r, column=1, value=f"n{r}")
+ ws.cell(row=r, column=2, value=f"c{r}")
+ ws.cell(row=inflate_row, column=3, value=" ")
+ buf = BytesIO()
+ wb.save(buf)
+ buf.seek(0)
+ return buf.read()
+
+
+def test_large_used_range_with_blank_preamble_parses_all_rows():
+ # Regression for #19236: data below row 100 must not be treated as empty.
+ xlsx = _make_large_used_range_xlsx(150, 350)
+ rows = RAGFlowExcelParser()(xlsx)
+ assert len(rows) == 201
+ assert "Q350" in rows[-1][0]
+
+
+def test_large_used_range_with_blank_gap_parses_both_blocks():
+ # Regression for #19185: a blank gap must not truncate rows after it.
+ xlsx = _make_gap_xlsx(gap_rows=600)
+ rows = RAGFlowExcelParser()(xlsx)
+ assert len(rows) == 400
+ assert "n201" in rows[199][0]
+ assert "n802" in rows[200][0]
+
+
+def test_row_number_includes_rows_after_blank_gap():
+ xlsx = _make_gap_xlsx(gap_rows=600)
+ total = RAGFlowExcelParser.row_number("test.xlsx", xlsx)
+ assert total >= 1001
diff --git a/test/unit_test/deepdoc/parser/test_json_parser.py b/test/unit_test/deepdoc/parser/test_json_parser.py
index a548bcf452..1c15b6154a 100644
--- a/test/unit_test/deepdoc/parser/test_json_parser.py
+++ b/test/unit_test/deepdoc/parser/test_json_parser.py
@@ -26,17 +26,15 @@ chunk directly.
"""
import importlib.util
+import json
import os
import sys
-from unittest import mock
+import types
# Load json_parser by file path so we don't trigger deepdoc/parser/__init__.py
-# (which pulls in heavy parsers). json_parser only imports ``find_codec`` from
-# rag.nlp, and only inside ``__call__``; stub rag.nlp so the module imports.
-if "rag" not in sys.modules:
- sys.modules["rag"] = mock.MagicMock()
-if "rag.nlp" not in sys.modules:
- sys.modules["rag.nlp"] = mock.MagicMock()
+# (which pulls in heavy parsers). json_parser imports ``decode_text`` from
+# rag.nlp; stub rag.nlp only while loading, then restore sys.modules so later
+# tests resolve the real package regardless of collection order.
def _find_project_root(marker="pyproject.toml"):
@@ -48,17 +46,43 @@ def _find_project_root(marker="pyproject.toml"):
return None
-_PROJECT_ROOT = _find_project_root()
-
-_json_spec = importlib.util.spec_from_file_location(
- "deepdoc.parser.json_parser",
- os.path.join(_PROJECT_ROOT, "deepdoc", "parser", "json_parser.py"),
-)
-_json_mod = importlib.util.module_from_spec(_json_spec)
-sys.modules["deepdoc.parser.json_parser"] = _json_mod
-_json_spec.loader.exec_module(_json_mod)
-
-RAGFlowJsonParser = _json_mod.RAGFlowJsonParser
+def _load_json_parser():
+ try:
+ from rag.nlp import decode_text as production_decode_text
+ except ImportError:
+
+ def production_decode_text(blob, document_type="text"):
+ if blob.startswith(b"\xef\xbb\xbf"):
+ return blob.decode("utf-8-sig"), "utf-8-sig"
+ return blob.decode("utf-8"), "utf-8"
+
+ rag_nlp = types.ModuleType("rag.nlp")
+ rag_nlp.decode_text = production_decode_text
+ sys.modules["rag.nlp"] = rag_nlp
+ rag = types.ModuleType("rag")
+ rag.nlp = rag_nlp
+ sys.modules["rag"] = rag
+
+ project_root = _find_project_root()
+ json_spec = importlib.util.spec_from_file_location(
+ "deepdoc.parser.json_parser",
+ os.path.join(project_root, "deepdoc", "parser", "json_parser.py"),
+ )
+ json_mod = importlib.util.module_from_spec(json_spec)
+ sys.modules["deepdoc.parser.json_parser"] = json_mod
+ json_spec.loader.exec_module(json_mod)
+ return json_mod.RAGFlowJsonParser
+
+
+_saved_modules = {name: sys.modules.get(name) for name in ("rag", "rag.nlp")}
+try:
+ RAGFlowJsonParser = _load_json_parser()
+finally:
+ for name, module in _saved_modules.items():
+ if module is None:
+ sys.modules.pop(name, None)
+ else:
+ sys.modules[name] = module
def test_top_level_scalars_do_not_crash():
@@ -84,3 +108,24 @@ def test_objects_and_arrays_still_chunk():
parser = RAGFlowJsonParser()
assert parser._parse_json('{"a": 1}') == ['{"a": 1}']
assert parser._parse_json("[1, 2, 3]") != []
+
+
+def test_utf8_bom_json_parses_same_as_without_bom():
+ parser = RAGFlowJsonParser()
+ doc = '{"title": "Quarterly report", "body": "Revenue grew 12 percent."}'
+ expected = parser(doc.encode("utf-8"))
+ assert expected == ['{"title": "Quarterly report", "body": "Revenue grew 12 percent."}']
+ assert parser(b"\xef\xbb\xbf" + doc.encode("utf-8")) == expected
+
+
+def test_utf8_bom_jsonl_keeps_all_records():
+ parser = RAGFlowJsonParser()
+ lines = [json.dumps({"id": i}) for i in range(1, 21)]
+ jsonl = "\n".join(lines).encode("utf-8")
+ with_bom = parser(b"\xef\xbb\xbf" + jsonl)
+ without_bom = parser(jsonl)
+ assert len(with_bom) == 20
+ assert len(without_bom) == 20
+ assert with_bom == without_bom
+ ids = [json.loads(chunk)["id"] for chunk in with_bom]
+ assert ids == list(range(1, 21))