RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.
Go agent-harnessagent-native-databaseagentic-aiagentic-retrievalagentic-searchaiai-agentscontext-enginecontext-engineeringcontext-managementharness-engineeringknowledge-compilationragretrieval-augmented-generationretrieval-harnesssearch-harness
fix(parser): handle large sparse spreadsheets, JSON BOM, and XLSX QA chunking (#19237)
Sourav Rajviec0ff067c198
diff --git a/deepdoc/parser/excel_parser.py b/deepdoc/parser/excel_parser.pyindex 5a9c908f40..bfa688b26f 100644--- a/deepdoc/parser/excel_parser.py+++ b/deepdoc/parser/excel_parser.py@@ -172,37 +172,23 @@ class RAGFlowExcelParser:max_col = min(ws.max_column or 1, 50)- def row_has_data(row_idx):- for col_idx in range(1, max_col + 1):- cell = ws.cell(row=row_idx, column=col_idx)- if cell.value is not None and str(cell.value).strip():- return True- return False-- if not any(row_has_data(i) for i in range(1, min(101, max_row + 1))):- return 0-- left, right = 1, max_row- last_data_row = 1-- while left <= right:- mid = (left + right) // 2- found = False- for r in range(mid, min(mid + 10, max_row + 1)):- if row_has_data(r):- found = True- last_data_row = max(last_data_row, r)- break- if found:- left = mid + 1- else:- right = mid - 1-- for r in range(last_data_row, min(last_data_row + 500, max_row + 1)):- if row_has_data(r):- last_data_row = r-- return last_data_row+ # max_row is often inflated by styling far below real data. Scan only+ # materialized cells so we do not call ws.cell() on every empty row.+ highest = 0+ for (row_idx, col_idx), cell in ws._cells.items():+ if col_idx > max_col:+ continue+ if cell.value is not None and str(cell.value).strip():+ highest = max(highest, row_idx)++ if highest:+ logging.debug(+ "Excel row scan: max_row=%s max_col=%s detected_highest_data_row=%s",+ max_row,+ max_col,+ highest,+ )+ return highest@staticmethoddef _get_rows_limited(ws):diff --git a/internal/ingestion/component/chunker/qa_test.go b/internal/ingestion/component/chunker/qa_test.goindex 61bc0b1195..a3af4c2cb6 100644--- a/internal/ingestion/component/chunker/qa_test.go+++ b/internal/ingestion/component/chunker/qa_test.go@@ -357,3 +357,40 @@ func TestQAChunker_MarkdownRendersHTML(t *testing.T) {t.Fatalf("markdown not rendered to HTML: %q", cww)}}++func TestQAChunker_XLSXJSONRegression(t *testing.T) {+ comp, err := NewQAChunker(map[string]any{"lang": "english"})+ if err != nil {+ t.Fatal(err)+ }+ inputs := map[string]any{+ "name": "qa.xlsx",+ "output_format": "json",+ "json": []map[string]any{+ {+ "text": "<table><caption>Sheet1</caption><tr><th>question</th><th>answer</th></tr><tr><td>What is RAGFlow?</td><td>A RAG engine.</td></tr><tr><td>Where are the docs?</td><td>On the website.</td></tr></table>",+ "doc_type_kwd": "table",+ },+ },+ }+ out, err := comp.Invoke(t.Context(), nil, inputs)+ if err != nil {+ t.Fatalf("Invoke failed: %v", err)+ }+ chunks, _ := out["chunks"].([]map[string]any)+ if len(chunks) != 3 {+ t.Fatalf("expected 3 chunks, got %d", len(chunks))+ }+ expected := []string{+ "Question: question\tAnswer: answer",+ // rmQAPrefix strips a leading "A " answer marker from "A RAG engine."+ "Question: What is RAGFlow?\tAnswer: RAG engine.",+ "Question: Where are the docs?\tAnswer: On the website.",+ }+ for i, want := range expected {+ cww, _ := chunks[i]["text"].(string)+ if cww != want {+ t.Fatalf("chunk[%d] text = %q, want %q", i, cww, want)+ }+ }+}diff --git a/test/unit_test/deepdoc/parser/test_excel_parser.py b/test/unit_test/deepdoc/parser/test_excel_parser.pyindex c3768ec4f4..71880e6413 100644--- a/test/unit_test/deepdoc/parser/test_excel_parser.py+++ b/test/unit_test/deepdoc/parser/test_excel_parser.py@@ -224,3 +224,68 @@ def test_call_emits_zero_based_sheet_index():assert sheets == {0, 1}second = next(pos for _, pos in lines if pos[0] == 1)assert second == (1, 2, 2, 1, 1)+++def _make_large_used_range_xlsx(first_data_row, last_data_row, inflate_row=12000):+ from openpyxl import Workbook+ from openpyxl.styles import Font++ wb = Workbook()+ ws = wb.active+ ws.cell(row=first_data_row, column=1, value="question")+ ws.cell(row=first_data_row, column=2, value="answer")+ for r in range(first_data_row + 1, last_data_row + 1):+ ws.cell(row=r, column=1, value=f"Q{r}")+ ws.cell(row=r, column=2, value=f"A{r}")+ ws.cell(row=inflate_row, column=1).font = Font(bold=True)+ buf = BytesIO()+ wb.save(buf)+ buf.seek(0)+ return buf.read()+++def _make_gap_xlsx(gap_rows, inflate_row=12000):+ from openpyxl import Workbook++ wb = Workbook()+ ws = wb.active+ ws["A1"] = "Name"+ ws["B1"] = "City"+ for r in range(2, 202):+ ws.cell(row=r, column=1, value=f"n{r}")+ ws.cell(row=r, column=2, value=f"c{r}")+ second_block_start = 202 + gap_rows+ for r in range(second_block_start, second_block_start + 200):+ ws.cell(row=r, column=1, value=f"n{r}")+ ws.cell(row=r, column=2, value=f"c{r}")+ ws.cell(row=inflate_row, column=3, value=" ")+ buf = BytesIO()+ wb.save(buf)+ buf.seek(0)+ return buf.read()+++def test_large_used_range_with_blank_preamble_parses_all_rows():+ # Regression for #19236: data below row 100 must not be treated as empty.+ xlsx = _make_large_used_range_xlsx(150, 350)+ rows = RAGFlowExcelParser()(xlsx)+ assert len(rows) == 201+ assert "Q350" in rows[-1][0]+++def test_large_used_range_with_blank_gap_parses_both_blocks():+ # Regression for #19185: a blank gap must not truncate rows after it.+ xlsx = _make_gap_xlsx(gap_rows=600)+ rows = RAGFlowExcelParser()(xlsx)+ assert len(rows) == 400+ assert "n201" in rows[199][0]+ assert "n802" in rows[200][0]+++def test_row_number_includes_rows_after_blank_gap():+ xlsx = _make_gap_xlsx(gap_rows=600)+ total = RAGFlowExcelParser.row_number("test.xlsx", xlsx)+ assert total >= 1001diff --git a/test/unit_test/deepdoc/parser/test_json_parser.py b/test/unit_test/deepdoc/parser/test_json_parser.pyindex a548bcf452..1c15b6154a 100644--- a/test/unit_test/deepdoc/parser/test_json_parser.py+++ b/test/unit_test/deepdoc/parser/test_json_parser.py@@ -26,17 +26,15 @@ chunk directly."""import importlib.util+import jsonimport osimport sys-from unittest import mock+import types# Load json_parser by file path so we don't trigger deepdoc/parser/__init__.py-# (which pulls in heavy parsers). json_parser only imports ``find_codec`` from-# rag.nlp, and only inside ``__call__``; stub rag.nlp so the module imports.-if "rag" not in sys.modules:- sys.modules["rag"] = mock.MagicMock()-if "rag.nlp" not in sys.modules:- sys.modules["rag.nlp"] = mock.MagicMock()+# (which pulls in heavy parsers). json_parser imports ``decode_text`` from+# rag.nlp; stub rag.nlp only while loading, then restore sys.modules so later+# tests resolve the real package regardless of collection order.def _find_project_root(marker="pyproject.toml"):@@ -48,17 +46,43 @@ def _find_project_root(marker="pyproject.toml"):return None-_PROJECT_ROOT = _find_project_root()--_json_spec = importlib.util.spec_from_file_location(- "deepdoc.parser.json_parser",- os.path.join(_PROJECT_ROOT, "deepdoc", "parser", "json_parser.py"),-)-_json_mod = importlib.util.module_from_spec(_json_spec)-sys.modules["deepdoc.parser.json_parser"] = _json_mod-_json_spec.loader.exec_module(_json_mod)--RAGFlowJsonParser = _json_mod.RAGFlowJsonParser+def _load_json_parser():+ try:+ from rag.nlp import decode_text as production_decode_text+ except ImportError:++ def production_decode_text(blob, document_type="text"):+ if blob.startswith(b"\xef\xbb\xbf"):+ return blob.decode("utf-8-sig"), "utf-8-sig"+ return blob.decode("utf-8"), "utf-8"++ rag_nlp = types.ModuleType("rag.nlp")+ rag_nlp.decode_text = production_decode_text+ sys.modules["rag.nlp"] = rag_nlp+ rag = types.ModuleType("rag")+ rag.nlp = rag_nlp+ sys.modules["rag"] = rag++ project_root = _find_project_root()+ json_spec = importlib.util.spec_from_file_location(+ "deepdoc.parser.json_parser",+ os.path.join(project_root, "deepdoc", "parser", "json_parser.py"),+ )+ json_mod = importlib.util.module_from_spec(json_spec)+ sys.modules["deepdoc.parser.json_parser"] = json_mod+ json_spec.loader.exec_module(json_mod)+ return json_mod.RAGFlowJsonParser+++_saved_modules = {name: sys.modules.get(name) for name in ("rag", "rag.nlp")}+try:+ RAGFlowJsonParser = _load_json_parser()+finally:+ for name, module in _saved_modules.items():+ if module is None:+ sys.modules.pop(name, None)+ else:+ sys.modules[name] = moduledef test_top_level_scalars_do_not_crash():@@ -84,3 +108,24 @@ def test_objects_and_arrays_still_chunk():parser = RAGFlowJsonParser()assert parser._parse_json('{"a": 1}') == ['{"a": 1}']assert parser._parse_json("[1, 2, 3]") != []+++def test_utf8_bom_json_parses_same_as_without_bom():+ parser = RAGFlowJsonParser()+ doc = '{"title": "Quarterly report", "body": "Revenue grew 12 percent."}'+ expected = parser(doc.encode("utf-8"))+ assert expected == ['{"title": "Quarterly report", "body": "Revenue grew 12 percent."}']+ assert parser(b"\xef\xbb\xbf" + doc.encode("utf-8")) == expected+++def test_utf8_bom_jsonl_keeps_all_records():+ parser = RAGFlowJsonParser()+ lines = [json.dumps({"id": i}) for i in range(1, 21)]+ jsonl = "\n".join(lines).encode("utf-8")+ with_bom = parser(b"\xef\xbb\xbf" + jsonl)+ without_bom = parser(jsonl)+ assert len(with_bom) == 20+ assert len(without_bom) == 20+ assert with_bom == without_bom+ ids = [json.loads(chunk)["id"] for chunk in with_bom]+ assert ids == list(range(1, 21))