RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.
Go agent-harnessagent-native-databaseagentic-aiagentic-retrievalagentic-searchaiai-agentscontext-enginecontext-engineeringcontext-managementharness-engineeringknowledge-compilationragretrieval-augmented-generationretrieval-harnesssearch-harness
fix(parser): flush pending text before table/image chunks in _build_cks (#19521)
wang39385f3e8961994
diff --git a/rag/nlp/__init__.py b/rag/nlp/__init__.pyindex 31a319f7f3..6270633dbd 100644--- a/rag/nlp/__init__.py+++ b/rag/nlp/__init__.py@@ -1645,6 +1645,12 @@ def concat_img(img1, img2):def _build_cks(sections, delimiter):+ """Split ``(text, image, table)`` sections into typed chunks.++ Text is buffered and split on the parsed ``delimiter`` field; each table+ or image section becomes its own chunk. Returns+ ``(cks, tables, images, has_custom)``.+ """cks = []tables = []images = []@@ -1659,6 +1665,29 @@ def _build_cks(sections, delimiter):pattern = r"(%s)" % split_pattern if split_pattern else ""seg = ""++ def _flush_seg():+ """Emit pending text before a table/image chunk is appended.++ Plain text is buffered in ``seg`` and only flushed when a delimiter+ matches, whereas table/image chunks are appended immediately. Without+ this flush the buffered text lands *after* the table/image, breaking+ document order and swapping context_above / context_below in+ _add_context() (which decides "above"/"below" by array position).+ """+ nonlocal seg+ if seg and seg.strip():+ s = seg.strip()+ cks.append(+ {+ "text": s,+ "image": None,+ "ck_type": "text",+ "tk_nums": num_tokens_from_string(s),+ }+ )+ seg = ""+for text, image, table in sections:# normalize text: ensure string and prepend newline for continuityif not text:@@ -1668,6 +1697,7 @@ def _build_cks(sections, delimiter):if table:# table chunk+ _flush_seg()ck_text = text + str(table)idx = len(cks)cks.append(@@ -1683,6 +1713,7 @@ def _build_cks(sections, delimiter):if image:# image chunk (text kept as-is for context)+ _flush_seg()idx = len(cks)cks.append({diff --git a/test/unit_test/rag/nlp/test_docx_chunk_order.py b/test/unit_test/rag/nlp/test_docx_chunk_order.pynew file mode 100644index 0000000000..129612eaa1--- /dev/null+++ b/test/unit_test/rag/nlp/test_docx_chunk_order.py@@ -0,0 +1,124 @@+#+# Copyright 2025 The InfiniFlow Authors. All Rights Reserved.+#+# Licensed under the Apache License, Version 2.0 (the "License");+# you may not use this file except in compliance with the License.+# You may obtain a copy of the License at+#+# http://www.apache.org/licenses/LICENSE-2.0+#+# Unless required by applicable law or agreed to in writing, software+# distributed under the License is distributed on an "AS IS" BASIS,+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+# See the License for the specific language governing permissions and+# limitations under the License.+#++"""Pin the chunk order produced by ``_build_cks`` when a table/image follows text.++Reported in #19520: plain text is buffered in ``seg`` until a delimiter+matches, whereas table/image chunks are appended immediately. A caption+paragraph that preceded a table (or image) was therefore emitted *after* it,+which both broke the document order and inverted ``context_above`` /+``context_below`` in ``_add_context`` (it decides "above" / "below" purely by+array position).++The fix flushes the pending text before a table/image chunk is appended.+These tests pin:++* a caption paragraph that buffered before a table/image stays before it;+* ``context_above`` / ``context_below`` follow the document order;+* the two control cases — a table with no preceding text, and an empty+ ``delimiter`` field — keep their pre-existing, already-correct behaviour.+"""++from rag.nlp import _add_context, _build_cks++TABLE_HTML = "<table><tr><td>x</td></tr></table>"+DELIMITER = "\n"+CONTEXT_SIZE = 128+++def _build(sections, delimiter=DELIMITER):+ """Return ``(cks, tables, images)`` for ``sections`` (drops ``has_custom``)."""+ cks, tables, images, _ = _build_cks(sections, delimiter)+ return cks, tables, images+++def test_caption_paragraph_precedes_table_chunk_when_buffered_before_it():+ cks, tables, _ = _build(+ [+ ("table caption", None, None),+ ("", None, TABLE_HTML),+ ("following paragraph", None, None),+ ]+ )+ assert [ck["ck_type"] for ck in cks] == ["text", "table", "text"]+ assert cks[tables[0] - 1]["text"] == "table caption"+ assert cks[tables[0] + 1]["text"] == "following paragraph"+++def test_table_context_above_and_below_follow_document_order():+ cks, tables, _ = _build(+ [+ ("table caption", None, None),+ ("", None, TABLE_HTML),+ ("following paragraph", None, None),+ ]+ )+ _add_context(cks, tables[0], CONTEXT_SIZE)+ assert cks[tables[0]]["context_above"] == "table caption"+ assert cks[tables[0]]["context_below"] == "following paragraph"+++def test_caption_paragraph_precedes_image_chunk_when_buffered_before_it():+ cks, _, images = _build(+ [+ ("figure caption", None, None),+ ("", "IMG", None),+ ("following paragraph", None, None),+ ]+ )+ assert [ck["ck_type"] for ck in cks] == ["text", "image", "text"]+ assert cks[images[0] - 1]["text"] == "figure caption"+ assert cks[images[0] + 1]["text"] == "following paragraph"+++def test_image_context_above_and_below_follow_document_order():+ cks, _, images = _build(+ [+ ("figure caption", None, None),+ ("", "IMG", None),+ ("following paragraph", None, None),+ ]+ )+ _add_context(cks, images[0], CONTEXT_SIZE)+ assert cks[images[0]]["context_above"] == "figure caption"+ assert cks[images[0]]["context_below"] == "following paragraph"+++def test_table_without_preceding_text_keeps_its_order():+ """Control: nothing is buffered, so the table already led on main."""+ cks, tables, _ = _build(+ [+ ("", None, TABLE_HTML),+ ("following paragraph", None, None),+ ]+ )+ assert [ck["ck_type"] for ck in cks] == ["table", "text"]+ assert tables == [0]+++def test_empty_delimiter_keeps_its_order():+ """Control: with no delimiter, text is appended directly and never buffered."""+ cks, tables, _ = _build(+ [+ ("table caption", None, None),+ ("", None, TABLE_HTML),+ ("following paragraph", None, None),+ ],+ delimiter="",+ )+ assert [ck["ck_type"] for ck in cks] == ["text", "table", "text"]+ assert cks[0]["text"] == "table caption"+ assert cks[tables[0] + 1]["text"] == "following paragraph"