Skip to content

RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.

fix(parser): flush pending text before table/image chunks in _build_cks (#19521)

wang39385f3e8961994
diff --git a/rag/nlp/__init__.py b/rag/nlp/__init__.py
index 31a319f7f3..6270633dbd 100644
--- a/rag/nlp/__init__.py
+++ b/rag/nlp/__init__.py
@@ -1645,6 +1645,12 @@ def concat_img(img1, img2):
def _build_cks(sections, delimiter):
+ """Split ``(text, image, table)`` sections into typed chunks.
+
+ Text is buffered and split on the parsed ``delimiter`` field; each table
+ or image section becomes its own chunk. Returns
+ ``(cks, tables, images, has_custom)``.
+ """
cks = []
tables = []
images = []
@@ -1659,6 +1665,29 @@ def _build_cks(sections, delimiter):
pattern = r"(%s)" % split_pattern if split_pattern else ""
seg = ""
+
+ def _flush_seg():
+ """Emit pending text before a table/image chunk is appended.
+
+ Plain text is buffered in ``seg`` and only flushed when a delimiter
+ matches, whereas table/image chunks are appended immediately. Without
+ this flush the buffered text lands *after* the table/image, breaking
+ document order and swapping context_above / context_below in
+ _add_context() (which decides "above"/"below" by array position).
+ """
+ nonlocal seg
+ if seg and seg.strip():
+ s = seg.strip()
+ cks.append(
+ {
+ "text": s,
+ "image": None,
+ "ck_type": "text",
+ "tk_nums": num_tokens_from_string(s),
+ }
+ )
+ seg = ""
+
for text, image, table in sections:
# normalize text: ensure string and prepend newline for continuity
if not text:
@@ -1668,6 +1697,7 @@ def _build_cks(sections, delimiter):
if table:
# table chunk
+ _flush_seg()
ck_text = text + str(table)
idx = len(cks)
cks.append(
@@ -1683,6 +1713,7 @@ def _build_cks(sections, delimiter):
if image:
# image chunk (text kept as-is for context)
+ _flush_seg()
idx = len(cks)
cks.append(
{
diff --git a/test/unit_test/rag/nlp/test_docx_chunk_order.py b/test/unit_test/rag/nlp/test_docx_chunk_order.py
new file mode 100644
index 0000000000..129612eaa1
--- /dev/null
+++ b/test/unit_test/rag/nlp/test_docx_chunk_order.py
@@ -0,0 +1,124 @@
+#
+# Copyright 2025 The InfiniFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+"""Pin the chunk order produced by ``_build_cks`` when a table/image follows text.
+
+Reported in #19520: plain text is buffered in ``seg`` until a delimiter
+matches, whereas table/image chunks are appended immediately. A caption
+paragraph that preceded a table (or image) was therefore emitted *after* it,
+which both broke the document order and inverted ``context_above`` /
+``context_below`` in ``_add_context`` (it decides "above" / "below" purely by
+array position).
+
+The fix flushes the pending text before a table/image chunk is appended.
+These tests pin:
+
+* a caption paragraph that buffered before a table/image stays before it;
+* ``context_above`` / ``context_below`` follow the document order;
+* the two control cases — a table with no preceding text, and an empty
+ ``delimiter`` field — keep their pre-existing, already-correct behaviour.
+"""
+
+from rag.nlp import _add_context, _build_cks
+
+TABLE_HTML = "<table><tr><td>x</td></tr></table>"
+DELIMITER = "\n"
+CONTEXT_SIZE = 128
+
+
+def _build(sections, delimiter=DELIMITER):
+ """Return ``(cks, tables, images)`` for ``sections`` (drops ``has_custom``)."""
+ cks, tables, images, _ = _build_cks(sections, delimiter)
+ return cks, tables, images
+
+
+def test_caption_paragraph_precedes_table_chunk_when_buffered_before_it():
+ cks, tables, _ = _build(
+ [
+ ("table caption", None, None),
+ ("", None, TABLE_HTML),
+ ("following paragraph", None, None),
+ ]
+ )
+ assert [ck["ck_type"] for ck in cks] == ["text", "table", "text"]
+ assert cks[tables[0] - 1]["text"] == "table caption"
+ assert cks[tables[0] + 1]["text"] == "following paragraph"
+
+
+def test_table_context_above_and_below_follow_document_order():
+ cks, tables, _ = _build(
+ [
+ ("table caption", None, None),
+ ("", None, TABLE_HTML),
+ ("following paragraph", None, None),
+ ]
+ )
+ _add_context(cks, tables[0], CONTEXT_SIZE)
+ assert cks[tables[0]]["context_above"] == "table caption"
+ assert cks[tables[0]]["context_below"] == "following paragraph"
+
+
+def test_caption_paragraph_precedes_image_chunk_when_buffered_before_it():
+ cks, _, images = _build(
+ [
+ ("figure caption", None, None),
+ ("", "IMG", None),
+ ("following paragraph", None, None),
+ ]
+ )
+ assert [ck["ck_type"] for ck in cks] == ["text", "image", "text"]
+ assert cks[images[0] - 1]["text"] == "figure caption"
+ assert cks[images[0] + 1]["text"] == "following paragraph"
+
+
+def test_image_context_above_and_below_follow_document_order():
+ cks, _, images = _build(
+ [
+ ("figure caption", None, None),
+ ("", "IMG", None),
+ ("following paragraph", None, None),
+ ]
+ )
+ _add_context(cks, images[0], CONTEXT_SIZE)
+ assert cks[images[0]]["context_above"] == "figure caption"
+ assert cks[images[0]]["context_below"] == "following paragraph"
+
+
+def test_table_without_preceding_text_keeps_its_order():
+ """Control: nothing is buffered, so the table already led on main."""
+ cks, tables, _ = _build(
+ [
+ ("", None, TABLE_HTML),
+ ("following paragraph", None, None),
+ ]
+ )
+ assert [ck["ck_type"] for ck in cks] == ["table", "text"]
+ assert tables == [0]
+
+
+def test_empty_delimiter_keeps_its_order():
+ """Control: with no delimiter, text is appended directly and never buffered."""
+ cks, tables, _ = _build(
+ [
+ ("table caption", None, None),
+ ("", None, TABLE_HTML),
+ ("following paragraph", None, None),
+ ],
+ delimiter="",
+ )
+ assert [ck["ck_type"] for ck in cks] == ["text", "table", "text"]
+ assert cks[0]["text"] == "table caption"
+ assert cks[tables[0] + 1]["text"] == "following paragraph"