Skip to content

RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.

fix(parser): keep the word boundary a styled space carries into markdown (#19693)

Lukas Buck73061944d843
diff --git a/common/data_source/moodle_connector.py b/common/data_source/moodle_connector.py
index aef473f381..d4b54c931f 100644
--- a/common/data_source/moodle_connector.py
+++ b/common/data_source/moodle_connector.py
@@ -7,7 +7,7 @@ from datetime import datetime, timezone
from retry import retry
from typing import Any, Optional
-from markdownify import markdownify as md
+from common.markdown_utils import html_to_markdown as md
from moodle import Moodle as MoodleClient, MoodleException
from common.data_source.config import INDEX_BATCH_SIZE
diff --git a/common/markdown_utils.py b/common/markdown_utils.py
new file mode 100644
index 0000000000..94d1e4e766
--- /dev/null
+++ b/common/markdown_utils.py
@@ -0,0 +1,70 @@
+#
+# Copyright 2025 The InfiniFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+"""HTML to Markdown conversion for the ingestion paths."""
+
+from typing import Any
+
+from markdownify import MarkdownConverter
+
+# Inline tags whose markdownify conversion runs the text through chomp(), which lifts the
+# surrounding whitespace out of the text and then returns an empty string once nothing is
+# left. A whitespace-only element therefore disappears together with its whitespace and the
+# words on either side of it run together. A word processor keeps a differently formatted
+# space as a run of its own, so `First<strong> </strong>Last` is what a .docx with a bolded
+# space converts to, and `FirstLast` is what gets chunked, embedded and retrieved.
+_WHITESPACE_ONLY_PRESERVING_TAGS = frozenset(
+ {
+ "a",
+ "b",
+ "code",
+ "del",
+ "em",
+ "i",
+ "kbd",
+ "s",
+ "samp",
+ "strike",
+ "strong",
+ "sub",
+ "sup",
+ "u",
+ }
+)
+
+
+class _WhitespacePreservingConverter(MarkdownConverter):
+ """Same as markdownify's converter, but a whitespace-only inline element keeps its text."""
+
+ def get_conv_fn(self, tag_name: str) -> Any:
+ convert_fn = super().get_conv_fn(tag_name)
+ if convert_fn is None or tag_name.lower() not in _WHITESPACE_ONLY_PRESERVING_TAGS:
+ return convert_fn
+
+ def _keep_whitespace_only(el: Any, text: str, *args: Any, **kwargs: Any) -> str:
+ if not text.strip():
+ return text
+
+ return convert_fn(el, text, *args, **kwargs)
+
+ return _keep_whitespace_only
+
+
+def html_to_markdown(html: str, **options: Any) -> str:
+ """Convert `html` to Markdown without losing the word boundaries it carries.
+
+ `options` are markdownify's own, so this is a drop-in for `markdownify(html, ...)`.
+ """
+ return _WhitespacePreservingConverter(**options).convert(html)
diff --git a/rag/app/naive.py b/rag/app/naive.py
index 272fa1a896..690de2d81e 100644
--- a/rag/app/naive.py
+++ b/rag/app/naive.py
@@ -800,7 +800,8 @@ class Docx(DocxParser):
import uuid
import mammoth
- from markdownify import markdownify
+
+ from common.markdown_utils import html_to_markdown
docx_file = BytesIO(binary) if binary is not None else open(filename, "rb")
@@ -827,7 +828,7 @@ class Docx(DocxParser):
html = result.value
- markdown_text = markdownify(html)
+ markdown_text = html_to_markdown(html)
return markdown_text
finally:
diff --git a/test/unit_test/common/test_markdown_utils.py b/test/unit_test/common/test_markdown_utils.py
new file mode 100644
index 0000000000..0d3c0189ef
--- /dev/null
+++ b/test/unit_test/common/test_markdown_utils.py
@@ -0,0 +1,62 @@
+#
+# Copyright 2025 The InfiniFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+"""`html_to_markdown` keeps the word boundaries the HTML carries.
+
+markdownify drops an inline element that holds nothing but whitespace, and takes the
+whitespace with it, so two words end up as one in whatever gets chunked and embedded.
+"""
+
+import pytest
+
+from common.markdown_utils import html_to_markdown
+
+
+ "tag",
+ ["a", "b", "strong", "em", "i", "u", "s", "del", "code", "sub", "sup"],
+)
+def test_a_whitespace_only_element_keeps_the_word_boundary(tag):
+ attributes = ' href="https://example.com"' if tag == "a" else ""
+
+ assert html_to_markdown(f"<p>First<{tag}{attributes}> </{tag}>Last</p>").strip() == "First Last"
+
+
+def test_a_space_between_two_styled_runs_survives():
+ """A word processor keeps a differently formatted space as a run of its own."""
+ assert html_to_markdown("<p><b>First</b><b> </b><b>Last</b></p>").strip() == "**First** **Last**"
+
+
+def test_a_non_breaking_space_stays_itself():
+ assert html_to_markdown("<p>First<b>&#160;</b>Last</p>").strip() == "First Last"
+
+
+ ("html", "expected"),
+ [
+ ("<p>First <b>bold</b> Last</p>", "First **bold** Last"),
+ ("<p>First <em>italic</em> Last</p>", "First *italic* Last"),
+ ("<p>First <code>code</code> Last</p>", "First `code` Last"),
+ ('<p>First <a href="https://example.com">link</a> Last</p>', "First [link](https://example.com) Last"),
+ ("<p>First<b></b>Last</p>", "FirstLast"),
+ ],
+)
+def test_an_element_with_content_converts_as_before(html, expected):
+ """The control: only an element with nothing but whitespace in it is passed through."""
+ assert html_to_markdown(html).strip() == expected
+
+
+def test_markdownify_options_are_forwarded():
+ assert html_to_markdown("<h1>Title</h1>", heading_style="ATX").strip() == "# Title"