RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.
Go agent-harnessagent-native-databaseagentic-aiagentic-retrievalagentic-searchaiai-agentscontext-enginecontext-engineeringcontext-managementharness-engineeringknowledge-compilationragretrieval-augmented-generationretrieval-harnesssearch-harness
fix(parser): keep the word boundary a styled space carries into markdown (#19693)
Lukas Buck73061944d843
diff --git a/common/data_source/moodle_connector.py b/common/data_source/moodle_connector.pyindex aef473f381..d4b54c931f 100644--- a/common/data_source/moodle_connector.py+++ b/common/data_source/moodle_connector.py@@ -7,7 +7,7 @@ from datetime import datetime, timezonefrom retry import retryfrom typing import Any, Optional-from markdownify import markdownify as md+from common.markdown_utils import html_to_markdown as mdfrom moodle import Moodle as MoodleClient, MoodleExceptionfrom common.data_source.config import INDEX_BATCH_SIZEdiff --git a/common/markdown_utils.py b/common/markdown_utils.pynew file mode 100644index 0000000000..94d1e4e766--- /dev/null+++ b/common/markdown_utils.py@@ -0,0 +1,70 @@+#+# Copyright 2025 The InfiniFlow Authors. All Rights Reserved.+#+# Licensed under the Apache License, Version 2.0 (the "License");+# you may not use this file except in compliance with the License.+# You may obtain a copy of the License at+#+# http://www.apache.org/licenses/LICENSE-2.0+#+# Unless required by applicable law or agreed to in writing, software+# distributed under the License is distributed on an "AS IS" BASIS,+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+# See the License for the specific language governing permissions and+# limitations under the License.+#+"""HTML to Markdown conversion for the ingestion paths."""++from typing import Any++from markdownify import MarkdownConverter++# Inline tags whose markdownify conversion runs the text through chomp(), which lifts the+# surrounding whitespace out of the text and then returns an empty string once nothing is+# left. A whitespace-only element therefore disappears together with its whitespace and the+# words on either side of it run together. A word processor keeps a differently formatted+# space as a run of its own, so `First<strong> </strong>Last` is what a .docx with a bolded+# space converts to, and `FirstLast` is what gets chunked, embedded and retrieved.+_WHITESPACE_ONLY_PRESERVING_TAGS = frozenset(+ {+ "a",+ "b",+ "code",+ "del",+ "em",+ "i",+ "kbd",+ "s",+ "samp",+ "strike",+ "strong",+ "sub",+ "sup",+ "u",+ }+)+++class _WhitespacePreservingConverter(MarkdownConverter):+ """Same as markdownify's converter, but a whitespace-only inline element keeps its text."""++ def get_conv_fn(self, tag_name: str) -> Any:+ convert_fn = super().get_conv_fn(tag_name)+ if convert_fn is None or tag_name.lower() not in _WHITESPACE_ONLY_PRESERVING_TAGS:+ return convert_fn++ def _keep_whitespace_only(el: Any, text: str, *args: Any, **kwargs: Any) -> str:+ if not text.strip():+ return text++ return convert_fn(el, text, *args, **kwargs)++ return _keep_whitespace_only+++def html_to_markdown(html: str, **options: Any) -> str:+ """Convert `html` to Markdown without losing the word boundaries it carries.++ `options` are markdownify's own, so this is a drop-in for `markdownify(html, ...)`.+ """+ return _WhitespacePreservingConverter(**options).convert(html)diff --git a/rag/app/naive.py b/rag/app/naive.pyindex 272fa1a896..690de2d81e 100644--- a/rag/app/naive.py+++ b/rag/app/naive.py@@ -800,7 +800,8 @@ class Docx(DocxParser):import uuidimport mammoth- from markdownify import markdownify++ from common.markdown_utils import html_to_markdowndocx_file = BytesIO(binary) if binary is not None else open(filename, "rb")@@ -827,7 +828,7 @@ class Docx(DocxParser):html = result.value- markdown_text = markdownify(html)+ markdown_text = html_to_markdown(html)return markdown_textfinally:diff --git a/test/unit_test/common/test_markdown_utils.py b/test/unit_test/common/test_markdown_utils.pynew file mode 100644index 0000000000..0d3c0189ef--- /dev/null+++ b/test/unit_test/common/test_markdown_utils.py@@ -0,0 +1,62 @@+#+# Copyright 2025 The InfiniFlow Authors. All Rights Reserved.+#+# Licensed under the Apache License, Version 2.0 (the "License");+# you may not use this file except in compliance with the License.+# You may obtain a copy of the License at+#+# http://www.apache.org/licenses/LICENSE-2.0+#+# Unless required by applicable law or agreed to in writing, software+# distributed under the License is distributed on an "AS IS" BASIS,+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+# See the License for the specific language governing permissions and+# limitations under the License.+#+"""`html_to_markdown` keeps the word boundaries the HTML carries.++markdownify drops an inline element that holds nothing but whitespace, and takes the+whitespace with it, so two words end up as one in whatever gets chunked and embedded.+"""++import pytest++from common.markdown_utils import html_to_markdown+++ "tag",+ ["a", "b", "strong", "em", "i", "u", "s", "del", "code", "sub", "sup"],+)+def test_a_whitespace_only_element_keeps_the_word_boundary(tag):+ attributes = ' href="https://example.com"' if tag == "a" else ""++ assert html_to_markdown(f"<p>First<{tag}{attributes}> </{tag}>Last</p>").strip() == "First Last"+++def test_a_space_between_two_styled_runs_survives():+ """A word processor keeps a differently formatted space as a run of its own."""+ assert html_to_markdown("<p><b>First</b><b> </b><b>Last</b></p>").strip() == "**First** **Last**"+++def test_a_non_breaking_space_stays_itself():+ assert html_to_markdown("<p>First<b> </b>Last</p>").strip() == "First Last"+++ ("html", "expected"),+ [+ ("<p>First <b>bold</b> Last</p>", "First **bold** Last"),+ ("<p>First <em>italic</em> Last</p>", "First *italic* Last"),+ ("<p>First <code>code</code> Last</p>", "First `code` Last"),+ ('<p>First <a href="https://example.com">link</a> Last</p>', "First [link](https://example.com) Last"),+ ("<p>First<b></b>Last</p>", "FirstLast"),+ ],+)+def test_an_element_with_content_converts_as_before(html, expected):+ """The control: only an element with nothing but whitespace in it is passed through."""+ assert html_to_markdown(html).strip() == expected+++def test_markdownify_options_are_forwarded():+ assert html_to_markdown("<h1>Title</h1>", heading_style="ATX").strip() == "# Title"