Skip to content

RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.

fix(syncer): keep the word boundary a styled space carries into markdown (#19722)

mkaaadfc99054083cb
diff --git a/internal/syncer/connector/html_markdown.go b/internal/syncer/connector/html_markdown.go
new file mode 100644
index 0000000000..65ffa88266
--- /dev/null
+++ b/internal/syncer/connector/html_markdown.go
@@ -0,0 +1,88 @@
+//
+// Copyright 2026 The InfiniFlow Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+
+package connector
+
+import (
+ "strings"
+
+ md "github.com/JohannesKaufmann/html-to-markdown"
+ "github.com/PuerkitoBio/goquery"
+ "golang.org/x/net/html"
+)
+
+// inlineTagsWithDefaultRule are the tags for which html-to-markdown ships a
+// conversion rule of its own (strong/b/em/i/a). When such an element holds real
+// content we fall through to that rule; the remaining styled inline tags have no
+// rule of their own, so we keep their content like the library's default.
+var inlineTagsWithDefaultRule = map[string]bool{
+ "a": true, "b": true, "strong": true, "em": true, "i": true,
+}
+
+// newMarkdownConverter returns the HTML-to-Markdown converter shared by the
+// connectors in this package: html-to-markdown with an asterisk emphasis
+// delimiter and the whitespace-preserving plugin below.
+func newMarkdownConverter() *md.Converter {
+ converter := md.NewConverter("", true, &md.Options{EmDelimiter: "*"})
+ converter.Use(preserveInlineWhitespace())
+ return converter
+}
+
+// preserveInlineWhitespace keeps a whitespace-only styled inline element's text
+// when a word directly touches it, so the words on either side don't run
+// together. A word processor keeps a differently formatted space as a run of its
+// own, so `First<strong> </strong>Last` is what a document with a bolded space
+// converts to, and `FirstLast` is what gets chunked and embedded. Elements whose
+// neighbours are other elements are left to html-to-markdown's neighbour-aware
+// spacing, which already inserts the boundary there.
+func preserveInlineWhitespace() md.Plugin {
+ return func(converter *md.Converter) []md.Rule {
+ return []md.Rule{{
+ Filter: []string{"a", "b", "strong", "em", "i", "u", "s", "del", "strike", "sub", "sup", "span"},
+ Replacement: func(content string, selec *goquery.Selection, opt *md.Options) *string {
+ if strings.TrimSpace(content) != "" {
+ if inlineTagsWithDefaultRule[goquery.NodeName(selec)] {
+ return nil
+ }
+ return &content
+ }
+ if raw := selec.Text(); raw != "" && strings.TrimSpace(raw) == "" && hasAdjacentTextWord(selec) {
+ return &raw
+ }
+ if inlineTagsWithDefaultRule[goquery.NodeName(selec)] {
+ return nil
+ }
+ return &content
+ },
+ }}
+ }
+}
+
+// hasAdjacentTextWord reports whether a non-whitespace text node directly
+// neighbours the element, i.e. a word that would run into its neighbours if the
+// element's own whitespace were dropped.
+func hasAdjacentTextWord(selec *goquery.Selection) bool {
+ if len(selec.Nodes) == 0 {
+ return false
+ }
+ node := selec.Nodes[0]
+ for _, sibling := range []*html.Node{node.PrevSibling, node.NextSibling} {
+ if sibling != nil && sibling.Type == html.TextNode && strings.TrimSpace(sibling.Data) != "" {
+ return true
+ }
+ }
+ return false
+}
diff --git a/internal/syncer/connector/html_markdown_test.go b/internal/syncer/connector/html_markdown_test.go
new file mode 100644
index 0000000000..c2692aa842
--- /dev/null
+++ b/internal/syncer/connector/html_markdown_test.go
@@ -0,0 +1,98 @@
+//
+// Copyright 2026 The InfiniFlow Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+
+package connector
+
+import (
+ "strings"
+ "testing"
+)
+
+// convertMarkdownForTest runs the shared converter and trims the surrounding
+// whitespace html-to-markdown adds for a paragraph.
+func convertMarkdownForTest(t *testing.T, html string) string {
+ t.Helper()
+ converter := newMarkdownConverter()
+ out, err := converter.ConvertString(html)
+ if err != nil {
+ t.Fatalf("ConvertString(%q): %v", html, err)
+ }
+ return strings.TrimSpace(out)
+}
+
+func TestHTMLToMarkdownKeepsWhitespaceOnlyElementBoundary(t *testing.T) {
+ for _, tag := range []string{"a", "b", "strong", "em", "i", "u", "s", "del", "strike", "sub", "sup"} {
+ t.Run(tag, func(t *testing.T) {
+ attributes := ""
+ if tag == "a" {
+ attributes = ` href="https://example.com"`
+ }
+ got := convertMarkdownForTest(t, "<p>First<"+tag+attributes+"> </"+tag+">Last</p>")
+ if got != "First Last" {
+ t.Fatalf("got %q, want %q", got, "First Last")
+ }
+ })
+ }
+}
+
+func TestHTMLToMarkdownSpaceBetweenStyledRuns(t *testing.T) {
+ got := convertMarkdownForTest(t, "<p><b>First</b><b> </b><b>Last</b></p>")
+ if got != "**First** **Last**" {
+ t.Fatalf("got %q, want %q", got, "**First** **Last**")
+ }
+}
+
+func TestHTMLToMarkdownNonBreakingSpaceStaysItself(t *testing.T) {
+ got := convertMarkdownForTest(t, "<p>First<b>&#160;</b>Last</p>")
+ if got != "First\u00a0Last" {
+ t.Fatalf("got %q, want %q", got, "First\u00a0Last")
+ }
+}
+
+func TestHTMLToMarkdownRealContentUnchanged(t *testing.T) {
+ cases := []struct{ html, want string }{
+ {"<p>First <b>bold</b> Last</p>", "First **bold** Last"},
+ {"<p>First <em>italic</em> Last</p>", "First *italic* Last"},
+ {"<p>First <code>code</code> Last</p>", "First `code` Last"},
+ {`<p>First <a href="https://example.com">link</a> Last</p>`, "First [link](https://example.com) Last"},
+ {"<p>First<b></b>Last</p>", "FirstLast"},
+ }
+ for _, tc := range cases {
+ if got := convertMarkdownForTest(t, tc.html); got != tc.want {
+ t.Errorf("convertMarkdownForTest(%q) = %q, want %q", tc.html, got, tc.want)
+ }
+ }
+}
+
+func TestMoodleHTMLToMarkdownPreservesBoundary(t *testing.T) {
+ out, err := moodleHTMLToMarkdown("<p>First<strong> </strong>Last</p>")
+ if err != nil {
+ t.Fatalf("moodleHTMLToMarkdown: %v", err)
+ }
+ if strings.TrimSpace(out) != "First Last" {
+ t.Fatalf("got %q, want %q", strings.TrimSpace(out), "First Last")
+ }
+}
+
+func TestSitemapHTMLToMarkdownPreservesBoundary(t *testing.T) {
+ out, err := sitemapHTMLToMarkdown([]byte("<p>First<strong> </strong>Last</p>"))
+ if err != nil {
+ t.Fatalf("sitemapHTMLToMarkdown: %v", err)
+ }
+ if out != "First Last" {
+ t.Fatalf("got %q, want %q", out, "First Last")
+ }
+}
diff --git a/internal/syncer/connector/moodle.go b/internal/syncer/connector/moodle.go
index adcc3bf5f0..0d5177e213 100644
--- a/internal/syncer/connector/moodle.go
+++ b/internal/syncer/connector/moodle.go
@@ -30,8 +30,6 @@ import (
"strconv"
"strings"
"time"
-
- md "github.com/JohannesKaufmann/html-to-markdown"
)
const (
@@ -375,7 +373,7 @@ func addMoodleToken(fileURL, token string) string {
}
func moodleHTMLToMarkdown(html string) (string, error) {
- converter := md.NewConverter("", true, &md.Options{EmDelimiter: "*"})
+ converter := newMarkdownConverter()
out, err := converter.ConvertString(html)
if err != nil {
return "", fmt.Errorf("Moodle HTML to Markdown conversion failed: %w", err)
diff --git a/internal/syncer/connector/sitemap.go b/internal/syncer/connector/sitemap.go
index 14cfa8572a..2e25edf1e8 100644
--- a/internal/syncer/connector/sitemap.go
+++ b/internal/syncer/connector/sitemap.go
@@ -33,7 +33,6 @@ import (
"ragflow/internal/utility"
- md "github.com/JohannesKaufmann/html-to-markdown"
"golang.org/x/net/html"
)
@@ -726,7 +725,7 @@ func hostOf(rawURL string) string {
// sitemapHTMLToMarkdown converts a page to Markdown, dropping navigation chrome.
func sitemapHTMLToMarkdown(body []byte) (string, error) {
- converter := md.NewConverter("", true, &md.Options{EmDelimiter: "*"})
+ converter := newMarkdownConverter()
converter.Remove("script", "style", "noscript", "nav", "header", "footer", "aside", "form", "iframe", "svg")
out, err := converter.ConvertString(string(body))
if err != nil {