RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.
Go agent-harnessagent-native-databaseagentic-aiagentic-retrievalagentic-searchaiai-agentscontext-enginecontext-engineeringcontext-managementharness-engineeringknowledge-compilationragretrieval-augmented-generationretrieval-harnesssearch-harness
fix(syncer): keep the word boundary a styled space carries into markdown (#19722)
mkaaadfc99054083cb
diff --git a/internal/syncer/connector/html_markdown.go b/internal/syncer/connector/html_markdown.gonew file mode 100644index 0000000000..65ffa88266--- /dev/null+++ b/internal/syncer/connector/html_markdown.go@@ -0,0 +1,88 @@+//+// Copyright 2026 The InfiniFlow Authors. All Rights Reserved.+//+// Licensed under the Apache License, Version 2.0 (the "License");+// you may not use this file except in compliance with the License.+// You may obtain a copy of the License at+//+// http://www.apache.org/licenses/LICENSE-2.0+//+// Unless required by applicable law or agreed to in writing, software+// distributed under the License is distributed on an "AS IS" BASIS,+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+// See the License for the specific language governing permissions and+// limitations under the License.+//++package connector++import (+ "strings"++ md "github.com/JohannesKaufmann/html-to-markdown"+ "github.com/PuerkitoBio/goquery"+ "golang.org/x/net/html"+)++// inlineTagsWithDefaultRule are the tags for which html-to-markdown ships a+// conversion rule of its own (strong/b/em/i/a). When such an element holds real+// content we fall through to that rule; the remaining styled inline tags have no+// rule of their own, so we keep their content like the library's default.+var inlineTagsWithDefaultRule = map[string]bool{+ "a": true, "b": true, "strong": true, "em": true, "i": true,+}++// newMarkdownConverter returns the HTML-to-Markdown converter shared by the+// connectors in this package: html-to-markdown with an asterisk emphasis+// delimiter and the whitespace-preserving plugin below.+func newMarkdownConverter() *md.Converter {+ converter := md.NewConverter("", true, &md.Options{EmDelimiter: "*"})+ converter.Use(preserveInlineWhitespace())+ return converter+}++// preserveInlineWhitespace keeps a whitespace-only styled inline element's text+// when a word directly touches it, so the words on either side don't run+// together. A word processor keeps a differently formatted space as a run of its+// own, so `First<strong> </strong>Last` is what a document with a bolded space+// converts to, and `FirstLast` is what gets chunked and embedded. Elements whose+// neighbours are other elements are left to html-to-markdown's neighbour-aware+// spacing, which already inserts the boundary there.+func preserveInlineWhitespace() md.Plugin {+ return func(converter *md.Converter) []md.Rule {+ return []md.Rule{{+ Filter: []string{"a", "b", "strong", "em", "i", "u", "s", "del", "strike", "sub", "sup", "span"},+ Replacement: func(content string, selec *goquery.Selection, opt *md.Options) *string {+ if strings.TrimSpace(content) != "" {+ if inlineTagsWithDefaultRule[goquery.NodeName(selec)] {+ return nil+ }+ return &content+ }+ if raw := selec.Text(); raw != "" && strings.TrimSpace(raw) == "" && hasAdjacentTextWord(selec) {+ return &raw+ }+ if inlineTagsWithDefaultRule[goquery.NodeName(selec)] {+ return nil+ }+ return &content+ },+ }}+ }+}++// hasAdjacentTextWord reports whether a non-whitespace text node directly+// neighbours the element, i.e. a word that would run into its neighbours if the+// element's own whitespace were dropped.+func hasAdjacentTextWord(selec *goquery.Selection) bool {+ if len(selec.Nodes) == 0 {+ return false+ }+ node := selec.Nodes[0]+ for _, sibling := range []*html.Node{node.PrevSibling, node.NextSibling} {+ if sibling != nil && sibling.Type == html.TextNode && strings.TrimSpace(sibling.Data) != "" {+ return true+ }+ }+ return false+}diff --git a/internal/syncer/connector/html_markdown_test.go b/internal/syncer/connector/html_markdown_test.gonew file mode 100644index 0000000000..c2692aa842--- /dev/null+++ b/internal/syncer/connector/html_markdown_test.go@@ -0,0 +1,98 @@+//+// Copyright 2026 The InfiniFlow Authors. All Rights Reserved.+//+// Licensed under the Apache License, Version 2.0 (the "License");+// you may not use this file except in compliance with the License.+// You may obtain a copy of the License at+//+// http://www.apache.org/licenses/LICENSE-2.0+//+// Unless required by applicable law or agreed to in writing, software+// distributed under the License is distributed on an "AS IS" BASIS,+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+// See the License for the specific language governing permissions and+// limitations under the License.+//++package connector++import (+ "strings"+ "testing"+)++// convertMarkdownForTest runs the shared converter and trims the surrounding+// whitespace html-to-markdown adds for a paragraph.+func convertMarkdownForTest(t *testing.T, html string) string {+ t.Helper()+ converter := newMarkdownConverter()+ out, err := converter.ConvertString(html)+ if err != nil {+ t.Fatalf("ConvertString(%q): %v", html, err)+ }+ return strings.TrimSpace(out)+}++func TestHTMLToMarkdownKeepsWhitespaceOnlyElementBoundary(t *testing.T) {+ for _, tag := range []string{"a", "b", "strong", "em", "i", "u", "s", "del", "strike", "sub", "sup"} {+ t.Run(tag, func(t *testing.T) {+ attributes := ""+ if tag == "a" {+ attributes = ` href="https://example.com"`+ }+ got := convertMarkdownForTest(t, "<p>First<"+tag+attributes+"> </"+tag+">Last</p>")+ if got != "First Last" {+ t.Fatalf("got %q, want %q", got, "First Last")+ }+ })+ }+}++func TestHTMLToMarkdownSpaceBetweenStyledRuns(t *testing.T) {+ got := convertMarkdownForTest(t, "<p><b>First</b><b> </b><b>Last</b></p>")+ if got != "**First** **Last**" {+ t.Fatalf("got %q, want %q", got, "**First** **Last**")+ }+}++func TestHTMLToMarkdownNonBreakingSpaceStaysItself(t *testing.T) {+ got := convertMarkdownForTest(t, "<p>First<b> </b>Last</p>")+ if got != "First\u00a0Last" {+ t.Fatalf("got %q, want %q", got, "First\u00a0Last")+ }+}++func TestHTMLToMarkdownRealContentUnchanged(t *testing.T) {+ cases := []struct{ html, want string }{+ {"<p>First <b>bold</b> Last</p>", "First **bold** Last"},+ {"<p>First <em>italic</em> Last</p>", "First *italic* Last"},+ {"<p>First <code>code</code> Last</p>", "First `code` Last"},+ {`<p>First <a href="https://example.com">link</a> Last</p>`, "First [link](https://example.com) Last"},+ {"<p>First<b></b>Last</p>", "FirstLast"},+ }+ for _, tc := range cases {+ if got := convertMarkdownForTest(t, tc.html); got != tc.want {+ t.Errorf("convertMarkdownForTest(%q) = %q, want %q", tc.html, got, tc.want)+ }+ }+}++func TestMoodleHTMLToMarkdownPreservesBoundary(t *testing.T) {+ out, err := moodleHTMLToMarkdown("<p>First<strong> </strong>Last</p>")+ if err != nil {+ t.Fatalf("moodleHTMLToMarkdown: %v", err)+ }+ if strings.TrimSpace(out) != "First Last" {+ t.Fatalf("got %q, want %q", strings.TrimSpace(out), "First Last")+ }+}++func TestSitemapHTMLToMarkdownPreservesBoundary(t *testing.T) {+ out, err := sitemapHTMLToMarkdown([]byte("<p>First<strong> </strong>Last</p>"))+ if err != nil {+ t.Fatalf("sitemapHTMLToMarkdown: %v", err)+ }+ if out != "First Last" {+ t.Fatalf("got %q, want %q", out, "First Last")+ }+}diff --git a/internal/syncer/connector/moodle.go b/internal/syncer/connector/moodle.goindex adcc3bf5f0..0d5177e213 100644--- a/internal/syncer/connector/moodle.go+++ b/internal/syncer/connector/moodle.go@@ -30,8 +30,6 @@ import ("strconv""strings""time"-- md "github.com/JohannesKaufmann/html-to-markdown")const (@@ -375,7 +373,7 @@ func addMoodleToken(fileURL, token string) string {}func moodleHTMLToMarkdown(html string) (string, error) {- converter := md.NewConverter("", true, &md.Options{EmDelimiter: "*"})+ converter := newMarkdownConverter()out, err := converter.ConvertString(html)if err != nil {return "", fmt.Errorf("Moodle HTML to Markdown conversion failed: %w", err)diff --git a/internal/syncer/connector/sitemap.go b/internal/syncer/connector/sitemap.goindex 14cfa8572a..2e25edf1e8 100644--- a/internal/syncer/connector/sitemap.go+++ b/internal/syncer/connector/sitemap.go@@ -33,7 +33,6 @@ import ("ragflow/internal/utility"- md "github.com/JohannesKaufmann/html-to-markdown""golang.org/x/net/html")@@ -726,7 +725,7 @@ func hostOf(rawURL string) string {// sitemapHTMLToMarkdown converts a page to Markdown, dropping navigation chrome.func sitemapHTMLToMarkdown(body []byte) (string, error) {- converter := md.NewConverter("", true, &md.Options{EmDelimiter: "*"})+ converter := newMarkdownConverter()converter.Remove("script", "style", "noscript", "nav", "header", "footer", "aside", "form", "iframe", "svg")out, err := converter.ConvertString(string(body))if err != nil {