Skip to content

RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.

fix(parser): read text and inline elements under <body> as one item (#21108)

Lukas0cdbe73d4da7
diff --git a/internal/parser/parser/html_parser.go b/internal/parser/parser/html_parser.go
index d7457a8e92..82040a4550 100644
--- a/internal/parser/parser/html_parser.go
+++ b/internal/parser/parser/html_parser.go
@@ -126,21 +126,37 @@ func (s *htmlWalkState) warnings() []string {
// order. Inline formatting is collapsed into its parent text; script, style,
// and noscript subtrees are skipped.
func walkHTMLBlocksWithState(root *html.Node, out *[]map[string]any, state *htmlWalkState) {
+ // Text and inline elements directly under <body> are one run of text, as
+ // a browser lays them out, so they are collected into one item. Emitted
+ // one item per text node and per inline element, "Hello <b>world</b>!"
+ // came out as the three items "Hello", "world" and "!".
+ var run *leafWriter
+ flushRun := func() {
+ if run != nil {
+ flushLeafText(run, out, "text", true)
+ run = nil
+ }
+ }
+ defer flushRun()
for child := root.FirstChild; child != nil; child = child.NextSibling {
- if child.Type == html.TextNode {
- if emitsLooseHTMLText(root) {
- appendHTMLTextItem(out, child.Data, "text", true)
+ // Invisible nodes neither show up nor end the run around them.
+ if child.Type == html.CommentNode || (child.Type == html.ElementNode &&
+ (child.Data == "script" || child.Data == "style" || child.Data == "noscript")) {
+ continue
+ }
+ if emitsLooseHTMLText(root) && (child.Type == html.TextNode || isHTMLInlineElement(child)) {
+ if run == nil {
+ run = &leafWriter{b: &bytes.Buffer{}}
}
+ walkHTMLLeaf(child, run, out, "text", true, state)
continue
}
+ flushRun()
if child.Type != html.ElementNode {
continue
}
tag := child.Data
switch tag {
- case "script", "style", "noscript":
- // Skip executable / stylistic blocks entirely.
- continue
case "head":
// Skip document metadata so it does not pollute body text.
continue
@@ -188,6 +204,22 @@ func emitsLooseHTMLText(root *html.Node) bool {
return root.Type == html.ElementNode && root.Data == "body"
}
+// isHTMLInlineElement reports whether n is a phrasing element, one a browser
+// lays out inside the surrounding line of text rather than as a block.
+func isHTMLInlineElement(n *html.Node) bool {
+ if n.Type != html.ElementNode {
+ return false
+ }
+ switch n.Data {
+ case "a", "abbr", "b", "bdi", "bdo", "big", "br", "cite", "code", "data",
+ "del", "dfn", "em", "font", "i", "img", "ins", "kbd", "label", "mark",
+ "q", "s", "samp", "small", "span", "strike", "strong", "sub", "sup",
+ "time", "tt", "u", "var", "wbr":
+ return true
+ }
+ return false
+}
+
func appendHTMLTextItem(out *[]map[string]any, text, ckType string, trim bool) {
if trim {
text = strings.TrimSpace(text)
diff --git a/internal/parser/parser/html_parser_test.go b/internal/parser/parser/html_parser_test.go
index 02eaadeead..91f6cf2b67 100644
--- a/internal/parser/parser/html_parser_test.go
+++ b/internal/parser/parser/html_parser_test.go
@@ -136,3 +136,28 @@ func TestHTMLParser_RemoveTOC(t *testing.T) {
}
}
}
+
+// TestHTMLParser_BodyTextAndInlineElementsFormOneItem checks that a run of
+// text and inline elements directly under <body> is read as one item, the way
+// a browser lays it out. Each text node and each inline element was its own
+// item, so "Hello <b>world</b>, how are <a>you</a>?" came out as the five
+// items "Hello", "world", ", how are", "you" and "?". A comment or a script
+// inside the run does not end it; a block element does.
+func TestHTMLParser_BodyTextAndInlineElementsFormOneItem(t *testing.T) {
+ const doc = `<html><body>Hello <b>world</b>, how are <a href="x">you</a>?<br>` +
+ `Run <code>make</code><!-- note --><script>track()</script> now` +
+ `<p>A paragraph</p>tail <i>end</i></body></html>`
+ res := NewHTMLParser().ParseWithResult(context.Background(), "page.html", []byte(doc))
+ if res.Err != nil {
+ t.Fatalf("ParseWithResult: %v", res.Err)
+ }
+ var got []string
+ for _, item := range ItemsToMaps(res.JSON) {
+ text, _ := item["text"].(string)
+ got = append(got, text)
+ }
+ want := []string{"Hello world, how are you?\nRun make now", "A paragraph", "tail end"}
+ if strings.Join(got, "|") != strings.Join(want, "|") {
+ t.Errorf("items = %q, want %q", got, want)
+ }
+}