RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.
Go agent-harnessagent-native-databaseagentic-aiagentic-retrievalagentic-searchaiai-agentscontext-enginecontext-engineeringcontext-managementharness-engineeringknowledge-compilationragretrieval-augmented-generationretrieval-harnesssearch-harness
fix(parser): read text and inline elements under <body> as one item (#21108)
Lukas0cdbe73d4da7
diff --git a/internal/parser/parser/html_parser.go b/internal/parser/parser/html_parser.goindex d7457a8e92..82040a4550 100644--- a/internal/parser/parser/html_parser.go+++ b/internal/parser/parser/html_parser.go@@ -126,21 +126,37 @@ func (s *htmlWalkState) warnings() []string {// order. Inline formatting is collapsed into its parent text; script, style,// and noscript subtrees are skipped.func walkHTMLBlocksWithState(root *html.Node, out *[]map[string]any, state *htmlWalkState) {+ // Text and inline elements directly under <body> are one run of text, as+ // a browser lays them out, so they are collected into one item. Emitted+ // one item per text node and per inline element, "Hello <b>world</b>!"+ // came out as the three items "Hello", "world" and "!".+ var run *leafWriter+ flushRun := func() {+ if run != nil {+ flushLeafText(run, out, "text", true)+ run = nil+ }+ }+ defer flushRun()for child := root.FirstChild; child != nil; child = child.NextSibling {- if child.Type == html.TextNode {- if emitsLooseHTMLText(root) {- appendHTMLTextItem(out, child.Data, "text", true)+ // Invisible nodes neither show up nor end the run around them.+ if child.Type == html.CommentNode || (child.Type == html.ElementNode &&+ (child.Data == "script" || child.Data == "style" || child.Data == "noscript")) {+ continue+ }+ if emitsLooseHTMLText(root) && (child.Type == html.TextNode || isHTMLInlineElement(child)) {+ if run == nil {+ run = &leafWriter{b: &bytes.Buffer{}}}+ walkHTMLLeaf(child, run, out, "text", true, state)continue}+ flushRun()if child.Type != html.ElementNode {continue}tag := child.Dataswitch tag {- case "script", "style", "noscript":- // Skip executable / stylistic blocks entirely.- continuecase "head":// Skip document metadata so it does not pollute body text.continue@@ -188,6 +204,22 @@ func emitsLooseHTMLText(root *html.Node) bool {return root.Type == html.ElementNode && root.Data == "body"}+// isHTMLInlineElement reports whether n is a phrasing element, one a browser+// lays out inside the surrounding line of text rather than as a block.+func isHTMLInlineElement(n *html.Node) bool {+ if n.Type != html.ElementNode {+ return false+ }+ switch n.Data {+ case "a", "abbr", "b", "bdi", "bdo", "big", "br", "cite", "code", "data",+ "del", "dfn", "em", "font", "i", "img", "ins", "kbd", "label", "mark",+ "q", "s", "samp", "small", "span", "strike", "strong", "sub", "sup",+ "time", "tt", "u", "var", "wbr":+ return true+ }+ return false+}+func appendHTMLTextItem(out *[]map[string]any, text, ckType string, trim bool) {if trim {text = strings.TrimSpace(text)diff --git a/internal/parser/parser/html_parser_test.go b/internal/parser/parser/html_parser_test.goindex 02eaadeead..91f6cf2b67 100644--- a/internal/parser/parser/html_parser_test.go+++ b/internal/parser/parser/html_parser_test.go@@ -136,3 +136,28 @@ func TestHTMLParser_RemoveTOC(t *testing.T) {}}}++// TestHTMLParser_BodyTextAndInlineElementsFormOneItem checks that a run of+// text and inline elements directly under <body> is read as one item, the way+// a browser lays it out. Each text node and each inline element was its own+// item, so "Hello <b>world</b>, how are <a>you</a>?" came out as the five+// items "Hello", "world", ", how are", "you" and "?". A comment or a script+// inside the run does not end it; a block element does.+func TestHTMLParser_BodyTextAndInlineElementsFormOneItem(t *testing.T) {+ const doc = `<html><body>Hello <b>world</b>, how are <a href="x">you</a>?<br>` ++ `Run <code>make</code><!-- note --><script>track()</script> now` ++ `<p>A paragraph</p>tail <i>end</i></body></html>`+ res := NewHTMLParser().ParseWithResult(context.Background(), "page.html", []byte(doc))+ if res.Err != nil {+ t.Fatalf("ParseWithResult: %v", res.Err)+ }+ var got []string+ for _, item := range ItemsToMaps(res.JSON) {+ text, _ := item["text"].(string)+ got = append(got, text)+ }+ want := []string{"Hello world, how are you?\nRun make now", "A paragraph", "tail end"}+ if strings.Join(got, "|") != strings.Join(want, "|") {+ t.Errorf("items = %q, want %q", got, want)+ }+}