Skip to content

RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.

fix(parser): start a new line before every HTML block element (#21107)

Lukas781397b3ba4c
diff --git a/internal/parser/parser/email_parser_test.go b/internal/parser/parser/email_parser_test.go
index dc6df40941..b9bc028149 100644
--- a/internal/parser/parser/email_parser_test.go
+++ b/internal/parser/parser/email_parser_test.go
@@ -1503,6 +1503,7 @@ func TestHTMLBodyToText(t *testing.T) {
{"layout table cells", `<table><tr><td>a</td><td>b</td></tr></table>`, "a\nb"},
{"multi-row layout table stays line-separated", `<p>intro</p><table><tr><td>a</td><td>b</td></tr><tr><td>c</td></tr></table><p>outro</p>`, "intro\na\nb\nc\noutro"},
{"inline tags stay on one line", `<p>Hello <b>world</b></p>`, "Hello world"},
+ {"definition list and rule", `<div>A<dl><dt>Term</dt><dd>Def</dd></dl>B<hr>C</div>`, "A\nTerm\nDef\nB\nC"},
{"br forces a break", `one<br>two`, "one\ntwo"},
{"head script style skipped", `<html><head><title>t</title><style>.x{color:red}</style></head><body><p>visible</p><script>var x=1;</script></body></html>`, "visible"},
}
diff --git a/internal/parser/parser/html_parser.go b/internal/parser/parser/html_parser.go
index 82040a4550..375d61eb0f 100644
--- a/internal/parser/parser/html_parser.go
+++ b/internal/parser/parser/html_parser.go
@@ -405,15 +405,11 @@ func walkHTMLLeaf(n *html.Node, w *leafWriter, out *[]map[string]any, ckType str
return
}
// Add a line break between block children so headings, paragraphs,
- // and list items don't run together.
- if !w.pre {
- switch n.Data {
- case "h1", "h2", "h3", "h4", "h5", "h6", "p", "li", "pre",
- "tr", "blockquote":
- if w.b.Len() > 0 && !w.endsNL {
- w.hardBreak()
- }
- }
+ // and list items don't run together. A block starts on a new line as
+ // well as ending one: without the break before it, the text in front
+ // of a nested <div> ran into the block's own ("Name:Alice").
+ if !w.pre && isBlockTag(n.Data) && w.b.Len() > 0 && !w.endsNL {
+ w.hardBreak()
}
for child := n.FirstChild; child != nil; child = child.NextSibling {
walkHTMLLeaf(child, w, out, ckType, trim, state)
@@ -626,10 +622,16 @@ func htmlAttribute(n *html.Node, key string) string {
return ""
}
+// isBlockTag reports whether a browser lays tag out as a block, on lines of
+// its own (the elements the HTML rendering rules display as block or list
+// item), so its text must not run into the text around it.
func isBlockTag(tag string) bool {
switch tag {
case "h1", "h2", "h3", "h4", "h5", "h6", "p", "li", "pre",
- "tr", "blockquote", "div", "section", "article", "header", "footer":
+ "tr", "blockquote", "div", "section", "article", "header", "footer",
+ "main", "nav", "aside", "address", "hgroup", "search", "center",
+ "ul", "ol", "menu", "dl", "dt", "dd", "figure", "figcaption",
+ "details", "summary", "dialog", "form", "fieldset", "legend", "hr":
return true
}
return false
diff --git a/internal/parser/parser/html_parser_test.go b/internal/parser/parser/html_parser_test.go
index 91f6cf2b67..22abc64812 100644
--- a/internal/parser/parser/html_parser_test.go
+++ b/internal/parser/parser/html_parser_test.go
@@ -28,6 +28,42 @@ func TestHTMLParser_ConfigureFromSetup_NilSafe(t *testing.T) {
}
}
+// TestHTMLParser_BlockElementsStartANewLine checks that a block element's
+// text starts and ends a line of its own. Only p, li, headings, tr and
+// blockquote broke the line in front of them, and dl, figure, hr, nav and the
+// other block elements outside isBlockTag broke it on neither side, so the
+// text around them ran into theirs: "Name:Alice", "ATermDefinitionB".
+func TestHTMLParser_BlockElementsStartANewLine(t *testing.T) {
+ tests := []struct {
+ name string
+ body string
+ want string
+ }{
+ {"text before a nested div", `<div>Name:<div>Alice</div></div>`, "Name:\nAlice"},
+ {"inline before a nested div", `<div><span>Total</span><div>42</div></div>`, "Total\n42"},
+ {"definition list", `<div>A<dl><dt>Term</dt><dd>Definition</dd></dl>B</div>`, "A\nTerm\nDefinition\nB"},
+ {"figure caption", `<div>See<figure><figcaption>Fig 1</figcaption></figure>below</div>`, "See\nFig 1\nbelow"},
+ {"horizontal rule", `<div>above<hr>below</div>`, "above\nbelow"},
+ {"nav inside an article", `<article>left<nav>Menu</nav>right</article>`, "left\nMenu\nright"},
+ {"inline elements stay on the line", `<div>Hello <b>bold</b> and <a href="x">link</a> text</div>`, "Hello bold and link text"},
+ }
+ for _, tt := range tests {
+ t.Run(tt.name, func(t *testing.T) {
+ res := NewHTMLParser().ParseWithResult(context.Background(), "page.html", []byte("<html><body>"+tt.body+"</body></html>"))
+ if res.Err != nil {
+ t.Fatalf("ParseWithResult: %v", res.Err)
+ }
+ items := ItemsToMaps(res.JSON)
+ if len(items) != 1 {
+ t.Fatalf("items = %v, want one text item", items)
+ }
+ if got := items[0]["text"]; got != tt.want {
+ t.Errorf("text = %q, want %q", got, tt.want)
+ }
+ })
+ }
+}
+
// TestHTMLParser_RemoveHeaderFooter verifies that <header>/<footer>
// tags and role="banner"/role="contentinfo" elements are stripped
// before parsing (mirrors Python parser.py:1083-1084 pre-parse).