RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.
Go agent-harnessagent-native-databaseagentic-aiagentic-retrievalagentic-searchaiai-agentscontext-enginecontext-engineeringcontext-managementharness-engineeringknowledge-compilationragretrieval-augmented-generationretrieval-harnesssearch-harness
fix(parser): start a new line before every HTML block element (#21107)
Lukas781397b3ba4c
diff --git a/internal/parser/parser/email_parser_test.go b/internal/parser/parser/email_parser_test.goindex dc6df40941..b9bc028149 100644--- a/internal/parser/parser/email_parser_test.go+++ b/internal/parser/parser/email_parser_test.go@@ -1503,6 +1503,7 @@ func TestHTMLBodyToText(t *testing.T) {{"layout table cells", `<table><tr><td>a</td><td>b</td></tr></table>`, "a\nb"},{"multi-row layout table stays line-separated", `<p>intro</p><table><tr><td>a</td><td>b</td></tr><tr><td>c</td></tr></table><p>outro</p>`, "intro\na\nb\nc\noutro"},{"inline tags stay on one line", `<p>Hello <b>world</b></p>`, "Hello world"},+ {"definition list and rule", `<div>A<dl><dt>Term</dt><dd>Def</dd></dl>B<hr>C</div>`, "A\nTerm\nDef\nB\nC"},{"br forces a break", `one<br>two`, "one\ntwo"},{"head script style skipped", `<html><head><title>t</title><style>.x{color:red}</style></head><body><p>visible</p><script>var x=1;</script></body></html>`, "visible"},}diff --git a/internal/parser/parser/html_parser.go b/internal/parser/parser/html_parser.goindex 82040a4550..375d61eb0f 100644--- a/internal/parser/parser/html_parser.go+++ b/internal/parser/parser/html_parser.go@@ -405,15 +405,11 @@ func walkHTMLLeaf(n *html.Node, w *leafWriter, out *[]map[string]any, ckType strreturn}// Add a line break between block children so headings, paragraphs,- // and list items don't run together.- if !w.pre {- switch n.Data {- case "h1", "h2", "h3", "h4", "h5", "h6", "p", "li", "pre",- "tr", "blockquote":- if w.b.Len() > 0 && !w.endsNL {- w.hardBreak()- }- }+ // and list items don't run together. A block starts on a new line as+ // well as ending one: without the break before it, the text in front+ // of a nested <div> ran into the block's own ("Name:Alice").+ if !w.pre && isBlockTag(n.Data) && w.b.Len() > 0 && !w.endsNL {+ w.hardBreak()}for child := n.FirstChild; child != nil; child = child.NextSibling {walkHTMLLeaf(child, w, out, ckType, trim, state)@@ -626,10 +622,16 @@ func htmlAttribute(n *html.Node, key string) string {return ""}+// isBlockTag reports whether a browser lays tag out as a block, on lines of+// its own (the elements the HTML rendering rules display as block or list+// item), so its text must not run into the text around it.func isBlockTag(tag string) bool {switch tag {case "h1", "h2", "h3", "h4", "h5", "h6", "p", "li", "pre",- "tr", "blockquote", "div", "section", "article", "header", "footer":+ "tr", "blockquote", "div", "section", "article", "header", "footer",+ "main", "nav", "aside", "address", "hgroup", "search", "center",+ "ul", "ol", "menu", "dl", "dt", "dd", "figure", "figcaption",+ "details", "summary", "dialog", "form", "fieldset", "legend", "hr":return true}return falsediff --git a/internal/parser/parser/html_parser_test.go b/internal/parser/parser/html_parser_test.goindex 91f6cf2b67..22abc64812 100644--- a/internal/parser/parser/html_parser_test.go+++ b/internal/parser/parser/html_parser_test.go@@ -28,6 +28,42 @@ func TestHTMLParser_ConfigureFromSetup_NilSafe(t *testing.T) {}}+// TestHTMLParser_BlockElementsStartANewLine checks that a block element's+// text starts and ends a line of its own. Only p, li, headings, tr and+// blockquote broke the line in front of them, and dl, figure, hr, nav and the+// other block elements outside isBlockTag broke it on neither side, so the+// text around them ran into theirs: "Name:Alice", "ATermDefinitionB".+func TestHTMLParser_BlockElementsStartANewLine(t *testing.T) {+ tests := []struct {+ name string+ body string+ want string+ }{+ {"text before a nested div", `<div>Name:<div>Alice</div></div>`, "Name:\nAlice"},+ {"inline before a nested div", `<div><span>Total</span><div>42</div></div>`, "Total\n42"},+ {"definition list", `<div>A<dl><dt>Term</dt><dd>Definition</dd></dl>B</div>`, "A\nTerm\nDefinition\nB"},+ {"figure caption", `<div>See<figure><figcaption>Fig 1</figcaption></figure>below</div>`, "See\nFig 1\nbelow"},+ {"horizontal rule", `<div>above<hr>below</div>`, "above\nbelow"},+ {"nav inside an article", `<article>left<nav>Menu</nav>right</article>`, "left\nMenu\nright"},+ {"inline elements stay on the line", `<div>Hello <b>bold</b> and <a href="x">link</a> text</div>`, "Hello bold and link text"},+ }+ for _, tt := range tests {+ t.Run(tt.name, func(t *testing.T) {+ res := NewHTMLParser().ParseWithResult(context.Background(), "page.html", []byte("<html><body>"+tt.body+"</body></html>"))+ if res.Err != nil {+ t.Fatalf("ParseWithResult: %v", res.Err)+ }+ items := ItemsToMaps(res.JSON)+ if len(items) != 1 {+ t.Fatalf("items = %v, want one text item", items)+ }+ if got := items[0]["text"]; got != tt.want {+ t.Errorf("text = %q, want %q", got, tt.want)+ }+ })+ }+}+// TestHTMLParser_RemoveHeaderFooter verifies that <header>/<footer>// tags and role="banner"/role="contentinfo" elements are stripped// before parsing (mirrors Python parser.py:1083-1084 pre-parse).