RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.
Go agent-harnessagent-native-databaseagentic-aiagentic-retrievalagentic-searchaiai-agentscontext-enginecontext-engineeringcontext-managementharness-engineeringknowledge-compilationragretrieval-augmented-generationretrieval-harnesssearch-harness
Fix(parser): keep real line breaks when merging HTML fragments and PDF boxes (#17856)
Jackf41f866aa1ec
diff --git a/deepdoc/parser/html_parser.py b/deepdoc/parser/html_parser.pyindex b593ad849a..1f4b3a1a60 100644--- a/deepdoc/parser/html_parser.py+++ b/deepdoc/parser/html_parser.py@@ -105,27 +105,31 @@ class RAGFlowHtmlParser:return table_str_list@classmethod- def read_text_recursively(cls, element, parser_result, chunk_token_num=512, parent_name=None, block_id=None):+ def read_text_recursively(cls, element, parser_result, chunk_token_num=512, parent_name=None, block_id=None, newline_before=0):if isinstance(element, NavigableString):- content = element.strip()+ # Keep the text verbatim (do not strip): source whitespace is+ # folded later by merge_block_text, which preserves <br> breaks.+ content = str(element)- def is_valid_html(content):+ def is_valid_html(text):try:- soup = BeautifulSoup(content, "html.parser")+ soup = BeautifulSoup(text, "html.parser")return bool(soup.find())except Exception:return Falsereturn_info = []- if content:+ if content.strip():if is_valid_html(content):soup = BeautifulSoup(content, "html.parser")- child_info = cls.read_text_recursively(soup, parser_result, chunk_token_num, element.name, block_id)+ child_info = cls.read_text_recursively(soup, parser_result, chunk_token_num, element.name, block_id, newline_before)parser_result.extend(child_info)else:- info = {"content": element.strip(), "tag_name": "inner_text", "metadata": {"block_id": block_id}}+ info = {"content": content, "tag_name": "inner_text", "metadata": {"block_id": block_id}}if parent_name:info["tag_name"] = parent_name+ if newline_before:+ info["metadata"]["newline_before"] = newline_beforereturn_info.append(info)return return_infoelif isinstance(element, Tag):@@ -139,22 +143,55 @@ class RAGFlowHtmlParser:else:if str.lower(element.name) in BLOCK_TAGS:block_id = str(uuid.uuid1())+ # A <br> is a hard line break. Thread a pending break count+ # through so it survives nesting (e.g. "<p>A<br><span>B</span></p>")+ # and repeated breaks ("<br><br>"). The count resets only after+ # a descendant emits text, mirroring the Go walker.+ pending_newlines = int(newline_before)for child in element.children:- child_info = cls.read_text_recursively(child, parser_result, chunk_token_num, element.name, block_id)+ if isinstance(child, Tag) and str.lower(child.name) == "br":+ pending_newlines += 1+ continue+ result_count = len(parser_result)+ child_info = cls.read_text_recursively(child, parser_result, chunk_token_num, element.name, block_id, pending_newlines)parser_result.extend(child_info)+ if child_info or len(parser_result) > result_count:+ pending_newlines = 0return []+ # Hard line-break sentinel used between fragments so CSS whitespace+ # folding (which collapses real whitespace runs) does not eat explicit+ # <br> breaks. Replaced with "\n" after folding.+ _HARD_BREAK = "\x0b"+ _WS_RE = re.compile(r"[ \t\r\n\f]+")+ _PRE_TAGS = ("pre", "textarea")++ @classmethod+ def _fold_block(cls, raw, preserve):+ if preserve:+ return raw.replace(cls._HARD_BREAK, "\n")+ # Collapse each line between hard breaks: runs of collapsible+ # whitespace become one space, and line edges are trimmed. This+ # matches the Go walker's CSS whitespace folding.+ lines = raw.split(cls._HARD_BREAK)+ folded = [cls._WS_RE.sub(" ", ln).strip() for ln in lines]+ return "\n".join(folded)+@classmethoddef merge_block_text(cls, parser_result):block_content = []current_content = ""+ current_is_pre = Falsetable_info_list = []last_block_id = Nonefor item in parser_result:- content = item.get("content")+ content = item.get("content") or ""tag_name = item.get("tag_name")title_flag = tag_name in TITLE_TAGSblock_id = item.get("metadata", {}).get("block_id")+ # newline_before is a break count: 0 -> join verbatim (no+ # separator), >0 -> insert that many hard breaks (one per <br>).+ newline_before = item.get("metadata", {}).get("newline_before", 0)if block_id:if title_flag:content = f"{TITLE_TAGS[tag_name]} {content}"@@ -163,18 +200,21 @@ class RAGFlowHtmlParser:# precedes the first block (last_block_id is None there),# which was otherwise overwritten and lost.if current_content:- block_content.append(current_content)+ block_content.append(cls._fold_block(current_content, current_is_pre))current_content = content+ current_is_pre = tag_name in cls._PRE_TAGSlast_block_id = block_idelse:- current_content += (" " if current_content else "") + content+ sep = cls._HARD_BREAK * newline_before+ current_content += sep + contentelse:if tag_name == "table":table_info_list.append(item)else:- current_content += (" " if current_content else "") + content+ sep = cls._HARD_BREAK * newline_before+ current_content += sep + contentif current_content:- block_content.append(current_content)+ block_content.append(cls._fold_block(current_content, current_is_pre))return block_content, table_info_list# Characters from scripts written without spaces between words (CJK, kana,diff --git a/internal/parser/parser/html_parser.go b/internal/parser/parser/html_parser.goindex 13d9ab28f3..313f080843 100644--- a/internal/parser/parser/html_parser.go+++ b/internal/parser/parser/html_parser.go@@ -110,7 +110,7 @@ func walkHTMLBlocks(root *html.Node, out *[]map[string]any) {for child := root.FirstChild; child != nil; child = child.NextSibling {if child.Type == html.TextNode {if emitsLooseHTMLText(root) {- appendHTMLTextItem(out, child.Data, "text")+ appendHTMLTextItem(out, child.Data, "text", true)}continue}@@ -131,7 +131,7 @@ func walkHTMLBlocks(root *html.Node, out *[]map[string]any) {continue}text := htmlLeafText(child)- appendHTMLTextItem(out, text, htmlTagToCkType(tag))+ appendHTMLTextItem(out, text, htmlTagToCkType(tag), tag != "pre" && tag != "textarea")}}@@ -139,8 +139,10 @@ func emitsLooseHTMLText(root *html.Node) bool {return root.Type == html.ElementNode && root.Data == "body"}-func appendHTMLTextItem(out *[]map[string]any, text, ckType string) {- text = strings.TrimSpace(text)+func appendHTMLTextItem(out *[]map[string]any, text, ckType string, trim bool) {+ if trim {+ text = strings.TrimSpace(text)+ }if text == "" {return}@@ -174,38 +176,116 @@ func htmlTagToCkType(tag string) string {return "text"}+// leafWriter accumulates the visible text of an HTML subtree while applying+// CSS whitespace folding (the default white-space: normal rules):+// - collapsible whitespace runs collapse to a single space;+// - leading/trailing whitespace of a line is dropped;+// - a <br> forces a hard line break (and resets the leading-whitespace state);+// - <pre>/<textarea> are emitted verbatim (no folding, no injected breaks).+type leafWriter struct {+ b *bytes.Buffer+ lastSpace bool // last written rune was a collapsed single space+ lineStart bool // at the start of a line, so leading whitespace is dropped+ endsNL bool // builder currently ends with a hard line break+ pre bool // inside <pre>/<textarea>: emit verbatim+}++func isCollapsibleWS(r rune) bool {+ return r == ' ' || r == '\t' || r == '\n' || r == '\r' || r == '\f'+}++// writeText appends s, folding collapsible whitespace unless in pre mode.+func (w *leafWriter) writeText(s string) {+ if w.pre {+ for _, r := range s {+ w.b.WriteRune(r)+ w.endsNL = r == '\n'+ }+ w.lastSpace = false+ w.lineStart = false+ return+ }+ for _, r := range s {+ if isCollapsibleWS(r) {+ if w.lineStart || w.lastSpace {+ continue+ }+ w.b.WriteRune(' ')+ w.lastSpace = true+ w.lineStart = false+ w.endsNL = false+ continue+ }+ w.b.WriteRune(r)+ w.lastSpace = false+ w.lineStart = false+ w.endsNL = false+ }+}++// hardBreak inserts a forced line break (a <br> or block boundary). Per CSS,+// whitespace immediately before a break is dropped (so "Hello <br>" yields+// "Hello\n", not "Hello \n"). Inside <pre>/<textarea> whitespace is preserved,+// so the preceding space is kept.+func (w *leafWriter) hardBreak() {+ if !w.pre && w.lastSpace && w.b.Len() > 0 {+ w.b.Truncate(w.b.Len() - 1)+ }+ w.b.WriteByte('\n')+ w.lastSpace = false+ w.lineStart = true+ w.endsNL = true+}+// htmlLeafText joins the visible text of an HTML node and its-// descendants. <script>/<style>/<noscript> subtrees are skipped.-// The output preserves whitespace runs so headings like-// "<h1>Hello world</h1>" round-trip with their spacing intact.+// descendants. <script>/<style>/<noscript> subtrees are skipped. Whitespace+// is folded per CSS rules (so "<h1>Hello world</h1>" becomes "Hello world"+// and "<br>" survives as a real line break), while <pre>/<textarea> keep+// their source formatting verbatim.func htmlLeafText(n *html.Node) string {- var b strings.Builder- walkHTMLLeaf(n, &b)+ var b bytes.Buffer+ w := &leafWriter{b: &b}+ walkHTMLLeaf(n, w)return b.String()}-func walkHTMLLeaf(n *html.Node, b *strings.Builder) {+func walkHTMLLeaf(n *html.Node, w *leafWriter) {switch n.Type {case html.TextNode:- b.WriteString(n.Data)+ w.writeText(n.Data)case html.ElementNode:if n.Data == "script" || n.Data == "style" || n.Data == "noscript" {return}- // Add a line break between block children so headings,- // paragraphs, and list items don't run together.- switch n.Data {- case "h1", "h2", "h3", "h4", "h5", "h6", "p", "li", "pre",- "tr", "blockquote":- if b.Len() > 0 && !strings.HasSuffix(b.String(), "\n") {- b.WriteString("\n")+ if n.Data == "br" {+ w.hardBreak()+ return+ }+ if n.Data == "pre" || n.Data == "textarea" {+ // Verbatim: no folding, no injected block breaks.+ w.pre = true+ for child := n.FirstChild; child != nil; child = child.NextSibling {+ walkHTMLLeaf(child, w)+ }+ w.pre = false+ return+ }+ // Add a line break between block children so headings, paragraphs,+ // and list items don't run together.+ if !w.pre {+ switch n.Data {+ case "h1", "h2", "h3", "h4", "h5", "h6", "p", "li", "pre",+ "tr", "blockquote":+ if w.b.Len() > 0 && !w.endsNL {+ w.hardBreak()+ }}}for child := n.FirstChild; child != nil; child = child.NextSibling {- walkHTMLLeaf(child, b)+ walkHTMLLeaf(child, w)}- if isBlockTag(n.Data) && b.Len() > 0 && !strings.HasSuffix(b.String(), "\n") {- b.WriteString("\n")+ if !w.pre && isBlockTag(n.Data) && w.b.Len() > 0 && !w.endsNL {+ w.hardBreak()}}}diff --git a/internal/parser/parser/html_parser_unified_test.go b/internal/parser/parser/html_parser_unified_test.gonew file mode 100644index 0000000000..82031e31d2--- /dev/null+++ b/internal/parser/parser/html_parser_unified_test.go@@ -0,0 +1,114 @@+// Copyright 2025 The InfiniFlow Authors. All Rights Reserved.+//+// Licensed under the Apache License, Version 2.0 (the "License");+// you may not use this file except in compliance with the License.+// You may obtain a copy of the License at+//+// http://www.apache.org/licenses/LICENSE-2.0+//+// Unless required by applicable law or agreed to in writing, software+// distributed under the License is distributed on an "AS IS" BASIS,+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+// See the License for the specific language governing permissions and+// limitations under the License.++package parser++import (+ _ "embed"+ "encoding/json"+ "testing"+)++// unifiedHTMLCasesJSON is the single source of truth for the browser-faithful+// HTML parsing semantics that BOTH the Go and Python parsers must converge on.+// It is embedded from testdata/unified_html_cases.json, which the Python mirror+// (test/unit_test/deepdoc/parser/test_html_parser.py) also loads — so the two+// engines share one fixture and can no longer drift.+//+// Each case wraps its content in a single block element so that+// ParseWithResult emits exactly one item and the Python merge_block_text emits+// exactly one block string, enabling a 1:1 byte comparison between engines.+//+//go:embed testdata/unified_html_cases.json+var unifiedHTMLCasesJSON []byte++type unifiedHTMLCase struct {+ Name string `json:"name"`+ HTML string `json:"html"`+ Want string `json:"want"`+}++func loadUnifiedHTMLCases(t *testing.T) []unifiedHTMLCase {+ t.Helper()+ var cases []unifiedHTMLCase+ if err := json.Unmarshal(unifiedHTMLCasesJSON, &cases); err != nil {+ t.Fatalf("unmarshal unified html cases: %v", err)+ }+ return cases+}++// TestHTMLParser_ParseWithResult_UnifiedSemantics asserts the browser-faithful+// semantics on the Go engine. The cases are loaded from the shared embedded+// fixture, so this test and its Python mirror stay in lockstep.+func TestHTMLParser_ParseWithResult_UnifiedSemantics(t *testing.T) {+ for _, tc := range loadUnifiedHTMLCases(t) {+ t.Run(tc.Name, func(t *testing.T) {+ p := NewHTMLParser()+ res := p.ParseWithResult(t.Context(), "doc.html", []byte(tc.HTML))+ if res.Err != nil {+ t.Fatalf("ParseWithResult: %v", res.Err)+ }+ if len(res.JSON) != 1 {+ t.Fatalf("block count = %d, want 1: %#v", len(res.JSON), res.JSON)+ }+ if got := res.JSON[0]["text"].(string); got != tc.Want {+ t.Errorf("got %q, want %q", got, tc.Want)+ }+ })+ }+}++// TestHTMLParser_ParseWithResult_RealisticSmoke exercises the Go HTML walker on+// a realistic multi-block document: a heading, a paragraph with an inline+// <b> and a <br> line break, a CJK paragraph with an inline element, and a+// verbatim <pre> block. It guards the leafWriter CSS-folding rewrite against+// hidden regressions specific to the Go reimplementation:+// - <br> becomes a hard line break;+// - inline boundaries join verbatim, with NO inserted space even for CJK;+// - block-internal whitespace collapses to a single space and is trimmed;+// - <pre> keeps its source whitespace verbatim (leading/trailing included).+func TestHTMLParser_ParseWithResult_RealisticSmoke(t *testing.T) {+ const html = `<h1>产品说明 Product Guide</h1>+<p>第一步:打开应用<br>第二步:点击<b>设置</b>按钮完成配置。</p>+<p>欢迎使用我们的<b>智能助手</b>,它能帮您快速处理任务。</p>+<pre> code+ block</pre>`+ p := NewHTMLParser()+ res := p.ParseWithResult(t.Context(), "doc.html", []byte(html))+ if res.Err != nil {+ t.Fatalf("ParseWithResult: %v", res.Err)+ }+ var texts []string+ for _, item := range res.JSON {+ texts = append(texts, item["text"].(string))+ }+ want := []string{+ // Heading: whitespace folded, no injected break.+ "产品说明 Product Guide",+ // <br> => hard break; inline <b> joined verbatim (无空格).+ "第一步:打开应用\n第二步:点击设置按钮完成配置。",+ // CJK inline joined verbatim: 我们的 + 智能助手, no space.+ "欢迎使用我们的智能助手,它能帮您快速处理任务。",+ // <pre> preserved verbatim, leading/trailing whitespace intact.+ " code\n block",+ }+ if len(texts) != len(want) {+ t.Fatalf("block count = %d, want %d; got %#v", len(texts), len(want), texts)+ }+ for i := range want {+ if texts[i] != want[i] {+ t.Errorf("block %d: got %q, want %q", i, texts[i], want[i])+ }+ }+}diff --git a/internal/parser/parser/testdata/unified_html_cases.json b/internal/parser/parser/testdata/unified_html_cases.jsonnew file mode 100644index 0000000000..5898a32d97--- /dev/null+++ b/internal/parser/parser/testdata/unified_html_cases.json@@ -0,0 +1,12 @@+[+ {"name": "br_basic", "html": "<p>line1<br>line2</p>", "want": "line1\nline2"},+ {"name": "br_surrounding_space", "html": "<p>Hello <br> World</p>", "want": "Hello\nWorld"},+ {"name": "br_double", "html": "<p>A<br><br>B</p>", "want": "A\n\nB"},+ {"name": "br_before_inline", "html": "<p>Line1<br><span>Line2</span></p>", "want": "Line1\nLine2"},+ {"name": "inline_no_space_latin", "html": "<p>Hello<b>World</b></p>", "want": "HelloWorld"},+ {"name": "inline_no_space_cjk", "html": "<p>你好<b>世界</b></p>", "want": "你好世界"},+ {"name": "inline_with_space", "html": "<p>Hello <b>World</b></p>", "want": "Hello World"},+ {"name": "inline_three", "html": "<p>First<b>Second</b>Third</p>", "want": "FirstSecondThird"},+ {"name": "whitespace_collapse", "html": "<p>\n Hello\n <b>World</b>\n</p>", "want": "Hello World"},+ {"name": "pre_preserved", "html": "<pre> code\n block</pre>", "want": " code\n block"}+]diff --git a/test/unit_test/deepdoc/parser/test_html_parser.py b/test/unit_test/deepdoc/parser/test_html_parser.pyindex 02f36bed3f..f50a39d4db 100644--- a/test/unit_test/deepdoc/parser/test_html_parser.py+++ b/test/unit_test/deepdoc/parser/test_html_parser.py@@ -23,10 +23,15 @@ scripts that have no whitespace word boundaries (e.g. Chinese)."""import importlib.util+import jsonimport osimport sys+from pathlib import Pathfrom unittest import mock+import pytest+from bs4 import BeautifulSoup+# Load html_parser by file path so we don't trigger deepdoc/parser/__init__.py# (which pulls in heavy parsers) or the real rag.nlp tokenizer. The heavy# optional modules are stubbed; rag.nlp is stubbed so the module imports, and@@ -170,3 +175,59 @@ def test_parser_txt_keeps_loose_text_between_and_after_blocks():assert "loose" in betweentrailing = "\n".join(RAGFlowHtmlParser.parser_txt("<p>Only.</p>tail", chunk_token_num=512))assert "tail" in trailing+++# Unified HTML semantics: the browser-faithful rules that BOTH the Python and+# Go parsers must converge on. The cases are the single source of truth shared+# with the Go engine, loaded from the JSON fixture below (so the two engines+# can no longer drift). They must all currently PASS on both engines.+#+# The fixture lives in the Go package's testdata because //go:embed requires+# the file to sit inside the package directory tree; the Python mirror reads+# the same file by absolute repo-root path.+_REPO_ROOT = Path(__file__).resolve().parents[4]+_UNIFIED_HTML_FIXTURE = _REPO_ROOT / "internal/parser/parser/testdata/unified_html_cases.json"+_UNIFIED_HTML_CASES = [(c["name"], c["html"], c["want"]) for c in json.loads(_UNIFIED_HTML_FIXTURE.read_text())]+++def _merge_one_block(html):+ # Use read_text_recursively + merge_block_text (not parser_txt) so block+ # boundaries survive as separate list entries and no "#" heading prefix or+ # chunk-merge join is applied — enabling a 1:1 comparison with the Go+ # per-block output.+ soup = BeautifulSoup(html, "html.parser")+ temp = []+ RAGFlowHtmlParser.read_text_recursively(soup, temp)+ blocks, _ = RAGFlowHtmlParser.merge_block_text(temp)+ return blocks++[email protected]("name,html,want", _UNIFIED_HTML_CASES)+def test_unified_html_semantics_parity(name, html, want):+ assert _merge_one_block(html) == [want]+++def test_merge_block_text_loose_text_newline_join():+ # The block_id is None branch (loose text, e.g. text directly under+ # <body>) must join fragments with hard breaks, not silently+ # concatenate. Regression guard for the inline-verbatim / <br> rewrite in+ # merge_block_text (this branch is not exercised by the block_id cases+ # above, which all carry a block_id).+ # Single hard break between two loose fragments.+ blocks, _ = RAGFlowHtmlParser.merge_block_text(+ [+ {"content": "Loose one", "tag_name": "inner_text", "metadata": {}},+ {"content": "Loose two", "tag_name": "inner_text", "metadata": {"newline_before": 1}},+ ]+ )+ assert blocks == ["Loose one\nLoose two"]++ # Consecutive breaks between loose fragments are preserved (1:1 with the+ # <br><br> rule for blocks).+ blocks2, _ = RAGFlowHtmlParser.merge_block_text(+ [+ {"content": "A", "tag_name": "inner_text", "metadata": {}},+ {"content": "B", "tag_name": "inner_text", "metadata": {"newline_before": 2}},+ ]+ )+ assert blocks2 == ["A\n\nB"]