Skip to content

RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.

Fix(parser): keep real line breaks when merging HTML fragments and PDF boxes (#17856)

Jackf41f866aa1ec
diff --git a/deepdoc/parser/html_parser.py b/deepdoc/parser/html_parser.py
index b593ad849a..1f4b3a1a60 100644
--- a/deepdoc/parser/html_parser.py
+++ b/deepdoc/parser/html_parser.py
@@ -105,27 +105,31 @@ class RAGFlowHtmlParser:
return table_str_list
@classmethod
- def read_text_recursively(cls, element, parser_result, chunk_token_num=512, parent_name=None, block_id=None):
+ def read_text_recursively(cls, element, parser_result, chunk_token_num=512, parent_name=None, block_id=None, newline_before=0):
if isinstance(element, NavigableString):
- content = element.strip()
+ # Keep the text verbatim (do not strip): source whitespace is
+ # folded later by merge_block_text, which preserves <br> breaks.
+ content = str(element)
- def is_valid_html(content):
+ def is_valid_html(text):
try:
- soup = BeautifulSoup(content, "html.parser")
+ soup = BeautifulSoup(text, "html.parser")
return bool(soup.find())
except Exception:
return False
return_info = []
- if content:
+ if content.strip():
if is_valid_html(content):
soup = BeautifulSoup(content, "html.parser")
- child_info = cls.read_text_recursively(soup, parser_result, chunk_token_num, element.name, block_id)
+ child_info = cls.read_text_recursively(soup, parser_result, chunk_token_num, element.name, block_id, newline_before)
parser_result.extend(child_info)
else:
- info = {"content": element.strip(), "tag_name": "inner_text", "metadata": {"block_id": block_id}}
+ info = {"content": content, "tag_name": "inner_text", "metadata": {"block_id": block_id}}
if parent_name:
info["tag_name"] = parent_name
+ if newline_before:
+ info["metadata"]["newline_before"] = newline_before
return_info.append(info)
return return_info
elif isinstance(element, Tag):
@@ -139,22 +143,55 @@ class RAGFlowHtmlParser:
else:
if str.lower(element.name) in BLOCK_TAGS:
block_id = str(uuid.uuid1())
+ # A <br> is a hard line break. Thread a pending break count
+ # through so it survives nesting (e.g. "<p>A<br><span>B</span></p>")
+ # and repeated breaks ("<br><br>"). The count resets only after
+ # a descendant emits text, mirroring the Go walker.
+ pending_newlines = int(newline_before)
for child in element.children:
- child_info = cls.read_text_recursively(child, parser_result, chunk_token_num, element.name, block_id)
+ if isinstance(child, Tag) and str.lower(child.name) == "br":
+ pending_newlines += 1
+ continue
+ result_count = len(parser_result)
+ child_info = cls.read_text_recursively(child, parser_result, chunk_token_num, element.name, block_id, pending_newlines)
parser_result.extend(child_info)
+ if child_info or len(parser_result) > result_count:
+ pending_newlines = 0
return []
+ # Hard line-break sentinel used between fragments so CSS whitespace
+ # folding (which collapses real whitespace runs) does not eat explicit
+ # <br> breaks. Replaced with "\n" after folding.
+ _HARD_BREAK = "\x0b"
+ _WS_RE = re.compile(r"[ \t\r\n\f]+")
+ _PRE_TAGS = ("pre", "textarea")
+
+ @classmethod
+ def _fold_block(cls, raw, preserve):
+ if preserve:
+ return raw.replace(cls._HARD_BREAK, "\n")
+ # Collapse each line between hard breaks: runs of collapsible
+ # whitespace become one space, and line edges are trimmed. This
+ # matches the Go walker's CSS whitespace folding.
+ lines = raw.split(cls._HARD_BREAK)
+ folded = [cls._WS_RE.sub(" ", ln).strip() for ln in lines]
+ return "\n".join(folded)
+
@classmethod
def merge_block_text(cls, parser_result):
block_content = []
current_content = ""
+ current_is_pre = False
table_info_list = []
last_block_id = None
for item in parser_result:
- content = item.get("content")
+ content = item.get("content") or ""
tag_name = item.get("tag_name")
title_flag = tag_name in TITLE_TAGS
block_id = item.get("metadata", {}).get("block_id")
+ # newline_before is a break count: 0 -> join verbatim (no
+ # separator), >0 -> insert that many hard breaks (one per <br>).
+ newline_before = item.get("metadata", {}).get("newline_before", 0)
if block_id:
if title_flag:
content = f"{TITLE_TAGS[tag_name]} {content}"
@@ -163,18 +200,21 @@ class RAGFlowHtmlParser:
# precedes the first block (last_block_id is None there),
# which was otherwise overwritten and lost.
if current_content:
- block_content.append(current_content)
+ block_content.append(cls._fold_block(current_content, current_is_pre))
current_content = content
+ current_is_pre = tag_name in cls._PRE_TAGS
last_block_id = block_id
else:
- current_content += (" " if current_content else "") + content
+ sep = cls._HARD_BREAK * newline_before
+ current_content += sep + content
else:
if tag_name == "table":
table_info_list.append(item)
else:
- current_content += (" " if current_content else "") + content
+ sep = cls._HARD_BREAK * newline_before
+ current_content += sep + content
if current_content:
- block_content.append(current_content)
+ block_content.append(cls._fold_block(current_content, current_is_pre))
return block_content, table_info_list
# Characters from scripts written without spaces between words (CJK, kana,
diff --git a/internal/parser/parser/html_parser.go b/internal/parser/parser/html_parser.go
index 13d9ab28f3..313f080843 100644
--- a/internal/parser/parser/html_parser.go
+++ b/internal/parser/parser/html_parser.go
@@ -110,7 +110,7 @@ func walkHTMLBlocks(root *html.Node, out *[]map[string]any) {
for child := root.FirstChild; child != nil; child = child.NextSibling {
if child.Type == html.TextNode {
if emitsLooseHTMLText(root) {
- appendHTMLTextItem(out, child.Data, "text")
+ appendHTMLTextItem(out, child.Data, "text", true)
}
continue
}
@@ -131,7 +131,7 @@ func walkHTMLBlocks(root *html.Node, out *[]map[string]any) {
continue
}
text := htmlLeafText(child)
- appendHTMLTextItem(out, text, htmlTagToCkType(tag))
+ appendHTMLTextItem(out, text, htmlTagToCkType(tag), tag != "pre" && tag != "textarea")
}
}
@@ -139,8 +139,10 @@ func emitsLooseHTMLText(root *html.Node) bool {
return root.Type == html.ElementNode && root.Data == "body"
}
-func appendHTMLTextItem(out *[]map[string]any, text, ckType string) {
- text = strings.TrimSpace(text)
+func appendHTMLTextItem(out *[]map[string]any, text, ckType string, trim bool) {
+ if trim {
+ text = strings.TrimSpace(text)
+ }
if text == "" {
return
}
@@ -174,38 +176,116 @@ func htmlTagToCkType(tag string) string {
return "text"
}
+// leafWriter accumulates the visible text of an HTML subtree while applying
+// CSS whitespace folding (the default white-space: normal rules):
+// - collapsible whitespace runs collapse to a single space;
+// - leading/trailing whitespace of a line is dropped;
+// - a <br> forces a hard line break (and resets the leading-whitespace state);
+// - <pre>/<textarea> are emitted verbatim (no folding, no injected breaks).
+type leafWriter struct {
+ b *bytes.Buffer
+ lastSpace bool // last written rune was a collapsed single space
+ lineStart bool // at the start of a line, so leading whitespace is dropped
+ endsNL bool // builder currently ends with a hard line break
+ pre bool // inside <pre>/<textarea>: emit verbatim
+}
+
+func isCollapsibleWS(r rune) bool {
+ return r == ' ' || r == '\t' || r == '\n' || r == '\r' || r == '\f'
+}
+
+// writeText appends s, folding collapsible whitespace unless in pre mode.
+func (w *leafWriter) writeText(s string) {
+ if w.pre {
+ for _, r := range s {
+ w.b.WriteRune(r)
+ w.endsNL = r == '\n'
+ }
+ w.lastSpace = false
+ w.lineStart = false
+ return
+ }
+ for _, r := range s {
+ if isCollapsibleWS(r) {
+ if w.lineStart || w.lastSpace {
+ continue
+ }
+ w.b.WriteRune(' ')
+ w.lastSpace = true
+ w.lineStart = false
+ w.endsNL = false
+ continue
+ }
+ w.b.WriteRune(r)
+ w.lastSpace = false
+ w.lineStart = false
+ w.endsNL = false
+ }
+}
+
+// hardBreak inserts a forced line break (a <br> or block boundary). Per CSS,
+// whitespace immediately before a break is dropped (so "Hello <br>" yields
+// "Hello\n", not "Hello \n"). Inside <pre>/<textarea> whitespace is preserved,
+// so the preceding space is kept.
+func (w *leafWriter) hardBreak() {
+ if !w.pre && w.lastSpace && w.b.Len() > 0 {
+ w.b.Truncate(w.b.Len() - 1)
+ }
+ w.b.WriteByte('\n')
+ w.lastSpace = false
+ w.lineStart = true
+ w.endsNL = true
+}
+
// htmlLeafText joins the visible text of an HTML node and its
-// descendants. <script>/<style>/<noscript> subtrees are skipped.
-// The output preserves whitespace runs so headings like
-// "<h1>Hello world</h1>" round-trip with their spacing intact.
+// descendants. <script>/<style>/<noscript> subtrees are skipped. Whitespace
+// is folded per CSS rules (so "<h1>Hello world</h1>" becomes "Hello world"
+// and "<br>" survives as a real line break), while <pre>/<textarea> keep
+// their source formatting verbatim.
func htmlLeafText(n *html.Node) string {
- var b strings.Builder
- walkHTMLLeaf(n, &b)
+ var b bytes.Buffer
+ w := &leafWriter{b: &b}
+ walkHTMLLeaf(n, w)
return b.String()
}
-func walkHTMLLeaf(n *html.Node, b *strings.Builder) {
+func walkHTMLLeaf(n *html.Node, w *leafWriter) {
switch n.Type {
case html.TextNode:
- b.WriteString(n.Data)
+ w.writeText(n.Data)
case html.ElementNode:
if n.Data == "script" || n.Data == "style" || n.Data == "noscript" {
return
}
- // Add a line break between block children so headings,
- // paragraphs, and list items don't run together.
- switch n.Data {
- case "h1", "h2", "h3", "h4", "h5", "h6", "p", "li", "pre",
- "tr", "blockquote":
- if b.Len() > 0 && !strings.HasSuffix(b.String(), "\n") {
- b.WriteString("\n")
+ if n.Data == "br" {
+ w.hardBreak()
+ return
+ }
+ if n.Data == "pre" || n.Data == "textarea" {
+ // Verbatim: no folding, no injected block breaks.
+ w.pre = true
+ for child := n.FirstChild; child != nil; child = child.NextSibling {
+ walkHTMLLeaf(child, w)
+ }
+ w.pre = false
+ return
+ }
+ // Add a line break between block children so headings, paragraphs,
+ // and list items don't run together.
+ if !w.pre {
+ switch n.Data {
+ case "h1", "h2", "h3", "h4", "h5", "h6", "p", "li", "pre",
+ "tr", "blockquote":
+ if w.b.Len() > 0 && !w.endsNL {
+ w.hardBreak()
+ }
}
}
for child := n.FirstChild; child != nil; child = child.NextSibling {
- walkHTMLLeaf(child, b)
+ walkHTMLLeaf(child, w)
}
- if isBlockTag(n.Data) && b.Len() > 0 && !strings.HasSuffix(b.String(), "\n") {
- b.WriteString("\n")
+ if !w.pre && isBlockTag(n.Data) && w.b.Len() > 0 && !w.endsNL {
+ w.hardBreak()
}
}
}
diff --git a/internal/parser/parser/html_parser_unified_test.go b/internal/parser/parser/html_parser_unified_test.go
new file mode 100644
index 0000000000..82031e31d2
--- /dev/null
+++ b/internal/parser/parser/html_parser_unified_test.go
@@ -0,0 +1,114 @@
+// Copyright 2025 The InfiniFlow Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package parser
+
+import (
+ _ "embed"
+ "encoding/json"
+ "testing"
+)
+
+// unifiedHTMLCasesJSON is the single source of truth for the browser-faithful
+// HTML parsing semantics that BOTH the Go and Python parsers must converge on.
+// It is embedded from testdata/unified_html_cases.json, which the Python mirror
+// (test/unit_test/deepdoc/parser/test_html_parser.py) also loads — so the two
+// engines share one fixture and can no longer drift.
+//
+// Each case wraps its content in a single block element so that
+// ParseWithResult emits exactly one item and the Python merge_block_text emits
+// exactly one block string, enabling a 1:1 byte comparison between engines.
+//
+//go:embed testdata/unified_html_cases.json
+var unifiedHTMLCasesJSON []byte
+
+type unifiedHTMLCase struct {
+ Name string `json:"name"`
+ HTML string `json:"html"`
+ Want string `json:"want"`
+}
+
+func loadUnifiedHTMLCases(t *testing.T) []unifiedHTMLCase {
+ t.Helper()
+ var cases []unifiedHTMLCase
+ if err := json.Unmarshal(unifiedHTMLCasesJSON, &cases); err != nil {
+ t.Fatalf("unmarshal unified html cases: %v", err)
+ }
+ return cases
+}
+
+// TestHTMLParser_ParseWithResult_UnifiedSemantics asserts the browser-faithful
+// semantics on the Go engine. The cases are loaded from the shared embedded
+// fixture, so this test and its Python mirror stay in lockstep.
+func TestHTMLParser_ParseWithResult_UnifiedSemantics(t *testing.T) {
+ for _, tc := range loadUnifiedHTMLCases(t) {
+ t.Run(tc.Name, func(t *testing.T) {
+ p := NewHTMLParser()
+ res := p.ParseWithResult(t.Context(), "doc.html", []byte(tc.HTML))
+ if res.Err != nil {
+ t.Fatalf("ParseWithResult: %v", res.Err)
+ }
+ if len(res.JSON) != 1 {
+ t.Fatalf("block count = %d, want 1: %#v", len(res.JSON), res.JSON)
+ }
+ if got := res.JSON[0]["text"].(string); got != tc.Want {
+ t.Errorf("got %q, want %q", got, tc.Want)
+ }
+ })
+ }
+}
+
+// TestHTMLParser_ParseWithResult_RealisticSmoke exercises the Go HTML walker on
+// a realistic multi-block document: a heading, a paragraph with an inline
+// <b> and a <br> line break, a CJK paragraph with an inline element, and a
+// verbatim <pre> block. It guards the leafWriter CSS-folding rewrite against
+// hidden regressions specific to the Go reimplementation:
+// - <br> becomes a hard line break;
+// - inline boundaries join verbatim, with NO inserted space even for CJK;
+// - block-internal whitespace collapses to a single space and is trimmed;
+// - <pre> keeps its source whitespace verbatim (leading/trailing included).
+func TestHTMLParser_ParseWithResult_RealisticSmoke(t *testing.T) {
+ const html = `<h1>产品说明 Product Guide</h1>
+<p>第一步:打开应用<br>第二步:点击<b>设置</b>按钮完成配置。</p>
+<p>欢迎使用我们的<b>智能助手</b>,它能帮您快速处理任务。</p>
+<pre> code
+ block</pre>`
+ p := NewHTMLParser()
+ res := p.ParseWithResult(t.Context(), "doc.html", []byte(html))
+ if res.Err != nil {
+ t.Fatalf("ParseWithResult: %v", res.Err)
+ }
+ var texts []string
+ for _, item := range res.JSON {
+ texts = append(texts, item["text"].(string))
+ }
+ want := []string{
+ // Heading: whitespace folded, no injected break.
+ "产品说明 Product Guide",
+ // <br> => hard break; inline <b> joined verbatim (无空格).
+ "第一步:打开应用\n第二步:点击设置按钮完成配置。",
+ // CJK inline joined verbatim: 我们的 + 智能助手, no space.
+ "欢迎使用我们的智能助手,它能帮您快速处理任务。",
+ // <pre> preserved verbatim, leading/trailing whitespace intact.
+ " code\n block",
+ }
+ if len(texts) != len(want) {
+ t.Fatalf("block count = %d, want %d; got %#v", len(texts), len(want), texts)
+ }
+ for i := range want {
+ if texts[i] != want[i] {
+ t.Errorf("block %d: got %q, want %q", i, texts[i], want[i])
+ }
+ }
+}
diff --git a/internal/parser/parser/testdata/unified_html_cases.json b/internal/parser/parser/testdata/unified_html_cases.json
new file mode 100644
index 0000000000..5898a32d97
--- /dev/null
+++ b/internal/parser/parser/testdata/unified_html_cases.json
@@ -0,0 +1,12 @@
+[
+ {"name": "br_basic", "html": "<p>line1<br>line2</p>", "want": "line1\nline2"},
+ {"name": "br_surrounding_space", "html": "<p>Hello <br> World</p>", "want": "Hello\nWorld"},
+ {"name": "br_double", "html": "<p>A<br><br>B</p>", "want": "A\n\nB"},
+ {"name": "br_before_inline", "html": "<p>Line1<br><span>Line2</span></p>", "want": "Line1\nLine2"},
+ {"name": "inline_no_space_latin", "html": "<p>Hello<b>World</b></p>", "want": "HelloWorld"},
+ {"name": "inline_no_space_cjk", "html": "<p>你好<b>世界</b></p>", "want": "你好世界"},
+ {"name": "inline_with_space", "html": "<p>Hello <b>World</b></p>", "want": "Hello World"},
+ {"name": "inline_three", "html": "<p>First<b>Second</b>Third</p>", "want": "FirstSecondThird"},
+ {"name": "whitespace_collapse", "html": "<p>\n Hello\n <b>World</b>\n</p>", "want": "Hello World"},
+ {"name": "pre_preserved", "html": "<pre> code\n block</pre>", "want": " code\n block"}
+]
diff --git a/test/unit_test/deepdoc/parser/test_html_parser.py b/test/unit_test/deepdoc/parser/test_html_parser.py
index 02f36bed3f..f50a39d4db 100644
--- a/test/unit_test/deepdoc/parser/test_html_parser.py
+++ b/test/unit_test/deepdoc/parser/test_html_parser.py
@@ -23,10 +23,15 @@ scripts that have no whitespace word boundaries (e.g. Chinese).
"""
import importlib.util
+import json
import os
import sys
+from pathlib import Path
from unittest import mock
+import pytest
+from bs4 import BeautifulSoup
+
# Load html_parser by file path so we don't trigger deepdoc/parser/__init__.py
# (which pulls in heavy parsers) or the real rag.nlp tokenizer. The heavy
# optional modules are stubbed; rag.nlp is stubbed so the module imports, and
@@ -170,3 +175,59 @@ def test_parser_txt_keeps_loose_text_between_and_after_blocks():
assert "loose" in between
trailing = "\n".join(RAGFlowHtmlParser.parser_txt("<p>Only.</p>tail", chunk_token_num=512))
assert "tail" in trailing
+
+
+# Unified HTML semantics: the browser-faithful rules that BOTH the Python and
+# Go parsers must converge on. The cases are the single source of truth shared
+# with the Go engine, loaded from the JSON fixture below (so the two engines
+# can no longer drift). They must all currently PASS on both engines.
+#
+# The fixture lives in the Go package's testdata because //go:embed requires
+# the file to sit inside the package directory tree; the Python mirror reads
+# the same file by absolute repo-root path.
+_REPO_ROOT = Path(__file__).resolve().parents[4]
+_UNIFIED_HTML_FIXTURE = _REPO_ROOT / "internal/parser/parser/testdata/unified_html_cases.json"
+_UNIFIED_HTML_CASES = [(c["name"], c["html"], c["want"]) for c in json.loads(_UNIFIED_HTML_FIXTURE.read_text())]
+
+
+def _merge_one_block(html):
+ # Use read_text_recursively + merge_block_text (not parser_txt) so block
+ # boundaries survive as separate list entries and no "#" heading prefix or
+ # chunk-merge join is applied — enabling a 1:1 comparison with the Go
+ # per-block output.
+ soup = BeautifulSoup(html, "html.parser")
+ temp = []
+ RAGFlowHtmlParser.read_text_recursively(soup, temp)
+ blocks, _ = RAGFlowHtmlParser.merge_block_text(temp)
+ return blocks
+
+
[email protected]("name,html,want", _UNIFIED_HTML_CASES)
+def test_unified_html_semantics_parity(name, html, want):
+ assert _merge_one_block(html) == [want]
+
+
+def test_merge_block_text_loose_text_newline_join():
+ # The block_id is None branch (loose text, e.g. text directly under
+ # <body>) must join fragments with hard breaks, not silently
+ # concatenate. Regression guard for the inline-verbatim / <br> rewrite in
+ # merge_block_text (this branch is not exercised by the block_id cases
+ # above, which all carry a block_id).
+ # Single hard break between two loose fragments.
+ blocks, _ = RAGFlowHtmlParser.merge_block_text(
+ [
+ {"content": "Loose one", "tag_name": "inner_text", "metadata": {}},
+ {"content": "Loose two", "tag_name": "inner_text", "metadata": {"newline_before": 1}},
+ ]
+ )
+ assert blocks == ["Loose one\nLoose two"]
+
+ # Consecutive breaks between loose fragments are preserved (1:1 with the
+ # <br><br> rule for blocks).
+ blocks2, _ = RAGFlowHtmlParser.merge_block_text(
+ [
+ {"content": "A", "tag_name": "inner_text", "metadata": {}},
+ {"content": "B", "tag_name": "inner_text", "metadata": {"newline_before": 2}},
+ ]
+ )
+ assert blocks2 == ["A\n\nB"]