RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.
Go agent-harnessagent-native-databaseagentic-aiagentic-retrievalagentic-searchaiai-agentscontext-enginecontext-engineeringcontext-managementharness-engineeringknowledge-compilationragretrieval-augmented-generationretrieval-harnesssearch-harness
refactor(parser): preserve HTML <table> structure (inline + structured item) (#18129)
Jack875ca966e957
diff --git a/internal/parser/parser/html_parser.go b/internal/parser/parser/html_parser.goindex 313f080843..9feb8438b6 100644--- a/internal/parser/parser/html_parser.go+++ b/internal/parser/parser/html_parser.go@@ -107,6 +107,10 @@ func (p *HTMLParser) ParseWithResult(ctx context.Context, filename string, data// <style>, and <noscript> blocks are skipped entirely so they// don't pollute the downstream chunker input.func walkHTMLBlocks(root *html.Node, out *[]map[string]any) {+ // tableItems collects the structured doc_type_kwd:"table" items so they+ // can be appended after the walk (mirrors markdown_parser.go:366-367,+ // which appends tables after all sections to match Python's ordering).+ var tableItems []map[string]anyfor child := root.FirstChild; child != nil; child = child.NextSibling {if child.Type == html.TextNode {if emitsLooseHTMLText(root) {@@ -129,10 +133,53 @@ func walkHTMLBlocks(root *html.Node, out *[]map[string]any) {// Wrapper elements: descend into their children.walkHTMLBlocks(child, out)continue+ case "table":+ // Keep the table as its full HTML markup (NOT flattened) so+ // row/column structure survives into embedding, retrieval, and+ // LLM rendering. This mirrors the markdown table handling+ // (markdown_parser.go:305-328) and Python's HtmlParser, which+ // keeps the <table>…</table> string as the section text.+ markup := renderTableHTML(child)+ if strings.TrimSpace(markup) != "" {+ // Inlined copy in document order (matches Python's <table>+ // section text and markdown's text-flow copy).+ *out = append(*out, map[string]any{+ "text": markup,+ "doc_type_kwd": "text",+ })+ // Structured table item, appended after the walk.+ tableItems = append(tableItems, map[string]any{+ "text": markup,+ "doc_type_kwd": "table",+ "ck_type": "table",+ })+ }+ continue}- text := htmlLeafText(child)+ text := htmlLeafText(child, &tableItems)appendHTMLTextItem(out, text, htmlTagToCkType(tag), tag != "pre" && tag != "textarea")}+ if len(tableItems) > 0 {+ *out = append(*out, tableItems...)+ }+}++// renderTableHTML serializes a <table> node back to its outer HTML markup+// (tags preserved), mirroring Python's HtmlParser which keeps the full+// <table>…</table> string as the section text. This preserves row/column+// structure for embedding, retrieval, and LLM rendering, instead of+// flattening cells into a single text blob. It is used both for top-level+// tables (walkHTMLBlocks) and for tables reached via the leaf-text extractor+// (walkHTMLLeaf, i.e. a <table> nested in a div/section/…). On any rendering+// error it returns "" so callers skip the table rather than risk a render+// loop through the leaf extractor — html.Render only fails on unsupported+// node kinds, and a parsed <table> never triggers it.+func renderTableHTML(n *html.Node) string {+ var b bytes.Buffer+ if err := html.Render(&b, n); err != nil {+ return ""+ }+ return b.String()}func emitsLooseHTMLText(root *html.Node) bool {@@ -241,15 +288,18 @@ func (w *leafWriter) hardBreak() {// descendants. <script>/<style>/<noscript> subtrees are skipped. Whitespace// is folded per CSS rules (so "<h1>Hello world</h1>" becomes "Hello world"// and "<br>" survives as a real line break), while <pre>/<textarea> keep-// their source formatting verbatim.-func htmlLeafText(n *html.Node) string {+// their source formatting verbatim. Any <table> encountered in the subtree is+// rendered as its <table>…</table> markup (so row/column structure survives)+// and registered as a structured doc_type_kwd:"table" item via tableItems —+// see walkHTMLLeaf's "table" case.+func htmlLeafText(n *html.Node, tableItems *[]map[string]any) string {var b bytes.Bufferw := &leafWriter{b: &b}- walkHTMLLeaf(n, w)+ walkHTMLLeaf(n, w, tableItems)return b.String()}-func walkHTMLLeaf(n *html.Node, w *leafWriter) {+func walkHTMLLeaf(n *html.Node, w *leafWriter, tableItems *[]map[string]any) {switch n.Type {case html.TextNode:w.writeText(n.Data)@@ -265,11 +315,31 @@ func walkHTMLLeaf(n *html.Node, w *leafWriter) {// Verbatim: no folding, no injected block breaks.w.pre = truefor child := n.FirstChild; child != nil; child = child.NextSibling {- walkHTMLLeaf(child, w)+ walkHTMLLeaf(child, w, tableItems)}w.pre = falsereturn}+ if n.Data == "table" {+ // Keep the table as its full HTML markup (NOT flattened) so+ // row/column structure survives into the wrapper's text item,+ // embedding, retrieval, and LLM rendering. This mirrors the+ // top-level walkHTMLBlocks "table" case, and covers tables nested+ // in div/section/article/… (which the walkHTMLBlocks case never+ // reaches). The structured table item is collected for the+ // downstream chunker (appended after the walk by the caller, same+ // as the top-level path).+ markup := renderTableHTML(n)+ if strings.TrimSpace(markup) != "" {+ w.writeText(markup)+ *tableItems = append(*tableItems, map[string]any{+ "text": markup,+ "doc_type_kwd": "table",+ "ck_type": "table",+ })+ }+ return+ }// Add a line break between block children so headings, paragraphs,// and list items don't run together.if !w.pre {@@ -282,7 +352,7 @@ func walkHTMLLeaf(n *html.Node, w *leafWriter) {}}for child := n.FirstChild; child != nil; child = child.NextSibling {- walkHTMLLeaf(child, w)+ walkHTMLLeaf(child, w, tableItems)}if !w.pre && isBlockTag(n.Data) && w.b.Len() > 0 && !w.endsNL {w.hardBreak()diff --git a/internal/parser/parser/html_parser_align_test.go b/internal/parser/parser/html_parser_align_test.gonew file mode 100644index 0000000000..9c7eb38c35--- /dev/null+++ b/internal/parser/parser/html_parser_align_test.go@@ -0,0 +1,380 @@+package parser++import (+ "context"+ "os"+ "strings"+ "testing"+)++// TestHTMLParser_TableProducesStructuredItems asserts the HTML table+// alignment fix (approach a, mirroring markdown_parser.go:305-328 and+// :366-367): a <table> must NOT be flattened into a single text blob. The+// parser must emit two items:+//+// 1. an inlined copy in the text flow (doc_type_kwd:"text") whose text keeps+// the <table> markup, so embedding/retrieval/LLM rendering preserve the+// row/column structure (no "collapse");+// 2. a structured table item (doc_type_kwd:"table", ck_type:"table") appended+// after the walk, consumed by the downstream chunker (attachMediaContext).+//+// Before the fix, walkHTMLBlocks ran htmlLeafText over the <table>, producing+// ONE flat item (e.g. "Name Age Alice 30") tagged ck_type:"table" but with no+// markup — this test fails on that behavior.+func TestHTMLParser_TableProducesStructuredItems(t *testing.T) {+ const html = `<html><body>+<h1>Employee Table</h1>+<table>+<tr><th>Name</th><th>Age</th></tr>+<tr><td>Alice</td><td>30</td></tr>+</table>+<p>Trailing paragraph</p>+</body></html>`++ p := NewHTMLParser()+ res := p.ParseWithResult(context.Background(), "doc.html", []byte(html))+ if res.Err != nil {+ t.Fatalf("ParseWithResult: %v", res.Err)+ }++ var inlinedText, structuredText string+ inlinedIdx, structuredIdx, trailingIdx := -1, -1, -1+ for i, it := range res.JSON {+ text, _ := it["text"].(string)+ switch it["doc_type_kwd"] {+ case "text":+ if strings.Contains(text, "<table") {+ inlinedText = text+ inlinedIdx = i+ }+ if text == "Trailing paragraph" {+ trailingIdx = i+ }+ case "table":+ structuredText = text+ structuredIdx = i+ }+ }++ if inlinedIdx < 0 {+ t.Fatalf("no inlined doc_type_kwd:\"text\" item contains <table> markup; got items: %#v", res.JSON)+ }+ if !strings.Contains(inlinedText, "<table") ||+ !strings.Contains(inlinedText, "Name") ||+ !strings.Contains(inlinedText, "Alice") {+ t.Errorf("inlined table copy missing markup/cells: %q", inlinedText)+ }++ if structuredIdx < 0 {+ t.Fatalf("no structured doc_type_kwd:\"table\" item emitted; got items: %#v", res.JSON)+ }+ if got, want := res.JSON[structuredIdx]["ck_type"], "table"; got != want {+ t.Errorf("structured table item ck_type = %v, want %v", got, want)+ }+ if structuredText != inlinedText {+ t.Errorf("structured table text differs from inlined copy:\n inlined=%q\n struct =%q", inlinedText, structuredText)+ }++ // The structured table item must be appended after the walk, i.e. after+ // the in-walk text items (mirrors markdown_parser.go:366-367). The trailing+ // <p> is an in-walk text item, so the structured item must come after it.+ if trailingIdx < 0 {+ t.Fatalf("trailing paragraph text item missing; got items: %#v", res.JSON)+ }+ if structuredIdx <= trailingIdx {+ t.Errorf("structured table item at index %d must come after the trailing paragraph at index %d", structuredIdx, trailingIdx)+ }+}++// TestHTMLParser_NestedTableProducesStructuredItems asserts the table fix also+// covers tables that are NOT direct children of <body> — i.e. a <table> wrapped+// in a layout container such as <div>/<section>/<article> (the common real-world+// case). walkHTMLBlocks only special-cases a <table> that is a direct child of+// the walk root; a nested <table> is reached via the default-branch leaf-text+// extractor (htmlLeafText → walkHTMLLeaf). Before the fix, that path ran+// htmlLeafText over the wrapper, flattening the table into a single text blob+// ("Name Age Alice 30") with no markup — the same collapse as the top-level case,+// just hidden one level deeper. After the fix, walkHTMLLeaf must render the+// <table> markup inline (so it survives inside the wrapper's single text item)+// AND register a structured doc_type_kwd:"table"/ck_type:"table" item (appended+// after the walk like the top-level path).+func TestHTMLParser_NestedTableProducesStructuredItems(t *testing.T) {+ const html = `<html><body>+<h1>Heading</h1>+<div class="content">+<table>+<tr><th>Name</th><th>Age</th></tr>+<tr><td>Alice</td><td>30</td></tr>+</table>+</div>+<p>Trailing paragraph</p>+</body></html>`++ p := NewHTMLParser()+ res := p.ParseWithResult(context.Background(), "nested.html", []byte(html))+ if res.Err != nil {+ t.Fatalf("ParseWithResult: %v", res.Err)+ }++ var inlinedText, structuredText string+ inlinedIdx, structuredIdx, trailingIdx := -1, -1, -1+ for i, it := range res.JSON {+ text, _ := it["text"].(string)+ switch it["doc_type_kwd"] {+ case "text":+ if strings.Contains(text, "<table") {+ inlinedText = text+ inlinedIdx = i+ }+ if text == "Trailing paragraph" {+ trailingIdx = i+ }+ case "table":+ structuredText = text+ structuredIdx = i+ }+ }++ if inlinedIdx < 0 {+ t.Fatalf("no inlined doc_type_kwd:\"text\" item contains <table> markup (nested table flattened?); got items: %#v", res.JSON)+ }+ if !strings.Contains(inlinedText, "<table") ||+ !strings.Contains(inlinedText, "Name") ||+ !strings.Contains(inlinedText, "Alice") {+ t.Errorf("inlined nested table copy missing markup/cells: %q", inlinedText)+ }++ if structuredIdx < 0 {+ t.Fatalf("no structured doc_type_kwd:\"table\" item emitted for nested table; got items: %#v", res.JSON)+ }+ if got, want := res.JSON[structuredIdx]["ck_type"], "table"; got != want {+ t.Errorf("structured nested table item ck_type = %v, want %v", got, want)+ }+ if !strings.Contains(structuredText, "<table") ||+ !strings.Contains(structuredText, "Name") ||+ !strings.Contains(structuredText, "Alice") {+ t.Errorf("structured nested table item missing markup/cells: %q", structuredText)+ }++ // The structured table item must be appended after the walk, i.e. after+ // every in-walk text item (mirrors the top-level path and markdown).+ if trailingIdx < 0 {+ t.Fatalf("trailing paragraph text item missing; got items: %#v", res.JSON)+ }+ if structuredIdx <= inlinedIdx || structuredIdx <= trailingIdx {+ t.Errorf("structured nested table item at index %d must come after the inlined text item at %d and trailing paragraph at %d", structuredIdx, inlinedIdx, trailingIdx)+ }+}++// TestHTMLParser_NestedListItemsPreserved verifies a nested <ul> (a list+// containing a sublist) does NOT lose any item text — i.e. it is NOT subject to+// the same kind of collapse the table path once had. Unlike tables (whose markup is+// dropped and cells can fuse), list items are separated by hard breaks inside+// walkHTMLLeaf, so every <li> text survives in document order. This mirrors+// Python's RAGFlowHtmlParser: read_text_recursively (html_parser.py:108-160)+// assigns each block its own block_id and emits flat, depth-less records for+// nested <li>s — so BOTH sides preserve item content and NEITHER represents+// nesting depth. The accepted divergence is granularity: Python emits one+// record per <li> while Go emits the whole <ul> as a single text item (its+// "one item per block-level element" model); the primary alignment gate is+// content equivalence (PARSER_ALIGNMENT_HANDOFF.md §2.3), which this test+// guards. A regression here would be Go dropping or fusing list items.+func TestHTMLParser_NestedListItemsPreserved(t *testing.T) {+ const html = `<html><body>+<ul>+ <li>Item 1</li>+ <li>Item 2+ <ul>+ <li>Sub A</li>+ <li>Sub B</li>+ </ul>+ </li>+</ul>+</body></html>`++ p := NewHTMLParser()+ res := p.ParseWithResult(context.Background(), "nested-list.html", []byte(html))+ if res.Err != nil {+ t.Fatalf("ParseWithResult: %v", res.Err)+ }++ // Join every item's text in order to check content + order + fusion.+ var all string+ for _, it := range res.JSON {+ if txt, _ := it["text"].(string); txt != "" {+ all += txt + "\n"+ }+ }++ for _, want := range []string{"Item 1", "Item 2", "Sub A", "Sub B"} {+ if !strings.Contains(all, want) {+ t.Errorf("nested list lost item %q; joined output: %q", want, all)+ }+ }++ // Document order must survive: the sublist items must follow their parent.+ i2 := strings.Index(all, "Item 2")+ ia := strings.Index(all, "Sub A")+ ib := strings.Index(all, "Sub B")+ if i2 < 0 || ia < 0 || ib < 0 || !(ia > i2 && ib > i2) {+ t.Errorf("nested list order not preserved (parent must precede sub-items): %q", all)+ }++ // No collapse: items must not be fused without a separator+ // (e.g. "Item 2" directly adjacent to "Sub A" with no break).+ if strings.Contains(all, "Item 2Sub") || strings.Contains(all, "Sub ASub") {+ t.Errorf("nested list items fused together (collapse): %q", all)+ }++ // The whole top-level <ul> is emitted as ONE text item tagged+ // ck_type:"list" (Go's "one item per block-level element" model). Guard+ // against a regression that would drop the list tag or split it+ // unexpectedly. Note: a list nested inside a non-list container+ // (div/section/…) is folded into that container's single text item and+ // therefore loses ck_type:"list" (becomes "text") — a fidelity nuance,+ // not a collapse, since downstream chunking reads doc_type_kwd not ck_type.+ var listText string+ for _, it := range res.JSON {+ if it["ck_type"] == "list" {+ listText, _ = it["text"].(string)+ break+ }+ }+ if listText == "" {+ t.Fatalf("top-level <ul> not tagged ck_type:\"list\"; got items: %#v", res.JSON)+ }+ for _, want := range []string{"Item 1", "Item 2", "Sub A", "Sub B"} {+ if !strings.Contains(listText, want) {+ t.Errorf("ck_type:\"list\" item missing %q: %q", want, listText)+ }+ }+}++// TestHTMLParser_PrePreservesVerbatim guards that <pre> (and <textarea>) blocks+// are emitted verbatim — indentation and internal newlines must survive, NOT be+// CSS-collapsed into single spaces. Code is high-value content: if the verbatim+// path regressed to the folded leaf-text path, indented code would lose its+// structure and the LLM / user would see broken code. This mirrors Python's+// RAGFlowHtmlParser, which keeps pre/textarea verbatim (_PRE_TAGS in+// deepdoc/parser/html_parser.py). Both blocks keep doc_type_kwd:"text"; <pre>+// is tagged ck_type:"code" while <textarea> falls back to ck_type:"text"+// (htmlTagToCkType has no textarea case) — a labeling nuance, not a content+// divergence. Note: the single leading newline immediately after <pre>/<textarea>+// is dropped by the HTML parser per spec (pre/textarea eat one leading newline);+// that is correct browser behavior, not folding. What must survive is the+// indentation and the internal newlines.+func TestHTMLParser_PrePreservesVerbatim(t *testing.T) {+ const html = `<html><body>+<pre>+def foo():+ if x:+ return 1+ return 0+</pre>+<textarea>+SELECT * FROM t+ WHERE a = 1+</textarea>+</body></html>`++ p := NewHTMLParser()+ res := p.ParseWithResult(context.Background(), "pre.html", []byte(html))+ if res.Err != nil {+ t.Fatalf("ParseWithResult: %v", res.Err)+ }++ var preItem, textareaItem map[string]any+ for _, it := range res.JSON {+ txt, _ := it["text"].(string)+ switch {+ case strings.Contains(txt, "def foo()"):+ preItem = it+ case strings.Contains(txt, "SELECT * FROM t"):+ textareaItem = it+ }+ }++ if preItem == nil {+ t.Fatalf("no <pre> item found; got items: %#v", res.JSON)+ }+ if preItem["ck_type"] != "code" {+ t.Errorf("<pre> item ck_type = %v, want code", preItem["ck_type"])+ }+ preText, _ := preItem["text"].(string)+ // 4-space and 8-space indents must survive verbatim (a folded path would+ // collapse them to a single space).+ if !strings.Contains(preText, " if x:") {+ t.Errorf("<pre> lost 4-space indent (folded?): %q", preText)+ }+ if !strings.Contains(preText, " return 1") {+ t.Errorf("<pre> lost 8-space indent (folded?): %q", preText)+ }+ // Internal newlines must survive (line structure preserved).+ if !strings.Contains(preText, "def foo():\n") {+ t.Errorf("<pre> lost internal newline (folded?): %q", preText)+ }++ if textareaItem == nil {+ t.Fatalf("no <textarea> item found; got items: %#v", res.JSON)+ }+ taText, _ := textareaItem["text"].(string)+ // 2-space indent must survive verbatim.+ if !strings.Contains(taText, " WHERE a = 1") {+ t.Errorf("<textarea> lost 2-space indent (folded?): %q", taText)+ }+}++// TestHTMLParser_AlignmentGolden verifies Go's ParseWithResult output is+// content-equivalent to Python's RAGFlowHtmlParser on the shared sample, using+// the shared concatenation-normalization alignment tool (align_test.go).+// Python keeps raw HTML and splits on block elements (deepdoc/parser/html_parser.py:+// read_text_recursively + merge_block_text); Go emits clean per-block text.+// The comparison normalizes both (HTML heading markers "#{1,6}", HTML tags,+// whitespace collapsed) and ignores "table" items, which are accepted+// representation differences (the <table> markup stays inline as a "text"+// item in both, see TestHTMLParser_TableProducesStructuredItems). The+// accepted divergences are declared in the golden's meta block, not hardcoded+// here.+//+// The baseline lives in testdata/html.python.en.golden.json / testdata/html.python.zh.golden.json as {meta, items}+// document (see its "meta" block for how to regenerate it from the Python+// engine — no committed generator script).+func TestHTMLParser_AlignmentGolden(t *testing.T) {+ ctx := t.Context()+ p := NewHTMLParser()++ cases := []struct {+ name string+ sample string+ golden string+ }{+ {"en", "testdata/html.sample.en.html", "testdata/html.python.en.golden.json"},+ {"zh", "testdata/html.sample.zh.html", "testdata/html.python.zh.golden.json"},+ }+ for _, tc := range cases {+ t.Run(tc.name, func(t *testing.T) {+ sample, err := os.ReadFile(tc.sample)+ if err != nil {+ t.Fatalf("read sample: %v", err)+ }+ res := p.ParseWithResult(ctx, tc.sample, sample)+ if res.Err != nil {+ t.Fatalf("ParseWithResult: %v", res.Err)+ }++ doc := LoadGoldenDoc(t, tc.golden)++ // Drop the meta-declared accepted divergences on both sides (here "table"):+ // Go appends a structured doc_type_kwd:"table" item, while Python keeps the+ // <table> markup inline as a "text" item. The inline markup remains in both+ // runs and is still compared — that is exactly what guards against collapse.+ drop := AcceptedDivergences(doc.Meta)+ goText := FilterOutDocTypes(res.JSON, drop)+ pyText := FilterOutDocTypes(doc.Items, drop)++ if ok, diff := CompareAlignment(goText, pyText, HTMLAlignOptions()); !ok {+ t.Fatalf("html parser not aligned with Python golden:%s", diff)+ }+ })+ }+}diff --git a/internal/parser/parser/testdata/html.python.en.golden.json b/internal/parser/parser/testdata/html.python.en.golden.jsonnew file mode 100644index 0000000000..b172780e89--- /dev/null+++ b/internal/parser/parser/testdata/html.python.en.golden.json@@ -0,0 +1,33 @@+{+ "meta": {+ "python_engine": "deepdoc.parser.html_parser.RAGFlowHtmlParser",+ "generator": "rag/flow/parser/parser.py:_html (HtmlParser()(name, blob, 512))",+ "sample": "internal/parser/parser/testdata/html.sample.en.html",+ "delimiter": "\\n!?;。;!?",+ "separate_tables": false,+ "accepted_divergences": ["table"],+ "note": "No generator script is committed. To regenerate: call RAGFlowHtmlParser.read_text_recursively on the sample at chunk_token_num=512, apply the TITLE_TAGS heading prefix (\"# \"/\"## \" …) inline, emit every block (including the inline <table>…</table> markup) as a {\"text\": …, \"doc_type_kwd\": \"text\"} record, then wrap in {meta, items}. Go also appends a structured doc_type_kwd:\"table\" item; that extra record is excluded via meta.accepted_divergences in the alignment test."+ },+ "items": [+ {+ "text": "# Product Guide",+ "doc_type_kwd": "text"+ },+ {+ "text": "Step 1: Open the app. Step 2: Click Settings to complete the setup.",+ "doc_type_kwd": "text"+ },+ {+ "text": "<table>\n<tr><th>Name</th><th>Age</th></tr>\n<tr><td>Alice</td><td>30</td></tr>\n<tr><td>Bob</td><td>25</td></tr>\n</table>",+ "doc_type_kwd": "text"+ },+ {+ "text": "## Terms of Use",+ "doc_type_kwd": "text"+ },+ {+ "text": "Welcome to our smart assistant, which helps you complete tasks quickly.",+ "doc_type_kwd": "text"+ }+ ]+}diff --git a/internal/parser/parser/testdata/html.python.zh.golden.json b/internal/parser/parser/testdata/html.python.zh.golden.jsonnew file mode 100644index 0000000000..ec60d2f058--- /dev/null+++ b/internal/parser/parser/testdata/html.python.zh.golden.json@@ -0,0 +1,33 @@+{+ "meta": {+ "python_engine": "deepdoc.parser.html_parser.RAGFlowHtmlParser",+ "generator": "rag/flow/parser/parser.py:_html (HtmlParser()(name, blob, 512))",+ "sample": "internal/parser/parser/testdata/html.sample.zh.html",+ "delimiter": "\\n!?;。;!?",+ "separate_tables": false,+ "accepted_divergences": ["table"],+ "note": "No generator script is committed. To regenerate: call RAGFlowHtmlParser.read_text_recursively on the sample at chunk_token_num=512, apply the TITLE_TAGS heading prefix (\"# \"/\"## \" …) inline, emit every block (including the inline <table>…</table> markup) as a {\"text\": …, \"doc_type_kwd\": \"text\"} record, then wrap in {meta, items}. Go also appends a structured doc_type_kwd:\"table\" item; that extra record is excluded via meta.accepted_divergences in the alignment test."+ },+ "items": [+ {+ "text": "# 产品指南",+ "doc_type_kwd": "text"+ },+ {+ "text": "第一步:打开应用。第二步:点击设置完成配置。",+ "doc_type_kwd": "text"+ },+ {+ "text": "<table>\n<tr><th>姓名</th><th>年龄</th></tr>\n<tr><td>张三</td><td>30</td></tr>\n<tr><td>李四</td><td>25</td></tr>\n</table>",+ "doc_type_kwd": "text"+ },+ {+ "text": "## 使用条款",+ "doc_type_kwd": "text"+ },+ {+ "text": "欢迎使用我们的智能助手,它能帮你快速完成任务。",+ "doc_type_kwd": "text"+ }+ ]+}diff --git a/internal/parser/parser/testdata/html.sample.en.html b/internal/parser/parser/testdata/html.sample.en.htmlnew file mode 100644index 0000000000..e3a6f7e78b--- /dev/null+++ b/internal/parser/parser/testdata/html.sample.en.html@@ -0,0 +1,11 @@+<html><body>+<h1>Product Guide</h1>+<p>Step 1: Open the app. Step 2: Click Settings to complete the setup.</p>+<table>+<tr><th>Name</th><th>Age</th></tr>+<tr><td>Alice</td><td>30</td></tr>+<tr><td>Bob</td><td>25</td></tr>+</table>+<h2>Terms of Use</h2>+<p>Welcome to our smart assistant, which helps you complete tasks quickly.</p>+</body></html>diff --git a/internal/parser/parser/testdata/html.sample.zh.html b/internal/parser/parser/testdata/html.sample.zh.htmlnew file mode 100644index 0000000000..480df53629--- /dev/null+++ b/internal/parser/parser/testdata/html.sample.zh.html@@ -0,0 +1,11 @@+<html><body>+<h1>产品指南</h1>+<p>第一步:打开应用。第二步:点击设置完成配置。</p>+<table>+<tr><th>姓名</th><th>年龄</th></tr>+<tr><td>张三</td><td>30</td></tr>+<tr><td>李四</td><td>25</td></tr>+</table>+<h2>使用条款</h2>+<p>欢迎使用我们的智能助手,它能帮你快速完成任务。</p>+</body></html>