Skip to content

RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.

fix(parser): emit a single structured table item in document order (#18168)

Jack8c5c87047030
diff --git a/internal/parser/parser/align_test.go b/internal/parser/parser/align_test.go
index 0699948c87..640b14aadc 100644
--- a/internal/parser/parser/align_test.go
+++ b/internal/parser/parser/align_test.go
@@ -25,8 +25,11 @@ import (
"fmt"
"os"
"regexp"
+ "slices"
"strings"
"testing"
+
+ "golang.org/x/net/html"
)
// Normalizer transforms a single item's text before comparison. Normalizers
@@ -295,6 +298,132 @@ func FilterOutDocTypes(items []map[string]any, drop []string) []map[string]any {
return out
}
+// filterTableDivergence applies the "table" accepted divergence on BOTH sides
+// of the alignment comparison. Go emits a table as a single
+// doc_type_kwd:"table" item (already dropped by FilterOutDocTypes), while
+// Python keeps the <table> markup inlined as a doc_type_kwd:"text" item. The
+// table content is dropped from the prose comparison so the test focuses on
+// non-table prose, but the table is NOT silently removed from a combined prose
+// item: only items that are ENTIRELY a <table> block are dropped (see
+// isStandaloneTable). A prose item that merely *contains* a table (e.g.
+// "intro <table>…</table> outro") is kept whole, so structural differences in
+// surrounding prose are never masked. Table *equivalence* (column/cell content
+// matching between Go and Python) is checked separately by tableSignatures in
+// the golden tests.
+func filterTableDivergence(items []map[string]any, drop []string) []map[string]any {
+ items = FilterOutDocTypes(items, drop)
+ if !slices.Contains(drop, "table") {
+ return items
+ }
+ out := make([]map[string]any, 0, len(items))
+ for _, it := range items {
+ if text, _ := it["text"].(string); isStandaloneTable(text) {
+ continue
+ }
+ out = append(out, it)
+ }
+ return out
+}
+
+// isStandaloneTable reports whether text is exactly a <table> element with no
+// surrounding prose (ignoring surrounding whitespace) — i.e. the whole item IS
+// the table. This recognizes Python's HTML golden inline table text item
+// (doc_type_kwd:"text", text == "<table>…</table>") without accidentally
+// treating a prose item that merely contains a table as droppable. A combined
+// item such as "intro <table>…</table> outro" returns false and is kept in the
+// prose comparison.
+func isStandaloneTable(text string) bool {
+ t := strings.TrimSpace(text)
+ if !strings.HasPrefix(strings.ToLower(t), "<table") {
+ return false
+ }
+ close := strings.ToLower(t)
+ idx := strings.LastIndex(close, "</table>")
+ return idx >= 0 && strings.TrimSpace(t[idx+len("</table>"):]) == ""
+}
+
+// tableCellSignature returns a normalized, order-preserving signature of a
+// <table>'s cell text (th/td inner text, whitespace-collapsed, cells joined by
+// "\n"). It is intentionally tolerant of serialization differences (attribute
+// order, <tbody> insertion, self-closing vs paired tags) because only the
+// visible cell text is used — which is exactly what guards against a table
+// being collapsed or a column/cell being dropped. Returns "" if text is not a
+// parseable table.
+func tableCellSignature(text string) string {
+ doc, err := html.Parse(strings.NewReader(text))
+ if err != nil {
+ return ""
+ }
+ var cells []string
+ var walk func(*html.Node)
+ walk = func(n *html.Node) {
+ if n.Type == html.ElementNode && (n.Data == "th" || n.Data == "td") {
+ var b strings.Builder
+ var collect func(*html.Node)
+ collect = func(c *html.Node) {
+ if c.Type == html.TextNode {
+ b.WriteString(c.Data)
+ }
+ for ch := c.FirstChild; ch != nil; ch = ch.NextSibling {
+ collect(ch)
+ }
+ }
+ collect(n)
+ cells = append(cells, strings.Join(strings.Fields(b.String()), " "))
+ }
+ for ch := n.FirstChild; ch != nil; ch = ch.NextSibling {
+ walk(ch)
+ }
+ }
+ walk(doc)
+ return strings.Join(cells, "\n")
+}
+
+// tableSignatures extracts the distinct per-table cell signatures from items.
+// A table item is either doc_type_kwd:"table", or a text item that is entirely
+// a <table> block (Python's HTML golden keeps the table markup inline as a
+// standalone text item). Duplicate signatures are collapsed to a set, because
+// Python's Markdown golden keeps BOTH an inline copy and a structured
+// doc_type_kwd:"table" item for the same table — we compare DISTINCT tables,
+// not raw item counts. The returned set is the unit of Go↔Python table
+// equivalence compared by the alignment golden tests.
+func tableSignatures(items []map[string]any) map[string]bool {
+ sigs := map[string]bool{}
+ for _, it := range items {
+ kd, _ := it["doc_type_kwd"].(string)
+ text, _ := it["text"].(string)
+ if kd != "table" && !isStandaloneTable(text) {
+ continue
+ }
+ if sig := tableCellSignature(text); sig != "" {
+ sigs[sig] = true
+ }
+ }
+ return sigs
+}
+
+// assertTablesEquivalent fails the test if the set of tables (by cell-content
+// signature) differs between the Go output and the Python golden. This restores
+// the Go↔Python table-equivalence guard that filterTableDivergence intentionally
+// removes from the prose comparison: it catches a table being collapsed or a
+// column/cell dropped on either side, while tolerating serialization
+// differences (attribute order, <tbody> insertion).
+func assertTablesEquivalent(t *testing.T, goItems, pyItems []map[string]any) {
+ t.Helper()
+ goSigs := tableSignatures(goItems)
+ pySigs := tableSignatures(pyItems)
+ for sig := range goSigs {
+ if !pySigs[sig] {
+ t.Errorf("Go table not present in Python golden (possible collapse / dropped column):\n%s", sig)
+ }
+ }
+ for sig := range pySigs {
+ if !goSigs[sig] {
+ t.Errorf("Python golden table not present in Go output (possible collapse / dropped column):\n%s", sig)
+ }
+ }
+}
+
// MarkdownAlignOptions returns the normalizer preset for Markdown. The order
// matters:
// - StripMarkdownSyntax first: drops "#"/"-"/fenced-code markup that Python
diff --git a/internal/parser/parser/html_parser.go b/internal/parser/parser/html_parser.go
index 9feb8438b6..e6e73c5bf4 100644
--- a/internal/parser/parser/html_parser.go
+++ b/internal/parser/parser/html_parser.go
@@ -107,10 +107,6 @@ func (p *HTMLParser) ParseWithResult(ctx context.Context, filename string, data
// <style>, and <noscript> blocks are skipped entirely so they
// don't pollute the downstream chunker input.
func walkHTMLBlocks(root *html.Node, out *[]map[string]any) {
- // tableItems collects the structured doc_type_kwd:"table" items so they
- // can be appended after the walk (mirrors markdown_parser.go:366-367,
- // which appends tables after all sections to match Python's ordering).
- var tableItems []map[string]any
for child := root.FirstChild; child != nil; child = child.NextSibling {
if child.Type == html.TextNode {
if emitsLooseHTMLText(root) {
@@ -134,21 +130,17 @@ func walkHTMLBlocks(root *html.Node, out *[]map[string]any) {
walkHTMLBlocks(child, out)
continue
case "table":
- // Keep the table as its full HTML markup (NOT flattened) so
- // row/column structure survives into embedding, retrieval, and
- // LLM rendering. This mirrors the markdown table handling
- // (markdown_parser.go:305-328) and Python's HtmlParser, which
- // keeps the <table>…</table> string as the section text.
+ // Emit the <table> as a SINGLE structured doc_type_kwd:"table"
+ // item, in document order. Keeping the full <table>…</table>
+ // markup (not flattened) preserves row/column structure for
+ // embedding, retrieval, and LLM rendering, and doc_type_kwd/
+ // ck_type drives downstream table handling (discrete chunk +
+ // table context). We emit ONLY this item — no duplicate
+ // doc_type_kwd:"text" copy — so the table is embedded once and
+ // its markup does not pollute neighbouring prose chunks.
markup := renderTableHTML(child)
if strings.TrimSpace(markup) != "" {
- // Inlined copy in document order (matches Python's <table>
- // section text and markdown's text-flow copy).
*out = append(*out, map[string]any{
- "text": markup,
- "doc_type_kwd": "text",
- })
- // Structured table item, appended after the walk.
- tableItems = append(tableItems, map[string]any{
"text": markup,
"doc_type_kwd": "table",
"ck_type": "table",
@@ -156,11 +148,9 @@ func walkHTMLBlocks(root *html.Node, out *[]map[string]any) {
}
continue
}
- text := htmlLeafText(child, &tableItems)
- appendHTMLTextItem(out, text, htmlTagToCkType(tag), tag != "pre" && tag != "textarea")
- }
- if len(tableItems) > 0 {
- *out = append(*out, tableItems...)
+ ckType := htmlTagToCkType(tag)
+ trim := tag != "pre" && tag != "textarea"
+ htmlLeafText(child, out, ckType, trim)
}
}
@@ -285,21 +275,46 @@ func (w *leafWriter) hardBreak() {
}
// htmlLeafText joins the visible text of an HTML node and its
-// descendants. <script>/<style>/<noscript> subtrees are skipped. Whitespace
-// is folded per CSS rules (so "<h1>Hello world</h1>" becomes "Hello world"
-// and "<br>" survives as a real line break), while <pre>/<textarea> keep
-// their source formatting verbatim. Any <table> encountered in the subtree is
-// rendered as its <table>…</table> markup (so row/column structure survives)
-// and registered as a structured doc_type_kwd:"table" item via tableItems —
-// see walkHTMLLeaf's "table" case.
-func htmlLeafText(n *html.Node, tableItems *[]map[string]any) string {
+// descendants and emits items directly into out. <script>/<style>/<noscript>
+// subtrees are skipped. Whitespace is folded per CSS rules (so
+// "<h1>Hello world</h1>" becomes "Hello world" and "<br>" survives as a
+// real line break), while <pre>/<textarea> keep their source formatting
+// verbatim. ckType is the block's ck_type (from htmlTagToCkType) applied to the
+// accumulated prose item; trim controls whether trailing/leading whitespace is
+// collapsed (false for <pre>/<textarea>, which must stay verbatim). Any <table>
+// encountered in the subtree is emitted as a single structured
+// doc_type_kwd:"table" item at its document position (see walkHTMLLeaf's
+// "table" case) — the prose around it is flushed as ordinary text items, so
+// the table is never relocated to the end and never duplicated.
+func htmlLeafText(n *html.Node, out *[]map[string]any, ckType string, trim bool) {
var b bytes.Buffer
w := &leafWriter{b: &b}
- walkHTMLLeaf(n, w, tableItems)
- return b.String()
+ walkHTMLLeaf(n, w, out)
+ flushLeafText(w, out, ckType, trim)
}
-func walkHTMLLeaf(n *html.Node, w *leafWriter, tableItems *[]map[string]any) {
+// flushLeafText emits any text accumulated in w as a doc_type_kwd:"text" item
+// (dropping empty output) and resets the writer. ckType/trim mirror
+// appendHTMLTextItem. It is called at block boundaries and at <table> elements
+// so tables are emitted in their original document position rather than being
+// relocated. Prose with no specific block tag (e.g. text accumulated just
+// before a nested table) is flushed as ck_type "text".
+func flushLeafText(w *leafWriter, out *[]map[string]any, ckType string, trim bool) {
+ text := w.b.String()
+ if trim {
+ text = strings.TrimSpace(text)
+ }
+ if text == "" {
+ return
+ }
+ appendHTMLTextItem(out, text, ckType, false)
+ w.b.Reset()
+ w.lastSpace = false
+ w.lineStart = true
+ w.endsNL = false
+}
+
+func walkHTMLLeaf(n *html.Node, w *leafWriter, out *[]map[string]any) {
switch n.Type {
case html.TextNode:
w.writeText(n.Data)
@@ -315,24 +330,24 @@ func walkHTMLLeaf(n *html.Node, w *leafWriter, tableItems *[]map[string]any) {
// Verbatim: no folding, no injected block breaks.
w.pre = true
for child := n.FirstChild; child != nil; child = child.NextSibling {
- walkHTMLLeaf(child, w, tableItems)
+ walkHTMLLeaf(child, w, out)
}
w.pre = false
return
}
if n.Data == "table" {
- // Keep the table as its full HTML markup (NOT flattened) so
- // row/column structure survives into the wrapper's text item,
- // embedding, retrieval, and LLM rendering. This mirrors the
- // top-level walkHTMLBlocks "table" case, and covers tables nested
- // in div/section/article/… (which the walkHTMLBlocks case never
- // reaches). The structured table item is collected for the
- // downstream chunker (appended after the walk by the caller, same
- // as the top-level path).
+ // Emit the <table> as a SINGLE structured doc_type_kwd:"table"
+ // item, in document order. flushLeafText first emits any prose
+ // accumulated before the table so the table stays in its original
+ // position rather than being relocated to the end. The full
+ // <table>…</table> markup is preserved (not flattened) so
+ // row/column structure survives; we do NOT also inline the markup
+ // into the parent's text, which would duplicate the table and
+ // pollute the prose chunk with raw tags.
markup := renderTableHTML(n)
if strings.TrimSpace(markup) != "" {
- w.writeText(markup)
- *tableItems = append(*tableItems, map[string]any{
+ flushLeafText(w, out, "text", true)
+ *out = append(*out, map[string]any{
"text": markup,
"doc_type_kwd": "table",
"ck_type": "table",
@@ -352,7 +367,7 @@ func walkHTMLLeaf(n *html.Node, w *leafWriter, tableItems *[]map[string]any) {
}
}
for child := n.FirstChild; child != nil; child = child.NextSibling {
- walkHTMLLeaf(child, w, tableItems)
+ walkHTMLLeaf(child, w, out)
}
if !w.pre && isBlockTag(n.Data) && w.b.Len() > 0 && !w.endsNL {
w.hardBreak()
diff --git a/internal/parser/parser/html_parser_align_test.go b/internal/parser/parser/html_parser_align_test.go
index 9c7eb38c35..3feae909d0 100644
--- a/internal/parser/parser/html_parser_align_test.go
+++ b/internal/parser/parser/html_parser_align_test.go
@@ -7,20 +7,14 @@ import (
"testing"
)
-// TestHTMLParser_TableProducesStructuredItems asserts the HTML table
-// alignment fix (approach a, mirroring markdown_parser.go:305-328 and
-// :366-367): a <table> must NOT be flattened into a single text blob. The
-// parser must emit two items:
-//
-// 1. an inlined copy in the text flow (doc_type_kwd:"text") whose text keeps
-// the <table> markup, so embedding/retrieval/LLM rendering preserve the
-// row/column structure (no "collapse");
-// 2. a structured table item (doc_type_kwd:"table", ck_type:"table") appended
-// after the walk, consumed by the downstream chunker (attachMediaContext).
-//
-// Before the fix, walkHTMLBlocks ran htmlLeafText over the <table>, producing
-// ONE flat item (e.g. "Name Age Alice 30") tagged ck_type:"table" but with no
-// markup — this test fails on that behavior.
+// TestHTMLParser_TableProducesStructuredItems asserts the HTML table contract:
+// a <table> is emitted as exactly ONE structured doc_type_kwd:"table"/
+// ck_type:"table" item, in its original document position (not relocated to
+// the end), with NO duplicate doc_type_kwd:"text" copy carrying the <table>
+// markup. Keeping the full <table>…</table> markup (not flattened) preserves
+// row/column structure for embedding/retrieval/LLM rendering, and the single
+// structured item is what the downstream chunker keeps as a discrete,
+// independently retrievable chunk.
func TestHTMLParser_TableProducesStructuredItems(t *testing.T) {
const html = `<html><body>
<h1>Employee Table</h1>
@@ -37,67 +31,71 @@ func TestHTMLParser_TableProducesStructuredItems(t *testing.T) {
t.Fatalf("ParseWithResult: %v", res.Err)
}
- var inlinedText, structuredText string
- inlinedIdx, structuredIdx, trailingIdx := -1, -1, -1
+ var tableText string
+ tableIdx, headingIdx, trailingIdx, inlineTableCount, tableCount := -1, -1, -1, 0, 0
for i, it := range res.JSON {
text, _ := it["text"].(string)
switch it["doc_type_kwd"] {
case "text":
if strings.Contains(text, "<table") {
- inlinedText = text
- inlinedIdx = i
+ inlineTableCount++
+ }
+ if text == "Employee Table" {
+ headingIdx = i
}
if text == "Trailing paragraph" {
trailingIdx = i
}
case "table":
- structuredText = text
- structuredIdx = i
+ tableText = text
+ tableIdx = i
+ tableCount++
+ default:
+ t.Fatalf("unexpected doc_type_kwd %q", it["doc_type_kwd"])
}
}
- if inlinedIdx < 0 {
- t.Fatalf("no inlined doc_type_kwd:\"text\" item contains <table> markup; got items: %#v", res.JSON)
- }
- if !strings.Contains(inlinedText, "<table") ||
- !strings.Contains(inlinedText, "Name") ||
- !strings.Contains(inlinedText, "Alice") {
- t.Errorf("inlined table copy missing markup/cells: %q", inlinedText)
- }
-
- if structuredIdx < 0 {
+ if tableIdx < 0 {
t.Fatalf("no structured doc_type_kwd:\"table\" item emitted; got items: %#v", res.JSON)
}
- if got, want := res.JSON[structuredIdx]["ck_type"], "table"; got != want {
+ if tableCount != 1 {
+ t.Fatalf("structured doc_type_kwd:\"table\" item count = %d, want exactly 1", tableCount)
+ }
+ if got, want := res.JSON[tableIdx]["ck_type"], "table"; got != want {
t.Errorf("structured table item ck_type = %v, want %v", got, want)
}
- if structuredText != inlinedText {
- t.Errorf("structured table text differs from inlined copy:\n inlined=%q\n struct =%q", inlinedText, structuredText)
+ if !strings.Contains(tableText, "<table") ||
+ !strings.Contains(tableText, "Name") ||
+ !strings.Contains(tableText, "Alice") {
+ t.Errorf("structured table text missing markup/cells: %q", tableText)
+ }
+ // No duplicate inline text copy of the table markup.
+ if inlineTableCount != 0 {
+ t.Errorf("found %d doc_type_kwd:\"text\" item(s) containing <table> markup; the table must not be duplicated as text", inlineTableCount)
+ }
+ // In document order: between the heading and the trailing paragraph,
+ // NOT relocated after the trailing paragraph.
+ if headingIdx < 0 {
+ t.Fatalf("heading item missing")
}
-
- // The structured table item must be appended after the walk, i.e. after
- // the in-walk text items (mirrors markdown_parser.go:366-367). The trailing
- // <p> is an in-walk text item, so the structured item must come after it.
if trailingIdx < 0 {
- t.Fatalf("trailing paragraph text item missing; got items: %#v", res.JSON)
+ t.Fatalf("trailing paragraph item missing")
+ }
+ if tableIdx <= headingIdx {
+ t.Errorf("structured table item at %d must come after heading at %d", tableIdx, headingIdx)
}
- if structuredIdx <= trailingIdx {
- t.Errorf("structured table item at index %d must come after the trailing paragraph at index %d", structuredIdx, trailingIdx)
+ if tableIdx >= trailingIdx {
+ t.Errorf("structured table item at %d must come before trailing paragraph at %d (not relocated to end)", tableIdx, trailingIdx)
}
}
-// TestHTMLParser_NestedTableProducesStructuredItems asserts the table fix also
-// covers tables that are NOT direct children of <body> — i.e. a <table> wrapped
-// in a layout container such as <div>/<section>/<article> (the common real-world
-// case). walkHTMLBlocks only special-cases a <table> that is a direct child of
-// the walk root; a nested <table> is reached via the default-branch leaf-text
-// extractor (htmlLeafText → walkHTMLLeaf). Before the fix, that path ran
-// htmlLeafText over the wrapper, flattening the table into a single text blob
-// ("Name Age Alice 30") with no markup — the same collapse as the top-level case,
-// just hidden one level deeper. After the fix, walkHTMLLeaf must render the
-// <table> markup inline (so it survives inside the wrapper's single text item)
-// AND register a structured doc_type_kwd:"table"/ck_type:"table" item (appended
-// after the walk like the top-level path).
+// TestHTMLParser_NestedTableProducesStructuredItems asserts the contract also
+// covers tables wrapped in a layout container such as <div>/<section>/<article>
+// (the common real-world case). walkHTMLBlocks only special-cases a <table> that
+// is a direct child of <body>; a nested <table> is reached via htmlLeafText →
+// walkHTMLLeaf. The nested <table> must be emitted as exactly ONE structured
+// doc_type_kwd:"table" item in document order, with NO duplicate inline text
+// copy — the parent container's prose stays clean of raw <table> tags.
func TestHTMLParser_NestedTableProducesStructuredItems(t *testing.T) {
const html = `<html><body>
<h1>Heading</h1>
@@ -116,53 +114,211 @@ func TestHTMLParser_NestedTableProducesStructuredItems(t *testing.T) {
t.Fatalf("ParseWithResult: %v", res.Err)
}
- var inlinedText, structuredText string
- inlinedIdx, structuredIdx, trailingIdx := -1, -1, -1
+ var tableText string
+ tableIdx, headingIdx, trailingIdx, inlineTableCount, tableCount := -1, -1, -1, 0, 0
for i, it := range res.JSON {
text, _ := it["text"].(string)
switch it["doc_type_kwd"] {
case "text":
if strings.Contains(text, "<table") {
- inlinedText = text
- inlinedIdx = i
+ inlineTableCount++
+ }
+ if text == "Heading" {
+ headingIdx = i
}
if text == "Trailing paragraph" {
trailingIdx = i
}
case "table":
- structuredText = text
- structuredIdx = i
+ tableText = text
+ tableIdx = i
+ tableCount++
+ default:
+ t.Fatalf("unexpected doc_type_kwd %q", it["doc_type_kwd"])
}
}
- if inlinedIdx < 0 {
- t.Fatalf("no inlined doc_type_kwd:\"text\" item contains <table> markup (nested table flattened?); got items: %#v", res.JSON)
+ if tableIdx < 0 {
+ t.Fatalf("no structured doc_type_kwd:\"table\" item emitted for nested table; got items: %#v", res.JSON)
+ }
+ if tableCount != 1 {
+ t.Fatalf("structured doc_type_kwd:\"table\" item count = %d, want exactly 1", tableCount)
+ }
+ if got, want := res.JSON[tableIdx]["ck_type"], "table"; got != want {
+ t.Errorf("structured nested table item ck_type = %v, want %v", got, want)
+ }
+ if !strings.Contains(tableText, "<table") ||
+ !strings.Contains(tableText, "Name") ||
+ !strings.Contains(tableText, "Alice") {
+ t.Errorf("structured nested table item missing markup/cells: %q", tableText)
+ }
+ // No duplicate inline text copy of the table markup.
+ if inlineTableCount != 0 {
+ t.Errorf("found %d doc_type_kwd:\"text\" item(s) containing <table> markup; nested table must not be duplicated as inline text", inlineTableCount)
+ }
+ // In document order: between the heading and the trailing paragraph,
+ // NOT relocated after the trailing paragraph.
+ if headingIdx < 0 {
+ t.Fatalf("heading item missing")
+ }
+ if trailingIdx < 0 {
+ t.Fatalf("trailing paragraph item missing")
}
- if !strings.Contains(inlinedText, "<table") ||
- !strings.Contains(inlinedText, "Name") ||
- !strings.Contains(inlinedText, "Alice") {
- t.Errorf("inlined nested table copy missing markup/cells: %q", inlinedText)
+ if tableIdx <= headingIdx {
+ t.Errorf("structured nested table item at %d must come after heading at %d", tableIdx, headingIdx)
}
+ if tableIdx >= trailingIdx {
+ t.Errorf("structured nested table item at %d must come before trailing paragraph at %d (not relocated to end)", tableIdx, trailingIdx)
+ }
+}
- if structuredIdx < 0 {
- t.Fatalf("no structured doc_type_kwd:\"table\" item emitted for nested table; got items: %#v", res.JSON)
+// TestHTMLParser_NestedTableWithSurroundingText guards the structural contract
+// for a <table> nested inside a container that also carries prose before AND
+// after the table (the common real-world shape, e.g.
+// "<div><p>Before.</p><table>…</table><p>After.</p></div>"). The table must be
+// emitted as ONE structured doc_type_kwd:"table" item, and the surrounding prose
+// must be split into SEPARATE clean text items bracketing the table in document
+// order — NOT merged into a single blob that embeds the raw <table> markup.
+// This locks in the fix that removed the inline writeText(markup) of the table
+// into the parent's text flow (the old code would produce one text item
+// "Before <table>…</table> After").
+func TestHTMLParser_NestedTableWithSurroundingText(t *testing.T) {
+ const html = `<html><body>
+<div class="box">
+<p>Before the table.</p>
+<table>
+<tr><th>Name</th><th>Age</th></tr>
+<tr><td>Alice</td><td>30</td></tr>
+</table>
+<p>After the table.</p>
+</div>
+</body></html>`
+
+ p := NewHTMLParser()
+ res := p.ParseWithResult(context.Background(), "nested.html", []byte(html))
+ if res.Err != nil {
+ t.Fatalf("ParseWithResult: %v", res.Err)
}
- if got, want := res.JSON[structuredIdx]["ck_type"], "table"; got != want {
- t.Errorf("structured nested table item ck_type = %v, want %v", got, want)
+
+ var tableText string
+ tableIdx, beforeIdx, afterIdx, inlineTableCount, tableCount := -1, -1, -1, 0, 0
+ for i, it := range res.JSON {
+ text, _ := it["text"].(string)
+ switch it["doc_type_kwd"] {
+ case "text":
+ if strings.Contains(text, "<table") {
+ inlineTableCount++
+ }
+ if text == "Before the table." {
+ beforeIdx = i
+ }
+ if text == "After the table." {
+ afterIdx = i
+ }
+ case "table":
+ tableText = text
+ tableIdx = i
+ tableCount++
+ default:
+ t.Fatalf("unexpected doc_type_kwd %q", it["doc_type_kwd"])
+ }
+ }
+
+ if tableIdx < 0 {
+ t.Fatalf("no structured doc_type_kwd:\"table\" item emitted; got items: %#v", res.JSON)
+ }
+ if tableCount != 1 {
+ t.Fatalf("structured doc_type_kwd:\"table\" item count = %d, want exactly 1", tableCount)
+ }
+ if !strings.Contains(tableText, "<table") ||
+ !strings.Contains(tableText, "Name") ||
+ !strings.Contains(tableText, "Alice") {
+ t.Errorf("structured table item missing markup/cells: %q", tableText)
}
- if !strings.Contains(structuredText, "<table") ||
- !strings.Contains(structuredText, "Name") ||
- !strings.Contains(structuredText, "Alice") {
- t.Errorf("structured nested table item missing markup/cells: %q", structuredText)
+ // No duplicate inline text copy of the table markup.
+ if inlineTableCount != 0 {
+ t.Errorf("found %d doc_type_kwd:\"text\" item(s) containing <table> markup; nested table must not be duplicated as inline text", inlineTableCount)
}
+ // The surrounding prose is split into clean text items bracketing the
+ // table in document order — NOT collapsed into one blob embedding the tags.
+ if beforeIdx < 0 {
+ t.Fatalf("'Before the table.' text item missing")
+ }
+ if afterIdx < 0 {
+ t.Fatalf("'After the table.' text item missing")
+ }
+ if !(beforeIdx < tableIdx && tableIdx < afterIdx) {
+ t.Errorf("document order wrong: before=%d table=%d after=%d (prose must bracket table, not merge into one blob)", beforeIdx, tableIdx, afterIdx)
+ }
+}
- // The structured table item must be appended after the walk, i.e. after
- // every in-walk text item (mirrors the top-level path and markdown).
- if trailingIdx < 0 {
- t.Fatalf("trailing paragraph text item missing; got items: %#v", res.JSON)
+// TestHTMLParser_MultipleTablesOrdering is the HTML counterpart of
+// TestMarkdownParser_MultipleTablesOrdering: two top-level tables must each be
+// emitted as a SINGLE structured doc_type_kwd:"table" item, in document order,
+// bracketing the "Middle." paragraph (not relocated to the end of the stream —
+// the old deferred-append behaviour this PR removes). Cell text of both tables
+// must be present and in source order.
+func TestHTMLParser_MultipleTablesOrdering(t *testing.T) {
+ ctx := t.Context()
+ p := NewHTMLParser()
+ const html = `<html><body>
+<h1>Title</h1>
+<table>
+<tr><th>A</th><th>B</th></tr>
+<tr><td>x</td><td>y</td></tr>
+</table>
+<p>Middle.</p>
+<table>
+<tr><th>C</th><th>D</th></tr>
+<tr><td>p</td><td>q</td></tr>
+</table>
+<p>End.</p>
+</body></html>`
+
+ res := p.ParseWithResult(ctx, "doc.html", []byte(html))
+ if res.Err != nil {
+ t.Fatalf("ParseWithResult: %v", res.Err)
}
- if structuredIdx <= inlinedIdx || structuredIdx <= trailingIdx {
- t.Errorf("structured nested table item at index %d must come after the inlined text item at %d and trailing paragraph at %d", structuredIdx, inlinedIdx, trailingIdx)
+
+ var tableItemIdx []int
+ titleIdx, middleIdx, endIdx := -1, -1, -1
+ for i, it := range res.JSON {
+ text, _ := it["text"].(string)
+ switch it["doc_type_kwd"] {
+ case "text":
+ switch text {
+ case "Title":
+ titleIdx = i
+ case "Middle.":
+ middleIdx = i
+ case "End.":
+ endIdx = i
+ }
+ case "table":
+ tableItemIdx = append(tableItemIdx, i)
+ default:
+ t.Fatalf("unexpected doc_type_kwd %q", it["doc_type_kwd"])
+ }
+ }
+
+ if titleIdx < 0 || middleIdx < 0 || endIdx < 0 {
+ t.Fatalf("missing anchor text item (title=%d middle=%d end=%d)", titleIdx, middleIdx, endIdx)
+ }
+ if len(tableItemIdx) != 2 {
+ t.Fatalf("table items = %d, want 2", len(tableItemIdx))
+ }
+ // Both tables appear in document order, bracketing "Middle.":
+ // table1 before Middle, table2 between Middle and End.
+ if !(titleIdx < tableItemIdx[0] && tableItemIdx[0] < middleIdx && middleIdx < tableItemIdx[1] && tableItemIdx[1] < endIdx) {
+ t.Fatalf("table order wrong: tables=%v title=%d middle=%d end=%d", tableItemIdx, titleIdx, middleIdx, endIdx)
+ }
+ t1, _ := res.JSON[tableItemIdx[0]]["text"].(string)
+ t2, _ := res.JSON[tableItemIdx[1]]["text"].(string)
+ if !strings.Contains(t1, "x") || !strings.Contains(t1, "y") {
+ t.Errorf("first table item missing x/y cells: %q", t1)
+ }
+ if !strings.Contains(t2, "p") || !strings.Contains(t2, "q") {
+ t.Errorf("second table item missing p/q cells: %q", t2)
}
}
@@ -365,12 +521,18 @@ func TestHTMLParser_AlignmentGolden(t *testing.T) {
doc := LoadGoldenDoc(t, tc.golden)
// Drop the meta-declared accepted divergences on both sides (here "table"):
- // Go appends a structured doc_type_kwd:"table" item, while Python keeps the
- // <table> markup inline as a "text" item. The inline markup remains in both
- // runs and is still compared — that is exactly what guards against collapse.
+ // Go emits a single structured doc_type_kwd:"table" item, while Python keeps
+ // the <table> markup inlined as a "text" item. filterTableDivergence drops
+ // the table content from both sides so the comparison stays on non-table prose.
drop := AcceptedDivergences(doc.Meta)
- goText := FilterOutDocTypes(res.JSON, drop)
- pyText := FilterOutDocTypes(doc.Items, drop)
+ goText := filterTableDivergence(res.JSON, drop)
+ pyText := filterTableDivergence(doc.Items, drop)
+
+ // filterTableDivergence drops the table from the prose comparison;
+ // the table-equivalence guard below (assertTablesEquivalent) checks
+ // the table cell content still matches Python, so a collapse or
+ // dropped column on either side is caught independently.
+ assertTablesEquivalent(t, res.JSON, doc.Items)
if ok, diff := CompareAlignment(goText, pyText, HTMLAlignOptions()); !ok {
t.Fatalf("html parser not aligned with Python golden:%s", diff)
diff --git a/internal/parser/parser/markdown_parser.go b/internal/parser/parser/markdown_parser.go
index 54175449f6..680db0a5c1 100644
--- a/internal/parser/parser/markdown_parser.go
+++ b/internal/parser/parser/markdown_parser.go
@@ -269,17 +269,13 @@ func markdownTableCells(line string) []string {
//
// Tables: a GFM/HTML table is rendered inline as a single <table> HTML
// block by renderMarkdownTablesInlineText and kept as one HTML block.
-// It is emitted as TWO items, mirroring Python's _markdown
-// (separate_tables=False): an inlined copy in the text flow
-// (doc_type_kwd:"text") and a separate structured table item
-// (doc_type_kwd:"table", ck_type:"table"). The downstream chunker
-// consumes doc_type_kwd:"table" to keep the table whole and attach
-// table context to neighbouring chunks (chunker/token.go). Non-table
-// HTML blocks (<div>, <style>, …) are emitted as ordinary text with
-// no ck_type. Table items are appended after the walk so the order
-// matches Python (_markdown appends tables after all sections).
+// It is emitted as ONE structured item (doc_type_kwd:"table",
+// ck_type:"table") in its original document position — there is no
+// duplicate doc_type_kwd:"text" copy. The downstream chunker consumes
+// doc_type_kwd:"table" to keep the table whole and attach table context
+// to neighbouring chunks (chunker/token.go). Non-table HTML blocks
+// (<div>, <style>, …) are emitted as ordinary text with no ck_type.
func walkMarkdownBlocksWithImages(doc ast.Node, out *[]map[string]any, flatten bool) {
- var tableItems []map[string]any
for _, child := range doc.GetChildren() {
var ckType string
var docTypeKwd string
@@ -307,20 +303,13 @@ func walkMarkdownBlocksWithImages(doc ast.Node, out *[]map[string]any, flatten b
// HTML block thanks to the blank lines renderMarkdownTablesInline
// wraps around it) or a plain HTML block such as <div>/<style>.
// Only a table is emitted as a structured table item; everything
- // else is treated as ordinary text (no ck_type).
+ // else is treated as ordinary text (no ck_type). We emit exactly
+ // ONE item (doc_type_kwd:"table"/ck_type:"table") in document
+ // order — no duplicate doc_type_kwd:"text" copy — so the table is
+ // embedded once and its markup does not pollute prose chunks.
txt = leafText(n)
if isTableHTML(txt) {
- // Inlined copy kept in the text flow. This is what the
- // alignment golden compares against (Python inlines the
- // rendered <table> HTML into a text section).
*out = append(*out, map[string]any{
- "text": txt,
- "doc_type_kwd": "text",
- })
- // Separate structured table item (mirrors Python's extra
- // doc_type_kwd:"table" item). doc_type_kwd:"table" drives
- // downstream table handling.
- tableItems = append(tableItems, map[string]any{
"text": txt,
"doc_type_kwd": "table",
"ck_type": "table",
@@ -363,8 +352,6 @@ func walkMarkdownBlocksWithImages(doc ast.Node, out *[]map[string]any, flatten b
*out = append(*out, item)
}
- // Tables appended last, mirroring Python's _markdown ordering.
- *out = append(*out, tableItems...)
}
// isTableHTML reports whether block text is an outer <table> element (the
diff --git a/internal/parser/parser/markdown_parser_test.go b/internal/parser/parser/markdown_parser_test.go
index e0d5413b7a..d8c819d797 100644
--- a/internal/parser/parser/markdown_parser_test.go
+++ b/internal/parser/parser/markdown_parser_test.go
@@ -98,32 +98,42 @@ func TestMarkdownParser_ParseWithResult_RendersTableInline(t *testing.T) {
if res.Err != nil {
t.Fatalf("ParseWithResult: %v", res.Err)
}
- // 方案 Y: a document containing a table must NOT collapse into a single
- // item. Each top-level block is its own item, and the table is emitted as
- // both an inlined text copy (doc_type_kwd:"text") and a separate
- // structured table item (doc_type_kwd:"table"), so the item count is > 1.
- if len(res.JSON) < 2 {
- t.Fatalf("len(JSON) = %d, want >= 2 (table must not collapse the doc)", len(res.JSON))
- }
- var all strings.Builder
+ // The document has three top-level blocks: the leading paragraph, the
+ // table, and the trailing note. The table is emitted as exactly ONE
+ // structured doc_type_kwd:"table" item (not flattened, not duplicated as
+ // a doc_type_kwd:"text" copy).
+ if len(res.JSON) != 3 {
+ t.Fatalf("len(JSON) = %d, want 3 (leading paragraph, table, note)", len(res.JSON))
+ }
+ tableCount, inlineTableCount := 0, 0
for _, item := range res.JSON {
kd, _ := item["doc_type_kwd"].(string)
- if kd != "text" && kd != "table" {
+ text, _ := item["text"].(string)
+ switch kd {
+ case "table":
+ tableCount++
+ if ck, _ := item["ck_type"].(string); ck != "table" {
+ t.Errorf("table item ck_type = %q, want \"table\"", ck)
+ }
+ if !strings.Contains(text, "<table") {
+ t.Errorf("table item text is not raw <table> HTML: %q", text)
+ }
+ case "text":
+ if strings.Contains(text, "<table") {
+ inlineTableCount++
+ }
+ default:
t.Fatalf("unexpected doc_type_kwd %q; want text or table", kd)
}
- text, _ := item["text"].(string)
- all.WriteString(text)
- all.WriteString("\n")
if strings.Contains(text, "| Check item |") {
t.Fatalf("raw markdown table leaked into text: %q", text)
}
}
- // The table is kept as raw <table>…</table> HTML (one HTML block), so the
- // cell text survives as text inside the tags — possibly duplicated across
- // the inlined copy and the table item. Check the concatenated text.
- concat := all.String()
- if !strings.Contains(concat, "Check item") || !strings.Contains(concat, "Blood routine") {
- t.Fatalf("table cell content (Check item / Blood routine) not present: %q", concat)
+ if tableCount != 1 {
+ t.Fatalf("table items = %d, want 1", tableCount)
+ }
+ if inlineTableCount != 0 {
+ t.Fatalf("found %d doc_type_kwd:\"text\" item(s) with <table> markup; table must not be duplicated as text", inlineTableCount)
}
}
@@ -297,12 +307,12 @@ func TestFetchImageAsBase64_InvalidURL(t *testing.T) {
}
// TestMarkdownParser_TableNotCollapsed is the core regression guard for the
-// Markdown table fix (方案 Y): a document containing a GFM table must keep one
-// item per top-level block instead of collapsing into one giant item, AND the
-// table must be emitted as a single raw <table>…</table> HTML block (not
-// scattered cell text), appearing both as an inlined text copy and as a
-// separate doc_type_kwd:"table" item. A heading, the table, and the trailing
-// paragraph must each be present.
+// Markdown table fix: a document containing a GFM table must keep one item per
+// top-level block instead of collapsing into one giant item, AND the table must
+// be emitted as a SINGLE raw <table>…</table> HTML block (not scattered cell
+// text) carrying doc_type_kwd:"table" in its original document position. There
+// must be NO redundant inlined doc_type_kwd:"text" copy of the table. A heading,
+// the table, and the trailing paragraph must each be present and in order.
func TestMarkdownParser_TableNotCollapsed(t *testing.T) {
ctx := t.Context()
p, _ := NewMarkdownParser(GoMarkdown)
@@ -311,12 +321,13 @@ func TestMarkdownParser_TableNotCollapsed(t *testing.T) {
if res.Err != nil {
t.Fatalf("ParseWithResult: %v", res.Err)
}
- // Title, Intro, inlined table copy, separate table item, Trailing = 5.
- if len(res.JSON) < 5 {
- t.Fatalf("len(JSON) = %d, want >= 5 (one per top-level block + table item)", len(res.JSON))
+ // Title, Intro, table item, Trailing = 4.
+ if len(res.JSON) != 4 {
+ t.Fatalf("len(JSON) = %d, want 4 (one per top-level block, table as one item)", len(res.JSON))
}
var all strings.Builder
- sawTitle, sawTrailing, sawTableItem, sawRawTableHTML := false, false, false, false
+ sawTitle, sawTrailing, sawTableItem, sawInlineTable := false, false, false, false
+ tableCount := 0
for _, item := range res.JSON {
text, _ := item["text"].(string)
all.WriteString(text)
@@ -326,12 +337,13 @@ func TestMarkdownParser_TableNotCollapsed(t *testing.T) {
}
switch kd, _ := item["doc_type_kwd"].(string); kd {
case "text":
- // inlined copy carries the raw <table> HTML with cell text.
+ // A doc_type_kwd:"text" copy must NOT carry the raw <table> HTML.
if strings.Contains(text, "<table") && strings.Contains(text, "A") && strings.Contains(text, "B") {
- sawRawTableHTML = true
+ sawInlineTable = true
}
case "table":
sawTableItem = true
+ tableCount++
if !strings.Contains(text, "<table") {
t.Fatalf("table item text is not raw <table> HTML: %q", text)
}
@@ -347,17 +359,91 @@ func TestMarkdownParser_TableNotCollapsed(t *testing.T) {
if !sawTitle {
t.Fatal("heading item not emitted")
}
- if !sawRawTableHTML {
- t.Fatal("inlined table not emitted as raw <table> HTML with cell text")
+ if sawInlineTable {
+ t.Fatal("table wrongly emitted as redundant inlined doc_type_kwd:\"text\" copy")
}
if !sawTableItem {
t.Fatal("separate doc_type_kwd:\"table\" item not emitted")
}
+ if tableCount != 1 {
+ t.Fatalf("structured doc_type_kwd:\"table\" item count = %d, want exactly 1", tableCount)
+ }
if !sawTrailing {
t.Fatal("trailing paragraph content not present in concatenated items")
}
}
+// TestMarkdownParser_TableWithSurroundingText is the Markdown counterpart of
+// TestHTMLParser_NestedTableWithSurroundingText: a GFM table with prose both
+// before and after it must emit the table as ONE structured
+// doc_type_kwd:"table" item, with the surrounding paragraphs split into
+// SEPARATE clean text items bracketing the table in document order — NOT
+// merged into a single blob, and NOT duplicated as an inlined
+// doc_type_kwd:"text" copy. This locks in the single-item-in-document-order
+// contract for the common "intro / table / outro" shape.
+func TestMarkdownParser_TableWithSurroundingText(t *testing.T) {
+ ctx := t.Context()
+ p, _ := NewMarkdownParser(GoMarkdown)
+ md := "Before the table.\n\n| Name | Age |\n| --- | --- |\n| Alice | 30 |\n\nAfter the table.\n"
+ res := p.ParseWithResult(ctx, "test.md", []byte(md))
+ if res.Err != nil {
+ t.Fatalf("ParseWithResult: %v", res.Err)
+ }
+
+ var tableText string
+ tableIdx, beforeIdx, afterIdx, inlineTableCount, tableCount := -1, -1, -1, 0, 0
+ for i, item := range res.JSON {
+ text, _ := item["text"].(string)
+ switch kd, _ := item["doc_type_kwd"].(string); kd {
+ case "text":
+ if strings.Contains(text, "<table") {
+ inlineTableCount++
+ }
+ if text == "Before the table." {
+ beforeIdx = i
+ }
+ if text == "After the table." {
+ afterIdx = i
+ }
+ case "table":
+ tableText = text
+ tableIdx = i
+ tableCount++
+ if !strings.Contains(text, "<table") {
+ t.Fatalf("table item text is not raw <table> HTML: %q", text)
+ }
+ if ck, _ := item["ck_type"].(string); ck != "table" {
+ t.Fatalf("table item ck_type = %q, want \"table\"", ck)
+ }
+ }
+ }
+
+ if tableIdx < 0 {
+ t.Fatalf("no structured doc_type_kwd:\"table\" item emitted; got items: %#v", res.JSON)
+ }
+ if tableCount != 1 {
+ t.Fatalf("structured doc_type_kwd:\"table\" item count = %d, want exactly 1", tableCount)
+ }
+ if !strings.Contains(tableText, "Name") || !strings.Contains(tableText, "Alice") {
+ t.Errorf("structured table item missing cell text: %q", tableText)
+ }
+ // No duplicate inline text copy of the table markup.
+ if inlineTableCount != 0 {
+ t.Errorf("found %d doc_type_kwd:\"text\" item(s) containing <table> markup; table must not be duplicated as inline text", inlineTableCount)
+ }
+ // The surrounding prose is split into clean text items bracketing the
+ // table in document order — NOT collapsed into one blob.
+ if beforeIdx < 0 {
+ t.Fatalf("'Before the table.' text item missing")
+ }
+ if afterIdx < 0 {
+ t.Fatalf("'After the table.' text item missing")
+ }
+ if !(beforeIdx < tableIdx && tableIdx < afterIdx) {
+ t.Errorf("document order wrong: before=%d table=%d after=%d (prose must bracket table, not merge into one blob)", beforeIdx, tableIdx, afterIdx)
+ }
+}
+
// TestMarkdownParser_NonTableHTMLBlockNotTable guards against the blanket
// ck_type:"table" bug: a raw HTML block that is NOT a <table> (e.g. <div>,
// <style>) must be emitted as ordinary text with no ck_type, so downstream
@@ -421,8 +507,8 @@ func TestMarkdownParser_TableInCodeFenceNotRendered(t *testing.T) {
// TestMarkdownParser_RawHTMLTableHandled covers a user-written raw <table> HTML
// block (not a GFM pipe table). renderMarkdownTablesInline only rewrites GFM
// pipe tables, so the raw <table> passes through and is caught by isTableHTML
-// as an HTMLBlock, producing the same inlined copy + separate doc_type_kwd:"table"
-// item shape as a GFM table.
+// as an HTMLBlock, producing a single doc_type_kwd:"table" item in document
+// order. There must be NO redundant inlined doc_type_kwd:"text" copy.
func TestMarkdownParser_RawHTMLTableHandled(t *testing.T) {
ctx := t.Context()
p, _ := NewMarkdownParser(GoMarkdown)
@@ -432,6 +518,7 @@ func TestMarkdownParser_RawHTMLTableHandled(t *testing.T) {
t.Fatalf("ParseWithResult: %v", res.Err)
}
sawInlined, sawTableItem := false, false
+ tableCount := 0
for _, item := range res.JSON {
text, _ := item["text"].(string)
switch kd, _ := item["doc_type_kwd"].(string); kd {
@@ -441,6 +528,7 @@ func TestMarkdownParser_RawHTMLTableHandled(t *testing.T) {
}
case "table":
sawTableItem = true
+ tableCount++
if !strings.Contains(text, "<table") {
t.Fatalf("raw table item text is not raw <table> HTML: %q", text)
}
@@ -449,19 +537,22 @@ func TestMarkdownParser_RawHTMLTableHandled(t *testing.T) {
}
}
}
- if !sawInlined {
- t.Fatal("raw <table> HTML block not emitted as inlined copy in text flow")
+ if sawInlined {
+ t.Fatal("raw <table> HTML block wrongly emitted as redundant inlined copy in text flow")
}
if !sawTableItem {
- t.Fatal("raw <table> HTML block not emitted as separate doc_type_kwd:\"table\" item")
+ t.Fatal("raw <table> HTML block not emitted as single doc_type_kwd:\"table\" item")
+ }
+ if tableCount != 1 {
+ t.Fatalf("structured doc_type_kwd:\"table\" item count = %d, want exactly 1", tableCount)
}
}
// TestMarkdownParser_MultipleTablesOrdering guards the ordering contract: each
-// GFM table emits an inlined copy in its original document position (among the
-// surrounding text blocks) and a separate doc_type_kwd:"table" item appended at
-// the end of the stream (mirroring Python's _markdown, which appends tables
-// after all sections). Both tables' cell text must be present and in source order.
+// GFM table is emitted as a single doc_type_kwd:"table" item in its original
+// document position (among the surrounding text blocks). Both tables' cell text
+// must be present and in source order, bracketing "Middle." — NOT appended at
+// the end of the stream.
func TestMarkdownParser_MultipleTablesOrdering(t *testing.T) {
ctx := t.Context()
p, _ := NewMarkdownParser(GoMarkdown)
@@ -471,7 +562,6 @@ func TestMarkdownParser_MultipleTablesOrdering(t *testing.T) {
t.Fatalf("ParseWithResult: %v", res.Err)
}
- var inlinedTableIdx []int
var tableItemIdx []int
var titleIdx, middleIdx, endIdx = -1, -1, -1
for i, item := range res.JSON {
@@ -486,9 +576,6 @@ func TestMarkdownParser_MultipleTablesOrdering(t *testing.T) {
case "End.":
endIdx = i
}
- if strings.Contains(text, "<table") {
- inlinedTableIdx = append(inlinedTableIdx, i)
- }
case "table":
tableItemIdx = append(tableItemIdx, i)
}
@@ -497,21 +584,13 @@ func TestMarkdownParser_MultipleTablesOrdering(t *testing.T) {
if titleIdx < 0 || middleIdx < 0 || endIdx < 0 {
t.Fatalf("missing anchor text item (title=%d middle=%d end=%d)", titleIdx, middleIdx, endIdx)
}
- if len(inlinedTableIdx) != 2 {
- t.Fatalf("inlined table copies = %d, want 2", len(inlinedTableIdx))
- }
if len(tableItemIdx) != 2 {
- t.Fatalf("separate table items = %d, want 2", len(tableItemIdx))
+ t.Fatalf("table items = %d, want 2", len(tableItemIdx))
}
- // Inlined copies appear in document order, bracketing "Middle.":
+ // Table items appear in document order, bracketing "Middle.":
// table1 before Middle, table2 between Middle and End.
- if !(inlinedTableIdx[0] < middleIdx && middleIdx < inlinedTableIdx[1] && inlinedTableIdx[1] < endIdx) {
- t.Fatalf("inlined table order wrong: tables=%v middle=%d end=%d", inlinedTableIdx, middleIdx, endIdx)
- }
- // Separate table items are appended after all text items, in source order:
- // table1 (x,y) then table2 (p,q), each strictly after the previous.
- if !(tableItemIdx[0] > endIdx && tableItemIdx[0] < tableItemIdx[1]) {
- t.Fatalf("table items not appended after text items in source order: %v end=%d", tableItemIdx, endIdx)
+ if !(titleIdx < tableItemIdx[0] && tableItemIdx[0] < middleIdx && middleIdx < tableItemIdx[1] && tableItemIdx[1] < endIdx) {
+ t.Fatalf("table order wrong: tables=%v title=%d middle=%d end=%d", tableItemIdx, titleIdx, middleIdx, endIdx)
}
t1, _ := res.JSON[tableItemIdx[0]]["text"].(string)
t2, _ := res.JSON[tableItemIdx[1]]["text"].(string)
@@ -568,9 +647,17 @@ func TestMarkdownParser_AlignmentGolden(t *testing.T) {
// Exclude the doc types the golden declares as accepted divergences
// (meta.accepted_divergences) on both sides — no hardcoded list in the test.
+ // filterTableDivergence additionally drops the inlined <table> markup that
+ // Python keeps as a "text" item, so only non-table prose is compared.
ignore := AcceptedDivergences(gd.Meta)
- goText := FilterOutDocTypes(res.JSON, ignore)
- pyText := FilterOutDocTypes(gd.Items, ignore)
+ goText := filterTableDivergence(res.JSON, ignore)
+ pyText := filterTableDivergence(gd.Items, ignore)
+
+ // filterTableDivergence drops the table from the prose comparison;
+ // the table-equivalence guard below (assertTablesEquivalent) checks
+ // the table cell content still matches Python, so a collapse or
+ // dropped column on either side is caught independently.
+ assertTablesEquivalent(t, res.JSON, gd.Items)
if ok, diff := CompareAlignment(goText, pyText, MarkdownAlignOptions(DefaultMarkdownDelimiter)); !ok {
t.Fatalf("markdown parser not aligned with Python golden:%s", diff)