Skip to content

RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.

Fix(parser): don't collapse markdown doc into one item when a table is present (#18014)

Jackd6f6b6231f9b
diff --git a/internal/parser/parser/align_test.go b/internal/parser/parser/align_test.go
new file mode 100644
index 0000000000..d031f4f738
--- /dev/null
+++ b/internal/parser/parser/align_test.go
@@ -0,0 +1,237 @@
+//
+// Copyright 2026 The InfiniFlow Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// Warranties, INCLUDING THE WARRANTIES OF MERCHANTABILITY AND
+// FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE
+// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+// DEALINGS IN THE SOFTWARE.
+//
+
+package parser
+
+import (
+ "encoding/json"
+ "os"
+ "regexp"
+ "strings"
+ "testing"
+)
+
+// Normalizer transforms a single item's text before comparison. Normalizers
+// are composed per parser type so the same comparison core is reused across
+// every format (sessions A–E of the Go↔Python parser alignment).
+type Normalizer func(string) string
+
+// WithDelimiterStrip returns a Normalizer that replaces every rune present in
+// delims with a single space. This normalizes the delimiter-split difference:
+// Python splits the text at delimiters into separate items (so the delimiter
+// becomes an item boundary, i.e. whitespace), while Go keeps the delimiter
+// inline. Replacing with a space — rather than deleting — preserves the token
+// separation on the Go side, so after CollapseWhitespace both sides yield the
+// same space-joined text.
+func WithDelimiterStrip(delims string) Normalizer {
+ return func(s string) string {
+ return strings.Map(func(r rune) rune {
+ if strings.ContainsRune(delims, r) {
+ return ' '
+ }
+ return r
+ }, s)
+ }
+}
+
+// CollapseWhitespace returns a Normalizer that trims and collapses runs of
+// whitespace into a single space. Universal normalizer for tolerant compare.
+func CollapseWhitespace() Normalizer {
+ return func(s string) string {
+ return strings.Join(strings.Fields(s), " ")
+ }
+}
+
+// htmlTagRE matches an HTML tag so table/HTML markup can be ignored when
+// comparing content across the two markdown libraries (goldmark vs
+// Python-Markdown serialize tables differently but the cell text is the same).
+var htmlTagRE = regexp.MustCompile(`(?is)<[^>]+>`)
+
+// StripHTMLTags returns a Normalizer that removes HTML tags, leaving the
+// visible text. Tags are replaced with a single space (not deleted) so
+// adjacent cell text does not fuse — e.g. "<td>A</td><td>B</td>" becomes
+// "A B" rather than "AB". CollapseWhitespace then folds the extra space.
+// Used so table-markup differences between markdown libraries don't mask the
+// underlying content equivalence.
+func StripHTMLTags() Normalizer {
+ return func(s string) string {
+ return htmlTagRE.ReplaceAllString(s, " ")
+ }
+}
+
+// Markdown-syntax regexes removed by StripMarkdownSyntax. The Python flow
+// parser keeps raw markdown in its section text ("# Title", "- item",
+// ``` fenced ```); the Go parser emits clean per-block text. These are
+// representation differences (PARSER_ALIGNMENT_HANDOFF.md §3.1), not content
+// divergences, so the markdown alignment strips them before comparing.
+var (
+ mdHeaderRE = regexp.MustCompile(`(?m)^#{1,6}\s+`)
+ mdListRE = regexp.MustCompile(`(?m)^\s*[-*+]\s+`)
+ mdFenceRE = regexp.MustCompile("(?s)```[^\n]*\n(.*?)```")
+)
+
+// StripMarkdownSyntax returns a Normalizer that removes markdown presentation
+// characters (ATX headings, list bullets, fenced-code fences) from a section,
+// leaving the bare text. It must run before CollapseWhitespace because the
+// fence regex relies on the surrounding newlines.
+func StripMarkdownSyntax() Normalizer {
+ return func(s string) string {
+ s = mdHeaderRE.ReplaceAllString(s, "")
+ s = mdListRE.ReplaceAllString(s, "")
+ s = mdFenceRE.ReplaceAllString(s, "$1")
+ return s
+ }
+}
+
+// FilterByDocType returns only the items whose doc_type_kwd equals kwd.
+// Python emits duplicate table items (separate_tables=False still appends
+// them) — excluding doc_type_kwd:"table" lets the comparison focus on the
+// inlined textual content, which is what Go produces.
+func FilterByDocType(items []map[string]any, kwd string) []map[string]any {
+ out := make([]map[string]any, 0, len(items))
+ for _, it := range items {
+ if v, _ := it["doc_type_kwd"].(string); v == kwd {
+ out = append(out, it)
+ }
+ }
+ return out
+}
+
+// AlignOptions configures NormalizeConcat / CompareAlignment.
+type AlignOptions struct {
+ // Normalizers applied (in order) to each item's text before concat.
+ Normalizers []Normalizer
+ // ItemKey is the field holding the compared text (default "text").
+ ItemKey string
+}
+
+func alignItemText(item map[string]any, key string) string {
+ if key == "" {
+ key = "text"
+ }
+ if v, ok := item[key].(string); ok {
+ return v
+ }
+ return ""
+}
+
+// NormalizeConcat extracts the text field from each item, applies the
+// normalizers in order, and concatenates into one string. Format-agnostic.
+//
+// Items are joined with a single space (after whitespace is collapsed by the
+// normalizers) rather than by newlines: Go emits one item per top-level block
+// while Python splits the same text on delimiters into many smaller items, so
+// the item *boundaries* legitimately differ. Joining on whitespace makes the
+// comparison boundary-agnostic — only the concatenated content (order
+// preserved) is compared, which is exactly the alignment guarantee we want.
+// Empty items are skipped so Python's trailing/duplicate segments don't mask a
+// real content difference.
+func NormalizeConcat(items []map[string]any, opts AlignOptions) string {
+ key := opts.ItemKey
+ if key == "" {
+ key = "text"
+ }
+ parts := make([]string, 0, len(items))
+ for _, it := range items {
+ t := alignItemText(it, key)
+ for _, n := range opts.Normalizers {
+ t = n(t)
+ }
+ if strings.TrimSpace(t) == "" {
+ continue
+ }
+ // Trim so a delimiter turned into a trailing space (WithDelimiterStrip)
+ // doesn't combine with the join space into a double gap.
+ t = strings.TrimSpace(t)
+ parts = append(parts, t)
+ }
+ return strings.Join(parts, " ")
+}
+
+// CompareAlignment reports whether two parser outputs are aligned after
+// normalization. goItems come from Go's ParseResult.JSON; pyItems come from the
+// Python golden JSON. Returns (equal, diffReport).
+func CompareAlignment(goItems, pyItems []map[string]any, opts AlignOptions) (bool, string) {
+ g := NormalizeConcat(goItems, opts)
+ p := NormalizeConcat(pyItems, opts)
+ if g == p {
+ return true, ""
+ }
+ return false, diffReport(g, p)
+}
+
+func diffReport(g, p string) string {
+ const max = 2000
+ if len(g) > max {
+ g = g[:max] + "...(truncated)"
+ }
+ if len(p) > max {
+ p = p[:max] + "...(truncated)"
+ }
+ return "alignment mismatch after normalization:\n--- GO ---\n" + g + "\n--- PY ---\n" + p
+}
+
+// LoadGolden reads a Python golden JSON file (a JSON list of item objects)
+// produced by the Python flow parser for the same input.
+func LoadGolden(t *testing.T, path string) []map[string]any {
+ t.Helper()
+ data, err := os.ReadFile(path)
+ if err != nil {
+ t.Fatalf("load golden %s: %v", path, err)
+ }
+ var items []map[string]any
+ if err := json.Unmarshal(data, &items); err != nil {
+ t.Fatalf("parse golden %s: %v", path, err)
+ }
+ return items
+}
+
+// MarkdownAlignOptions returns the normalizer preset for markdown. The order
+// matters:
+// - StripMarkdownSyntax first: drops "#"/"-"/fenced-code markup that Python
+// keeps inline but Go parses out (relies on the surrounding newlines, so it
+// must run before CollapseWhitespace).
+// - StripHTMLTags next: replace table/HTML tags with a space (not delete) so
+// adjacent cell text does not fuse, e.g. "<td>A</td><td>B</td>" → "A B".
+// - WithDelimiterStrip: replace the delimiter set Python consumes at split
+// points with a space while Go keeps it inline, so both sides keep the same
+// token separation. Runs before CollapseWhitespace so the introduced space
+// is folded normally.
+// - CollapseWhitespace last: folds all remaining internal whitespace (the
+// inter-tag gaps of an HTML table, the space from delimiter replacement,
+// the newlines inside a fenced code block) into single spaces.
+//
+// Reused by every markdown alignment test; other formats define their own
+// preset and share CompareAlignment.
+func MarkdownAlignOptions(delimiter string) AlignOptions {
+ return AlignOptions{
+ Normalizers: []Normalizer{
+ StripMarkdownSyntax(),
+ StripHTMLTags(),
+ WithDelimiterStrip(delimiter),
+ CollapseWhitespace(),
+ },
+ ItemKey: "text",
+ }
+}
+
+// DefaultMarkdownDelimiter is the flow parser's default markdown delimiter
+// set, used when generating/loading the golden baseline.
+const DefaultMarkdownDelimiter = "\n!?;。;!?"
diff --git a/internal/parser/parser/markdown_parser.go b/internal/parser/parser/markdown_parser.go
index 54f9f838f0..e41897d152 100644
--- a/internal/parser/parser/markdown_parser.go
+++ b/internal/parser/parser/markdown_parser.go
@@ -25,7 +25,6 @@ import (
"net"
"net/http"
"net/url"
- "regexp"
"strings"
"sync"
"time"
@@ -35,9 +34,6 @@ import (
mdparser "github.com/gomarkdown/markdown/parser"
)
-// mdImagePattern matches markdown inline image syntax: ![alt](url).
-var mdImagePattern = regexp.MustCompile(`!\[[^\]]*\]\(([^)\s]+)\)`)
-
// dataURIPrefix is the MIME prefix for data URI images.
const dataURIPrefix = "data:image/"
@@ -96,20 +92,18 @@ func (p *MarkdownParser) ConfigureFromSetup(setup map[string]any) {
// been removed; callers consume ParseResult directly.
func (p *MarkdownParser) ParseWithResult(ctx context.Context, filename string, data []byte) ParseResult {
rawText := string(data)
- if rendered, ok := renderMarkdownTablesInline(rawText); ok {
- return ParseResult{
- OutputFormat: "json",
- File: map[string]any{
- "name": filename,
- },
- JSON: []map[string]any{{"text": rendered, "doc_type_kwd": "text"}},
- }
- }
+ // Render any GFM/HTML table inline as an HTML block before parsing. This
+ // keeps the document as one item per top-level block (the table becomes a
+ // normal text item) instead of collapsing the whole document into a single
+ // item. The result mirrors Python's `_markdown` (separate_tables=False),
+ // which also inlines tables into the surrounding text. When no table is
+ // present renderMarkdownTablesInlineText returns the input unchanged.
+ rendered := renderMarkdownTablesInlineText(rawText)
- doc := markdownNew().Parse(data)
+ doc := markdownNew().Parse([]byte(rendered))
var items []map[string]any
- walkMarkdownBlocksWithImages(doc, rawText, &items, p.FlattenMediaToText)
+ walkMarkdownBlocksWithImages(doc, &items, p.FlattenMediaToText)
if items == nil {
items = []map[string]any{{"text": "", "doc_type_kwd": "text"}}
}
@@ -165,8 +159,13 @@ func renderMarkdownTablesInline(text string) (string, bool) {
i++
}
tableHTML := markdownlib.ToHTML([]byte(strings.Join(lines[start:i], "")), markdownNew(), nil)
+ // Wrap the inlined <table> HTML in blank lines so gomarkdown
+ // keeps it as a single HTML block (one item) instead of
+ // re-parsing it into scattered cell text. See
+ // PARSER_ALIGNMENT_HANDOFF.md §3.1 (markdown session A, 方案 Y).
+ ensureTrailingBlankLine(&buf)
buf.WriteString(strings.TrimRight(string(tableHTML), "\r\n"))
- buf.WriteByte('\n')
+ buf.WriteString("\n\n")
changed = true
continue
}
@@ -176,6 +175,32 @@ func renderMarkdownTablesInline(text string) (string, bool) {
return buf.String(), changed
}
+// renderMarkdownTablesInlineText renders every GFM/HTML table inline as an
+// HTML block and returns the rewritten text. When no table is present the
+// input is returned unchanged. Unlike renderMarkdownTablesInline it always
+// returns the full text (ignoring the changed flag) so callers can parse the
+// result uniformly and emit one item per top-level block.
+func renderMarkdownTablesInlineText(text string) string {
+ out, _ := renderMarkdownTablesInline(text)
+ return out
+}
+
+// ensureTrailingBlankLine makes sure b ends with a blank line (two
+// newlines) so the next block is separated from what precedes it. gomarkdown
+// only treats a <table> as a standalone HTML block (rather than re-parsing it
+// into scattered cell nodes) when it is surrounded by blank lines.
+func ensureTrailingBlankLine(b *strings.Builder) {
+ s := b.String()
+ switch {
+ case strings.HasSuffix(s, "\n\n"):
+ // already separated.
+ case strings.HasSuffix(s, "\n"):
+ b.WriteByte('\n')
+ default:
+ b.WriteString("\n\n")
+ }
+}
+
func markdownFenceMarker(line string) (byte, int, bool) {
trimmed := strings.TrimLeft(line, " \t")
if len(line)-len(trimmed) > 3 || len(trimmed) < 3 {
@@ -236,11 +261,25 @@ func markdownTableCells(line string) []string {
// top-level block. Headings, paragraphs, lists, and code blocks are
// emitted with their text. When a block contains a markdown image
// reference (![alt](src)), the image data is resolved via
-// resolveMarkdownImage and the item carries `doc_type_kwd: "image"`
-// together with the base64-encoded image payload. When flatten is
-// true, all items are forced to doc_type_kwd="text" (mirrors Python
-// parser.py:1034 flatten_media_to_text).
-func walkMarkdownBlocksWithImages(doc ast.Node, rawText string, out *[]map[string]any, flatten bool) {
+// findBlockImage (per-block AST walk) and the item carries
+// `doc_type_kwd: "image"` together with the base64-encoded image
+// payload. When flatten is true, all items are forced to
+// doc_type_kwd="text" (mirrors Python parser.py:1034
+// flatten_media_to_text).
+//
+// Tables: a GFM/HTML table is rendered inline as a single <table> HTML
+// block by renderMarkdownTablesInlineText and kept as one HTML block.
+// It is emitted as TWO items, mirroring Python's _markdown
+// (separate_tables=False): an inlined copy in the text flow
+// (doc_type_kwd:"text") and a separate structured table item
+// (doc_type_kwd:"table", ck_type:"table"). The downstream chunker
+// consumes doc_type_kwd:"table" to keep the table whole and attach
+// table context to neighbouring chunks (chunker/token.go). Non-table
+// HTML blocks (<div>, <style>, …) are emitted as ordinary text with
+// no ck_type. Table items are appended after the walk so the order
+// matches Python (_markdown appends tables after all sections).
+func walkMarkdownBlocksWithImages(doc ast.Node, out *[]map[string]any, flatten bool) {
+ var tableItems []map[string]any
for _, child := range doc.GetChildren() {
var ckType string
var docTypeKwd string
@@ -263,6 +302,33 @@ func walkMarkdownBlocksWithImages(doc ast.Node, rawText string, out *[]map[strin
txt = leafText(n)
ckType = "code"
docTypeKwd = "text"
+ case *ast.HTMLBlock:
+ // An HTML block is either an inlined table (kept as a single
+ // HTML block thanks to the blank lines renderMarkdownTablesInline
+ // wraps around it) or a plain HTML block such as <div>/<style>.
+ // Only a table is emitted as a structured table item; everything
+ // else is treated as ordinary text (no ck_type).
+ txt = leafText(n)
+ if isTableHTML(txt) {
+ // Inlined copy kept in the text flow. This is what the
+ // alignment golden compares against (Python inlines the
+ // rendered <table> HTML into a text section).
+ *out = append(*out, map[string]any{
+ "text": txt,
+ "doc_type_kwd": "text",
+ })
+ // Separate structured table item (mirrors Python's extra
+ // doc_type_kwd:"table" item). doc_type_kwd:"table" drives
+ // downstream table handling.
+ tableItems = append(tableItems, map[string]any{
+ "text": txt,
+ "doc_type_kwd": "table",
+ "ck_type": "table",
+ })
+ continue
+ }
+ // Non-table HTML block: ordinary text, no ck_type.
+ docTypeKwd = "text"
default:
txt = leafText(n)
if strings.TrimSpace(txt) == "" {
@@ -279,52 +345,78 @@ func walkMarkdownBlocksWithImages(doc ast.Node, rawText string, out *[]map[strin
item["ck_type"] = ckType
}
- // Detect markdown image references in the raw source text
- // that corresponds to this block. When found, resolve the
- // image data so downstream vision enhancement can describe it.
- // When flatten is true, keep doc_type_kwd="text" (Python
+ // Resolve markdown images from the AST node of THIS block only, so
+ // the image payload (and doc_type_kwd:"image") is attached to the
+ // single block that actually contains the ![alt](src) reference.
+ // Scanning the whole document (the old approach) wrongly tagged
+ // every block as an image whenever any image was present. When
+ // flatten is true, keep doc_type_kwd="text" (Python
// parser.py:1034: flatten_media_to_text overrides image).
- if imgData, imgFound := resolveMarkdownImage(txt, rawText); imgFound && imgData != "" {
- item["image"] = imgData
- if !flatten {
- item["doc_type_kwd"] = "image"
+ if imgURL, ok := findBlockImage(child); ok {
+ if imgData, resolved := resolveImageURL(imgURL); resolved && imgData != "" {
+ item["image"] = imgData
+ if !flatten {
+ item["doc_type_kwd"] = "image"
+ }
}
}
*out = append(*out, item)
}
+ // Tables appended last, mirroring Python's _markdown ordering.
+ *out = append(*out, tableItems...)
+}
+
+// isTableHTML reports whether block text is an outer <table> element (the
+// inlined GFM/HTML table). Only such blocks are emitted as structured table
+// items; other raw HTML (e.g. <div>, <style>) is plain text.
+func isTableHTML(s string) bool {
+ return strings.HasPrefix(strings.TrimSpace(strings.ToLower(s)), "<table")
+}
+
+// findBlockImage returns the destination URL of the first image node found
+// anywhere under n. This associates an image with the specific block that
+// contains it, instead of scanning the whole document (which would wrongly
+// tag every block as an image when any image is present).
+func findBlockImage(n ast.Node) (string, bool) {
+ found := false
+ var url string
+ var walk func(c ast.Node)
+ walk = func(c ast.Node) {
+ if found || c == nil {
+ return
+ }
+ if img, ok := c.(*ast.Image); ok {
+ url = string(img.Destination)
+ found = true
+ return
+ }
+ for _, ch := range c.GetChildren() {
+ walk(ch)
+ }
+ }
+ walk(n)
+ return url, found
}
-// resolveMarkdownImage extracts the first markdown image reference
-// from the given text and returns its base64-encoded data. Supports:
+// resolveImageURL resolves a markdown image URL to its base64-encoded data.
+// Supports:
// - data:image/... URIs → decoded directly
// - http:// / https:// URLs → fetched (with basic SSRF filtering)
//
-// Returns (base64String, true) on success, ("", false) when no image
-// is found or resolution fails.
-func resolveMarkdownImage(leafText, rawFullText string) (string, bool) {
- // Prefer matching against the raw full text to catch images
- // whose alt-text was split across markdown rendering.
- searchIn := rawFullText
- if strings.TrimSpace(searchIn) == "" {
- searchIn = leafText
- }
- matches := mdImagePattern.FindStringSubmatch(searchIn)
- if len(matches) < 2 {
- return "", false
- }
- url := matches[1]
-
- if strings.HasPrefix(url, dataURIPrefix) {
+// Local / relative paths are not fetched (security). Returns
+// (base64String, true) on success, ("", false) when resolution fails.
+func resolveImageURL(imageURL string) (string, bool) {
+ if strings.HasPrefix(imageURL, dataURIPrefix) {
// data:image/png;base64,xxxx
- idx := strings.Index(url, "base64,")
+ idx := strings.Index(imageURL, "base64,")
if idx < 0 {
return "", false
}
- return url[idx+len("base64,"):], true
+ return imageURL[idx+len("base64,"):], true
}
- if strings.HasPrefix(url, "http://") || strings.HasPrefix(url, "https://") {
- b64, err := fetchImageAsBase64(url)
+ if strings.HasPrefix(imageURL, "http://") || strings.HasPrefix(imageURL, "https://") {
+ b64, err := fetchImageAsBase64(imageURL)
if err != nil {
return "", false
}
@@ -481,6 +573,17 @@ func walkLeaf(n ast.Node, buf *bytes.Buffer) {
buf.Write(t.Literal)
case *ast.Code:
buf.Write(t.Literal)
+ case *ast.CodeBlock:
+ // finalizeCodeBlock moves the fenced body into Literal and nils
+ // Content; the indented form keeps it in Content. Emit both so the
+ // code text is never dropped.
+ buf.Write(t.Literal)
+ buf.Write(t.Content)
+ case *ast.HTMLBlock:
+ // Inlined tables are HTML blocks. The generic parser path stores the
+ // markup in Content, the markdown-block path in Literal. Emit both.
+ buf.Write(t.Literal)
+ buf.Write(t.Content)
default:
for _, c := range n.GetChildren() {
walkLeaf(c, buf)
diff --git a/internal/parser/parser/markdown_parser_test.go b/internal/parser/parser/markdown_parser_test.go
index 049b9b12ad..34ab5693a3 100644
--- a/internal/parser/parser/markdown_parser_test.go
+++ b/internal/parser/parser/markdown_parser_test.go
@@ -4,6 +4,7 @@ import (
"encoding/base64"
"net/http"
"net/http/httptest"
+ "os"
"strings"
"testing"
)
@@ -97,21 +98,32 @@ func TestMarkdownParser_ParseWithResult_RendersTableInline(t *testing.T) {
if res.Err != nil {
t.Fatalf("ParseWithResult: %v", res.Err)
}
- if len(res.JSON) != 1 {
- t.Fatalf("len(JSON) = %d, want 1", len(res.JSON))
- }
- text, _ := res.JSON[0]["text"].(string)
- if got, _ := res.JSON[0]["doc_type_kwd"].(string); got != "text" {
- t.Fatalf("doc_type_kwd = %q, want text", got)
- }
- if !strings.Contains(text, "<table>") || !strings.Contains(text, "<th>Check item</th>") {
- t.Fatalf("table was not rendered inline: %q", text)
+ // 方案 Y: a document containing a table must NOT collapse into a single
+ // item. Each top-level block is its own item, and the table is emitted as
+ // both an inlined text copy (doc_type_kwd:"text") and a separate
+ // structured table item (doc_type_kwd:"table"), so the item count is > 1.
+ if len(res.JSON) < 2 {
+ t.Fatalf("len(JSON) = %d, want >= 2 (table must not collapse the doc)", len(res.JSON))
}
- if strings.Contains(text, "| Check item |") {
- t.Fatalf("raw markdown table leaked into text: %q", text)
+ var all strings.Builder
+ for _, item := range res.JSON {
+ kd, _ := item["doc_type_kwd"].(string)
+ if kd != "text" && kd != "table" {
+ t.Fatalf("unexpected doc_type_kwd %q; want text or table", kd)
+ }
+ text, _ := item["text"].(string)
+ all.WriteString(text)
+ all.WriteString("\n")
+ if strings.Contains(text, "| Check item |") {
+ t.Fatalf("raw markdown table leaked into text: %q", text)
+ }
}
- if gap := text[strings.Index(text, "</table>"):strings.Index(text, "Note:")]; gap != "</table>\n" {
- t.Fatalf("gap after table = %q, want %q", gap, "</table>\n")
+ // The table is kept as raw <table>…</table> HTML (one HTML block), so the
+ // cell text survives as text inside the tags — possibly duplicated across
+ // the inlined copy and the table item. Check the concatenated text.
+ concat := all.String()
+ if !strings.Contains(concat, "Check item") || !strings.Contains(concat, "Blood routine") {
+ t.Fatalf("table cell content (Check item / Blood routine) not present: %q", concat)
}
}
@@ -183,10 +195,9 @@ func TestMarkdownParser_FlattenMediaToText(t *testing.T) {
}
}
-func TestResolveMarkdownImage_DataURI(t *testing.T) {
+func TestResolveImageURL_DataURI(t *testing.T) {
b64 := base64.StdEncoding.EncodeToString([]byte("fakeimage"))
- md := "![alt](data:image/png;base64," + b64 + ")"
- result, found := resolveMarkdownImage("", md)
+ result, found := resolveImageURL("data:image/png;base64," + b64)
if !found {
t.Fatal("expected image found for data URI")
}
@@ -195,15 +206,14 @@ func TestResolveMarkdownImage_DataURI(t *testing.T) {
}
}
-func TestResolveMarkdownImage_NoImage(t *testing.T) {
- _, found := resolveMarkdownImage("", "# Hello\nNo image here")
- if found {
- t.Fatal("expected no image found")
+func TestResolveImageURL_LocalPathNotFetched(t *testing.T) {
+ // Local / relative paths are not fetched (security); resolution fails.
+ if _, found := resolveImageURL("./local/image.png"); found {
+ t.Fatal("expected no image resolved for a local path")
}
}
-func TestResolveMarkdownImage_HTTPImage(t *testing.T) {
- withSSRFBypass(t)
+func TestResolveImageURL_HTTPImage(t *testing.T) {
// httptest servers bind loopback, which the SSRF guard rejects by
// default. Allow loopback for this test so the HTTP fetch path is
// exercised (production keeps ssrfAllowLoopback == false).
@@ -211,15 +221,13 @@ func TestResolveMarkdownImage_HTTPImage(t *testing.T) {
ssrfAllowLoopback = true
defer func() { ssrfAllowLoopback = prev }()
- // Start a test HTTP server serving a fake PNG
ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
w.Header().Set("Content-Type", "image/png")
w.Write([]byte("fake-png-bytes"))
}))
defer ts.Close()
- md := "![alt](" + ts.URL + "/image.png)"
- result, found := resolveMarkdownImage("", md)
+ result, found := resolveImageURL(ts.URL + "/image.png")
if !found {
t.Fatal("expected image found for HTTP URL")
}
@@ -229,6 +237,45 @@ func TestResolveMarkdownImage_HTTPImage(t *testing.T) {
}
}
+// TestFindBlockImage resolves the image per-block from the AST so only the
+// owning block is tagged (the fix for the whole-document scan bug).
+func TestFindBlockImage(t *testing.T) {
+ ctx := t.Context()
+ p, _ := NewMarkdownParser(GoMarkdown)
+
+ withImg := "# T\n\nText without image.\n\n![alt](data:image/png;base64,AAA)\n"
+ res := p.ParseWithResult(ctx, "a.md", []byte(withImg))
+ if res.Err != nil {
+ t.Fatalf("ParseWithResult: %v", res.Err)
+ }
+ var imgItems, textItems int
+ for _, item := range res.JSON {
+ switch kd, _ := item["doc_type_kwd"].(string); kd {
+ case "image":
+ imgItems++
+ if _, ok := item["image"].(string); !ok {
+ t.Fatal("image item missing base64 payload")
+ }
+ case "text":
+ textItems++
+ }
+ }
+ if imgItems != 1 {
+ t.Fatalf("imgItems = %d, want 1 (only the block owning the image)", imgItems)
+ }
+ if textItems < 2 {
+ t.Fatalf("textItems = %d, want >= 2 (other blocks must stay text, not image)", textItems)
+ }
+
+ // A document with no image must not produce any image item.
+ noImg := p.ParseWithResult(ctx, "b.md", []byte("# T\n\nNo images here.\n"))
+ for _, item := range noImg.JSON {
+ if kd, _ := item["doc_type_kwd"].(string); kd == "image" {
+ t.Fatal("unexpected image item in text-only markdown")
+ }
+ }
+}
+
func TestFetchImageAsBase64_RejectsCredentials(t *testing.T) {
_, err := fetchImageAsBase64("https://user:[email protected]/img.png")
if err == nil {
@@ -248,3 +295,265 @@ func TestFetchImageAsBase64_InvalidURL(t *testing.T) {
t.Fatal("expected error for 404 response")
}
}
+
+// TestMarkdownParser_TableNotCollapsed is the core regression guard for the
+// markdown table fix (方案 Y): a document containing a GFM table must keep one
+// item per top-level block instead of collapsing into one giant item, AND the
+// table must be emitted as a single raw <table>…</table> HTML block (not
+// scattered cell text), appearing both as an inlined text copy and as a
+// separate doc_type_kwd:"table" item. A heading, the table, and the trailing
+// paragraph must each be present.
+func TestMarkdownParser_TableNotCollapsed(t *testing.T) {
+ ctx := t.Context()
+ p, _ := NewMarkdownParser(GoMarkdown)
+ md := "# Title\n\nIntro paragraph.\n\n| A | B |\n| --- | --- |\n| x | y |\n\nTrailing note.\n"
+ res := p.ParseWithResult(ctx, "test.md", []byte(md))
+ if res.Err != nil {
+ t.Fatalf("ParseWithResult: %v", res.Err)
+ }
+ // Title, Intro, inlined table copy, separate table item, Trailing = 5.
+ if len(res.JSON) < 5 {
+ t.Fatalf("len(JSON) = %d, want >= 5 (one per top-level block + table item)", len(res.JSON))
+ }
+ var all strings.Builder
+ sawTitle, sawTrailing, sawTableItem, sawRawTableHTML := false, false, false, false
+ for _, item := range res.JSON {
+ text, _ := item["text"].(string)
+ all.WriteString(text)
+ all.WriteString("\n")
+ if text == "Title" {
+ sawTitle = true
+ }
+ switch kd, _ := item["doc_type_kwd"].(string); kd {
+ case "text":
+ // inlined copy carries the raw <table> HTML with cell text.
+ if strings.Contains(text, "<table") && strings.Contains(text, "A") && strings.Contains(text, "B") {
+ sawRawTableHTML = true
+ }
+ case "table":
+ sawTableItem = true
+ if !strings.Contains(text, "<table") {
+ t.Fatalf("table item text is not raw <table> HTML: %q", text)
+ }
+ if ck, _ := item["ck_type"].(string); ck != "table" {
+ t.Fatalf("table item ck_type = %q, want \"table\"", ck)
+ }
+ }
+ }
+ concat := all.String()
+ if strings.Contains(concat, "Trailing note.") {
+ sawTrailing = true
+ }
+ if !sawTitle {
+ t.Fatal("heading item not emitted")
+ }
+ if !sawRawTableHTML {
+ t.Fatal("inlined table not emitted as raw <table> HTML with cell text")
+ }
+ if !sawTableItem {
+ t.Fatal("separate doc_type_kwd:\"table\" item not emitted")
+ }
+ if !sawTrailing {
+ t.Fatal("trailing paragraph content not present in concatenated items")
+ }
+}
+
+// TestMarkdownParser_NonTableHTMLBlockNotTable guards against the blanket
+// ck_type:"table" bug: a raw HTML block that is NOT a <table> (e.g. <div>,
+// <style>) must be emitted as ordinary text with no ck_type, so downstream
+// consumers (chunker) do not mistake it for a table.
+func TestMarkdownParser_NonTableHTMLBlockNotTable(t *testing.T) {
+ ctx := t.Context()
+ p, _ := NewMarkdownParser(GoMarkdown)
+ md := "Before.\n\n<div class=\"note\">just a div block</div>\n\n<style>.a{color:red}</style>\n\nAfter.\n"
+ res := p.ParseWithResult(ctx, "test.md", []byte(md))
+ if res.Err != nil {
+ t.Fatalf("ParseWithResult: %v", res.Err)
+ }
+ for _, item := range res.JSON {
+ text, _ := item["text"].(string)
+ if strings.Contains(text, "<div") || strings.Contains(text, "<style") {
+ if ck, ok := item["ck_type"].(string); ok && ck == "table" {
+ t.Fatalf("non-table HTML block wrongly tagged ck_type:\"table\": %q", text)
+ }
+ if kd, _ := item["doc_type_kwd"].(string); kd == "table" {
+ t.Fatalf("non-table HTML block wrongly tagged doc_type_kwd:\"table\": %q", text)
+ }
+ }
+ }
+}
+
+// TestMarkdownParser_TableInCodeFenceNotRendered guards against a regression
+// where pipe rows INSIDE a fenced code block would be mis-identified as a GFM
+// table and rewritten into <table> HTML, corrupting the code. renderMarkdownTablesInline
+// tracks fence state (inFence) so the table detector must skip lines inside a fence.
+func TestMarkdownParser_TableInCodeFenceNotRendered(t *testing.T) {
+ ctx := t.Context()
+ p, _ := NewMarkdownParser(GoMarkdown)
+ md := "# Title\n\n```\n| A | B |\n| --- | --- |\n| x | y |\n```\n\nAfter fence.\n"
+ res := p.ParseWithResult(ctx, "test.md", []byte(md))
+ if res.Err != nil {
+ t.Fatalf("ParseWithResult: %v", res.Err)
+ }
+ // No table item may be emitted — the pipe rows live inside a code block.
+ for _, item := range res.JSON {
+ if kd, _ := item["doc_type_kwd"].(string); kd == "table" {
+ t.Fatalf("code-fence pipe rows wrongly emitted as table item: %q", item["text"])
+ }
+ }
+ // The code block item must retain the raw pipe text and must NOT contain
+ // any <table> markup.
+ sawCode := false
+ for _, item := range res.JSON {
+ text, _ := item["text"].(string)
+ if strings.Contains(text, "| A | B |") {
+ sawCode = true
+ if strings.Contains(text, "<table") {
+ t.Fatalf("code block text was rewritten into table HTML: %q", text)
+ }
+ }
+ }
+ if !sawCode {
+ t.Fatal("code block with pipe rows not found in output")
+ }
+}
+
+// TestMarkdownParser_RawHTMLTableHandled covers a user-written raw <table> HTML
+// block (not a GFM pipe table). renderMarkdownTablesInline only rewrites GFM
+// pipe tables, so the raw <table> passes through and is caught by isTableHTML
+// as an HTMLBlock, producing the same inlined copy + separate doc_type_kwd:"table"
+// item shape as a GFM table.
+func TestMarkdownParser_RawHTMLTableHandled(t *testing.T) {
+ ctx := t.Context()
+ p, _ := NewMarkdownParser(GoMarkdown)
+ md := "Before.\n\n<table><tr><td>X</td><td>Y</td></tr></table>\n\nAfter.\n"
+ res := p.ParseWithResult(ctx, "test.md", []byte(md))
+ if res.Err != nil {
+ t.Fatalf("ParseWithResult: %v", res.Err)
+ }
+ sawInlined, sawTableItem := false, false
+ for _, item := range res.JSON {
+ text, _ := item["text"].(string)
+ switch kd, _ := item["doc_type_kwd"].(string); kd {
+ case "text":
+ if strings.Contains(text, "<table") && strings.Contains(text, "X") && strings.Contains(text, "Y") {
+ sawInlined = true
+ }
+ case "table":
+ sawTableItem = true
+ if !strings.Contains(text, "<table") {
+ t.Fatalf("raw table item text is not raw <table> HTML: %q", text)
+ }
+ if ck, _ := item["ck_type"].(string); ck != "table" {
+ t.Fatalf("raw table item ck_type = %q, want \"table\"", ck)
+ }
+ }
+ }
+ if !sawInlined {
+ t.Fatal("raw <table> HTML block not emitted as inlined copy in text flow")
+ }
+ if !sawTableItem {
+ t.Fatal("raw <table> HTML block not emitted as separate doc_type_kwd:\"table\" item")
+ }
+}
+
+// TestMarkdownParser_MultipleTablesOrdering guards the ordering contract: each
+// GFM table emits an inlined copy in its original document position (among the
+// surrounding text blocks) and a separate doc_type_kwd:"table" item appended at
+// the end of the stream (mirroring Python's _markdown, which appends tables
+// after all sections). Both tables' cell text must be present and in source order.
+func TestMarkdownParser_MultipleTablesOrdering(t *testing.T) {
+ ctx := t.Context()
+ p, _ := NewMarkdownParser(GoMarkdown)
+ md := "# Title\n\n| A | B |\n| --- | --- |\n| x | y |\n\nMiddle.\n\n| C | D |\n| --- | --- |\n| p | q |\n\nEnd.\n"
+ res := p.ParseWithResult(ctx, "test.md", []byte(md))
+ if res.Err != nil {
+ t.Fatalf("ParseWithResult: %v", res.Err)
+ }
+
+ var inlinedTableIdx []int
+ var tableItemIdx []int
+ var titleIdx, middleIdx, endIdx = -1, -1, -1
+ for i, item := range res.JSON {
+ text, _ := item["text"].(string)
+ switch kd, _ := item["doc_type_kwd"].(string); kd {
+ case "text":
+ switch text {
+ case "Title":
+ titleIdx = i
+ case "Middle.":
+ middleIdx = i
+ case "End.":
+ endIdx = i
+ }
+ if strings.Contains(text, "<table") {
+ inlinedTableIdx = append(inlinedTableIdx, i)
+ }
+ case "table":
+ tableItemIdx = append(tableItemIdx, i)
+ }
+ }
+
+ if titleIdx < 0 || middleIdx < 0 || endIdx < 0 {
+ t.Fatalf("missing anchor text item (title=%d middle=%d end=%d)", titleIdx, middleIdx, endIdx)
+ }
+ if len(inlinedTableIdx) != 2 {
+ t.Fatalf("inlined table copies = %d, want 2", len(inlinedTableIdx))
+ }
+ if len(tableItemIdx) != 2 {
+ t.Fatalf("separate table items = %d, want 2", len(tableItemIdx))
+ }
+ // Inlined copies appear in document order, bracketing "Middle.":
+ // table1 before Middle, table2 between Middle and End.
+ if !(inlinedTableIdx[0] < middleIdx && middleIdx < inlinedTableIdx[1] && inlinedTableIdx[1] < endIdx) {
+ t.Fatalf("inlined table order wrong: tables=%v middle=%d end=%d", inlinedTableIdx, middleIdx, endIdx)
+ }
+ // Separate table items are appended after all text items, in source order:
+ // table1 (x,y) then table2 (p,q), each strictly after the previous.
+ if !(tableItemIdx[0] > endIdx && tableItemIdx[0] < tableItemIdx[1]) {
+ t.Fatalf("table items not appended after text items in source order: %v end=%d", tableItemIdx, endIdx)
+ }
+ t1, _ := res.JSON[tableItemIdx[0]]["text"].(string)
+ t2, _ := res.JSON[tableItemIdx[1]]["text"].(string)
+ if !strings.Contains(t1, "x") || !strings.Contains(t1, "y") {
+ t.Fatalf("first table item missing x/y cells: %q", t1)
+ }
+ if !strings.Contains(t2, "p") || !strings.Contains(t2, "q") {
+ t.Fatalf("second table item missing p/q cells: %q", t2)
+ }
+}
+
+// TestMarkdownParser_AlignmentGolden verifies Go's ParseWithResult output is
+// content-equivalent to Python's _markdown on the shared sample, using the
+// shared concatenation-normalization alignment tool (align_test.go). Python
+// keeps raw markdown and splits on the delimiter set; Go emits clean per-block
+// text. The comparison normalizes both (markdown syntax, html tags, delimiters
+// stripped; whitespace collapsed) and ignores "table"/"image" items, which are
+// accepted representation differences (PARSER_ALIGNMENT_HANDOFF.md §3.1).
+//
+// Regenerate the baseline with:
+//
+// .venv/bin/python internal/parser/parser/testdata/gen_markdown_golden.py
+func TestMarkdownParser_AlignmentGolden(t *testing.T) {
+ ctx := t.Context()
+ p, _ := NewMarkdownParser(GoMarkdown)
+
+ sample, err := os.ReadFile("testdata/markdown.sample.md")
+ if err != nil {
+ t.Fatalf("read sample: %v", err)
+ }
+ res := p.ParseWithResult(ctx, "markdown.sample.md", sample)
+ if res.Err != nil {
+ t.Fatalf("ParseWithResult: %v", res.Err)
+ }
+
+ golden := LoadGolden(t, "testdata/markdown.python.golden.json")
+
+ // Ignore "table"/"image" items on both sides (accepted divergences).
+ goText := FilterByDocType(res.JSON, "text")
+ pyText := FilterByDocType(golden, "text")
+
+ if ok, diff := CompareAlignment(goText, pyText, MarkdownAlignOptions(DefaultMarkdownDelimiter)); !ok {
+ t.Fatalf("markdown parser not aligned with Python golden:%s", diff)
+ }
+}
diff --git a/internal/parser/parser/testdata/gen_markdown_golden.py b/internal/parser/parser/testdata/gen_markdown_golden.py
new file mode 100644
index 0000000000..fc0c113812
--- /dev/null
+++ b/internal/parser/parser/testdata/gen_markdown_golden.py
@@ -0,0 +1,84 @@
+#!/usr/bin/env python3
+"""Regenerate internal/parser/parser/testdata/markdown.python.golden.json.
+
+Drives the REAL Python markdown parser (deepdoc.parser.markdown_parser, the
+same engine rag/flow/parser/parser.py:_markdown delegates to via
+rag/app/naive.Markdown) so the golden is a faithful baseline rather than a
+hand approximation.
+
+Requires the project virtualenv (uv) because deepdoc needs markdown /
+beartype / etc.:
+
+ .venv/bin/python internal/parser/parser/testdata/gen_markdown_golden.py
+
+It mirrors _markdown with separate_tables=False and the default delimiter
+set, then assembles json items exactly as _markdown does:
+
+ * each extracted section -> {"text": <raw markdown section>, "doc_type_kwd": "text"}
+ * each standalone table -> {"text": <html table>, "doc_type_kwd": "table"}
+ * an image section -> {"text": <alt>, "doc_type_kwd": "image"}
+
+The Go alignment test then strips markdown syntax, html tags, and delimiters
+before comparing, and ignores "table"/"image" items (those representations are
+accepted divergences per PARSER_ALIGNMENT_HANDOFF.md §3.1).
+"""
+
+import json
+import os
+import re
+import sys
+
+# Make the repo root importable when run as a standalone script from testdata.
+# Script lives at <repo>/internal/parser/parser/testdata/, so five dirname hops.
+sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))))))
+
+SAMPLE = "internal/parser/parser/testdata/markdown.sample.md"
+OUT = "internal/parser/parser/testdata/markdown.python.golden.json"
+DELIM = "\n!?;。;!?"
+IMG_RE = re.compile(r"!\[([^\]]*)\]\(([^)]+)\)")
+SENTINEL = "@@IMAGE@@"
+
+
+def main():
+ from deepdoc.parser.markdown_parser import RAGFlowMarkdownParser, MarkdownElementExtractor
+
+ with open(SAMPLE, encoding="utf-8") as f:
+ raw = f.read()
+
+ # Model _markdown's return_section_images: the image is extracted as its
+ # own item (alt text only). Replace the markdown with a delimiter-free
+ # sentinel so the extractor does not split it.
+ alts = []
+
+ def _repl(m):
+ alts.append(m.group(1))
+ return SENTINEL
+
+ prepared = IMG_RE.sub(_repl, raw)
+
+ parser = RAGFlowMarkdownParser()
+ remainder, tables = parser.extract_tables_and_remainder(prepared + "\n", separate_tables=False)
+ extractor = MarkdownElementExtractor(remainder)
+ sections = extractor.extract_elements(DELIM, include_meta=True)
+
+ items = []
+ for s in sections:
+ content = s["content"]
+ if SENTINEL in content:
+ items.append({"text": alts.pop(0), "doc_type_kwd": "image"})
+ continue
+ items.append({"text": content, "doc_type_kwd": "text"})
+
+ for tbl in tables:
+ # _markdown (rag/flow/parser/parser.py:1103-1111) appends each
+ # extracted table as a duplicate "table" item even when inlined,
+ # carrying the table's raw text (GFM source or raw <table> HTML).
+ items.append({"text": tbl, "doc_type_kwd": "table"})
+
+ with open(OUT, "w", encoding="utf-8") as f:
+ json.dump(items, f, ensure_ascii=False, indent=2)
+ print("wrote %d items to %s" % (len(items), OUT))
+
+
+if __name__ == "__main__":
+ sys.exit(main())
diff --git a/internal/parser/parser/testdata/markdown.python.golden.json b/internal/parser/parser/testdata/markdown.python.golden.json
new file mode 100644
index 0000000000..73f5168110
--- /dev/null
+++ b/internal/parser/parser/testdata/markdown.python.golden.json
@@ -0,0 +1,50 @@
+[
+ {
+ "text": "# 健康检查套餐对比\n本文比较两种体检套餐,包含表格、列表与代码块",
+ "doc_type_kwd": "text"
+ },
+ {
+ "text": "## 套餐明细",
+ "doc_type_kwd": "text"
+ },
+ {
+ "text": "<table>\n<thead>\n<tr>\n<th>检查项目</th>\n<th>基础版 699 元</th>\n<th>进阶版 1299 元</th>\n</tr>\n</thead>\n<tbody>\n<tr>\n<td>血常规 / 尿常规</td>\n<td>包含</td>\n<td>包含</td>\n</tr>\n<tr>\n<td>心电图</td>\n<td>不包含</td>\n<td>包含</td>\n</tr>\n</tbody>\n</table>",
+ "doc_type_kwd": "text"
+ },
+ {
+ "text": "注意:所有套餐均需空腹",
+ "doc_type_kwd": "text"
+ },
+ {
+ "text": "## 注意事项",
+ "doc_type_kwd": "text"
+ },
+ {
+ "text": "- 体检前三天清淡饮食",
+ "doc_type_kwd": "text"
+ },
+ {
+ "text": "- 避免剧烈运动",
+ "doc_type_kwd": "text"
+ },
+ {
+ "text": "下面是示例配置:",
+ "doc_type_kwd": "text"
+ },
+ {
+ "text": "```yaml\nname: health-check\nversion: 1\n```",
+ "doc_type_kwd": "text"
+ },
+ {
+ "text": "示意图",
+ "doc_type_kwd": "image"
+ },
+ {
+ "text": "\n| 检查项目 | 基础版 699 元 | 进阶版 1299 元 |\n| --- | --- | --- |\n| 血常规 / 尿常规 | 包含 | 包含 |\n| 心电图 | 不包含 | 包含 |\n",
+ "doc_type_kwd": "table"
+ },
+ {
+ "text": "\n<table>\n<thead>\n<tr>\n<th>检查项目</th>\n<th>基础版 699 元</th>\n<th>进阶版 1299 元</th>\n</tr>\n</thead>\n<tbody>\n<tr>\n<td>血常规 / 尿常规</td>\n<td>包含</td>\n<td>包含</td>\n</tr>\n<tr>\n<td>心电图</td>\n<td>不包含</td>\n<td>包含</td>\n</tr>\n</tbody>\n</table>\n",
+ "doc_type_kwd": "table"
+ }
+]
diff --git a/internal/parser/parser/testdata/markdown.sample.md b/internal/parser/parser/testdata/markdown.sample.md
new file mode 100644
index 0000000000..7aec22ee26
--- /dev/null
+++ b/internal/parser/parser/testdata/markdown.sample.md
@@ -0,0 +1,26 @@
+# 健康检查套餐对比
+
+本文比较两种体检套餐,包含表格、列表与代码块。
+
+## 套餐明细
+
+| 检查项目 | 基础版 699 元 | 进阶版 1299 元 |
+| --- | --- | --- |
+| 血常规 / 尿常规 | 包含 | 包含 |
+| 心电图 | 不包含 | 包含 |
+
+注意:所有套餐均需空腹。
+
+## 注意事项
+
+- 体检前三天清淡饮食。
+- 避免剧烈运动!
+
+下面是示例配置:
+
+```yaml
+name: health-check
+version: 1
+```
+
+![示意图](data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAQAAAC1HAwCAAAAC0lEQVR42mNk+M8AAAMBAQDJ/pLvAAAAAElFTkSuQmCC)