RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.
Go agent-harnessagent-native-databaseagentic-aiagentic-retrievalagentic-searchaiai-agentscontext-enginecontext-engineeringcontext-managementharness-engineeringknowledge-compilationragretrieval-augmented-generationretrieval-harnesssearch-harness
Fix(parser): don't collapse markdown doc into one item when a table is present (#18014)
Jackd6f6b6231f9b
diff --git a/internal/parser/parser/align_test.go b/internal/parser/parser/align_test.gonew file mode 100644index 0000000000..d031f4f738--- /dev/null+++ b/internal/parser/parser/align_test.go@@ -0,0 +1,237 @@+//+// Copyright 2026 The InfiniFlow Authors. All Rights Reserved.+//+// Licensed under the Apache License, Version 2.0 (the "License");+// you may not use this file except in compliance with the License.+// You may obtain a copy of the License at+//+// http://www.apache.org/licenses/LICENSE-2.0+//+// Unless required by applicable law or agreed to in writing, software+// distributed under the License is distributed on an "AS IS" BASIS,+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+// Warranties, INCLUDING THE WARRANTIES OF MERCHANTABILITY AND+// FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE+// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER+// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING+// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER+// DEALINGS IN THE SOFTWARE.+//++package parser++import (+ "encoding/json"+ "os"+ "regexp"+ "strings"+ "testing"+)++// Normalizer transforms a single item's text before comparison. Normalizers+// are composed per parser type so the same comparison core is reused across+// every format (sessions A–E of the Go↔Python parser alignment).+type Normalizer func(string) string++// WithDelimiterStrip returns a Normalizer that replaces every rune present in+// delims with a single space. This normalizes the delimiter-split difference:+// Python splits the text at delimiters into separate items (so the delimiter+// becomes an item boundary, i.e. whitespace), while Go keeps the delimiter+// inline. Replacing with a space — rather than deleting — preserves the token+// separation on the Go side, so after CollapseWhitespace both sides yield the+// same space-joined text.+func WithDelimiterStrip(delims string) Normalizer {+ return func(s string) string {+ return strings.Map(func(r rune) rune {+ if strings.ContainsRune(delims, r) {+ return ' '+ }+ return r+ }, s)+ }+}++// CollapseWhitespace returns a Normalizer that trims and collapses runs of+// whitespace into a single space. Universal normalizer for tolerant compare.+func CollapseWhitespace() Normalizer {+ return func(s string) string {+ return strings.Join(strings.Fields(s), " ")+ }+}++// htmlTagRE matches an HTML tag so table/HTML markup can be ignored when+// comparing content across the two markdown libraries (goldmark vs+// Python-Markdown serialize tables differently but the cell text is the same).+var htmlTagRE = regexp.MustCompile(`(?is)<[^>]+>`)++// StripHTMLTags returns a Normalizer that removes HTML tags, leaving the+// visible text. Tags are replaced with a single space (not deleted) so+// adjacent cell text does not fuse — e.g. "<td>A</td><td>B</td>" becomes+// "A B" rather than "AB". CollapseWhitespace then folds the extra space.+// Used so table-markup differences between markdown libraries don't mask the+// underlying content equivalence.+func StripHTMLTags() Normalizer {+ return func(s string) string {+ return htmlTagRE.ReplaceAllString(s, " ")+ }+}++// Markdown-syntax regexes removed by StripMarkdownSyntax. The Python flow+// parser keeps raw markdown in its section text ("# Title", "- item",+// ``` fenced ```); the Go parser emits clean per-block text. These are+// representation differences (PARSER_ALIGNMENT_HANDOFF.md §3.1), not content+// divergences, so the markdown alignment strips them before comparing.+var (+ mdHeaderRE = regexp.MustCompile(`(?m)^#{1,6}\s+`)+ mdListRE = regexp.MustCompile(`(?m)^\s*[-*+]\s+`)+ mdFenceRE = regexp.MustCompile("(?s)```[^\n]*\n(.*?)```")+)++// StripMarkdownSyntax returns a Normalizer that removes markdown presentation+// characters (ATX headings, list bullets, fenced-code fences) from a section,+// leaving the bare text. It must run before CollapseWhitespace because the+// fence regex relies on the surrounding newlines.+func StripMarkdownSyntax() Normalizer {+ return func(s string) string {+ s = mdHeaderRE.ReplaceAllString(s, "")+ s = mdListRE.ReplaceAllString(s, "")+ s = mdFenceRE.ReplaceAllString(s, "$1")+ return s+ }+}++// FilterByDocType returns only the items whose doc_type_kwd equals kwd.+// Python emits duplicate table items (separate_tables=False still appends+// them) — excluding doc_type_kwd:"table" lets the comparison focus on the+// inlined textual content, which is what Go produces.+func FilterByDocType(items []map[string]any, kwd string) []map[string]any {+ out := make([]map[string]any, 0, len(items))+ for _, it := range items {+ if v, _ := it["doc_type_kwd"].(string); v == kwd {+ out = append(out, it)+ }+ }+ return out+}++// AlignOptions configures NormalizeConcat / CompareAlignment.+type AlignOptions struct {+ // Normalizers applied (in order) to each item's text before concat.+ Normalizers []Normalizer+ // ItemKey is the field holding the compared text (default "text").+ ItemKey string+}++func alignItemText(item map[string]any, key string) string {+ if key == "" {+ key = "text"+ }+ if v, ok := item[key].(string); ok {+ return v+ }+ return ""+}++// NormalizeConcat extracts the text field from each item, applies the+// normalizers in order, and concatenates into one string. Format-agnostic.+//+// Items are joined with a single space (after whitespace is collapsed by the+// normalizers) rather than by newlines: Go emits one item per top-level block+// while Python splits the same text on delimiters into many smaller items, so+// the item *boundaries* legitimately differ. Joining on whitespace makes the+// comparison boundary-agnostic — only the concatenated content (order+// preserved) is compared, which is exactly the alignment guarantee we want.+// Empty items are skipped so Python's trailing/duplicate segments don't mask a+// real content difference.+func NormalizeConcat(items []map[string]any, opts AlignOptions) string {+ key := opts.ItemKey+ if key == "" {+ key = "text"+ }+ parts := make([]string, 0, len(items))+ for _, it := range items {+ t := alignItemText(it, key)+ for _, n := range opts.Normalizers {+ t = n(t)+ }+ if strings.TrimSpace(t) == "" {+ continue+ }+ // Trim so a delimiter turned into a trailing space (WithDelimiterStrip)+ // doesn't combine with the join space into a double gap.+ t = strings.TrimSpace(t)+ parts = append(parts, t)+ }+ return strings.Join(parts, " ")+}++// CompareAlignment reports whether two parser outputs are aligned after+// normalization. goItems come from Go's ParseResult.JSON; pyItems come from the+// Python golden JSON. Returns (equal, diffReport).+func CompareAlignment(goItems, pyItems []map[string]any, opts AlignOptions) (bool, string) {+ g := NormalizeConcat(goItems, opts)+ p := NormalizeConcat(pyItems, opts)+ if g == p {+ return true, ""+ }+ return false, diffReport(g, p)+}++func diffReport(g, p string) string {+ const max = 2000+ if len(g) > max {+ g = g[:max] + "...(truncated)"+ }+ if len(p) > max {+ p = p[:max] + "...(truncated)"+ }+ return "alignment mismatch after normalization:\n--- GO ---\n" + g + "\n--- PY ---\n" + p+}++// LoadGolden reads a Python golden JSON file (a JSON list of item objects)+// produced by the Python flow parser for the same input.+func LoadGolden(t *testing.T, path string) []map[string]any {+ t.Helper()+ data, err := os.ReadFile(path)+ if err != nil {+ t.Fatalf("load golden %s: %v", path, err)+ }+ var items []map[string]any+ if err := json.Unmarshal(data, &items); err != nil {+ t.Fatalf("parse golden %s: %v", path, err)+ }+ return items+}++// MarkdownAlignOptions returns the normalizer preset for markdown. The order+// matters:+// - StripMarkdownSyntax first: drops "#"/"-"/fenced-code markup that Python+// keeps inline but Go parses out (relies on the surrounding newlines, so it+// must run before CollapseWhitespace).+// - StripHTMLTags next: replace table/HTML tags with a space (not delete) so+// adjacent cell text does not fuse, e.g. "<td>A</td><td>B</td>" → "A B".+// - WithDelimiterStrip: replace the delimiter set Python consumes at split+// points with a space while Go keeps it inline, so both sides keep the same+// token separation. Runs before CollapseWhitespace so the introduced space+// is folded normally.+// - CollapseWhitespace last: folds all remaining internal whitespace (the+// inter-tag gaps of an HTML table, the space from delimiter replacement,+// the newlines inside a fenced code block) into single spaces.+//+// Reused by every markdown alignment test; other formats define their own+// preset and share CompareAlignment.+func MarkdownAlignOptions(delimiter string) AlignOptions {+ return AlignOptions{+ Normalizers: []Normalizer{+ StripMarkdownSyntax(),+ StripHTMLTags(),+ WithDelimiterStrip(delimiter),+ CollapseWhitespace(),+ },+ ItemKey: "text",+ }+}++// DefaultMarkdownDelimiter is the flow parser's default markdown delimiter+// set, used when generating/loading the golden baseline.+const DefaultMarkdownDelimiter = "\n!?;。;!?"diff --git a/internal/parser/parser/markdown_parser.go b/internal/parser/parser/markdown_parser.goindex 54f9f838f0..e41897d152 100644--- a/internal/parser/parser/markdown_parser.go+++ b/internal/parser/parser/markdown_parser.go@@ -25,7 +25,6 @@ import ("net""net/http""net/url"- "regexp""strings""sync""time"@@ -35,9 +34,6 @@ import (mdparser "github.com/gomarkdown/markdown/parser")-// mdImagePattern matches markdown inline image syntax: .-var mdImagePattern = regexp.MustCompile(`!\[[^\]]*\]\(([^)\s]+)\)`)-// dataURIPrefix is the MIME prefix for data URI images.const dataURIPrefix = "data:image/"@@ -96,20 +92,18 @@ func (p *MarkdownParser) ConfigureFromSetup(setup map[string]any) {// been removed; callers consume ParseResult directly.func (p *MarkdownParser) ParseWithResult(ctx context.Context, filename string, data []byte) ParseResult {rawText := string(data)- if rendered, ok := renderMarkdownTablesInline(rawText); ok {- return ParseResult{- OutputFormat: "json",- File: map[string]any{- "name": filename,- },- JSON: []map[string]any{{"text": rendered, "doc_type_kwd": "text"}},- }- }+ // Render any GFM/HTML table inline as an HTML block before parsing. This+ // keeps the document as one item per top-level block (the table becomes a+ // normal text item) instead of collapsing the whole document into a single+ // item. The result mirrors Python's `_markdown` (separate_tables=False),+ // which also inlines tables into the surrounding text. When no table is+ // present renderMarkdownTablesInlineText returns the input unchanged.+ rendered := renderMarkdownTablesInlineText(rawText)- doc := markdownNew().Parse(data)+ doc := markdownNew().Parse([]byte(rendered))var items []map[string]any- walkMarkdownBlocksWithImages(doc, rawText, &items, p.FlattenMediaToText)+ walkMarkdownBlocksWithImages(doc, &items, p.FlattenMediaToText)if items == nil {items = []map[string]any{{"text": "", "doc_type_kwd": "text"}}}@@ -165,8 +159,13 @@ func renderMarkdownTablesInline(text string) (string, bool) {i++}tableHTML := markdownlib.ToHTML([]byte(strings.Join(lines[start:i], "")), markdownNew(), nil)+ // Wrap the inlined <table> HTML in blank lines so gomarkdown+ // keeps it as a single HTML block (one item) instead of+ // re-parsing it into scattered cell text. See+ // PARSER_ALIGNMENT_HANDOFF.md §3.1 (markdown session A, 方案 Y).+ ensureTrailingBlankLine(&buf)buf.WriteString(strings.TrimRight(string(tableHTML), "\r\n"))- buf.WriteByte('\n')+ buf.WriteString("\n\n")changed = truecontinue}@@ -176,6 +175,32 @@ func renderMarkdownTablesInline(text string) (string, bool) {return buf.String(), changed}+// renderMarkdownTablesInlineText renders every GFM/HTML table inline as an+// HTML block and returns the rewritten text. When no table is present the+// input is returned unchanged. Unlike renderMarkdownTablesInline it always+// returns the full text (ignoring the changed flag) so callers can parse the+// result uniformly and emit one item per top-level block.+func renderMarkdownTablesInlineText(text string) string {+ out, _ := renderMarkdownTablesInline(text)+ return out+}++// ensureTrailingBlankLine makes sure b ends with a blank line (two+// newlines) so the next block is separated from what precedes it. gomarkdown+// only treats a <table> as a standalone HTML block (rather than re-parsing it+// into scattered cell nodes) when it is surrounded by blank lines.+func ensureTrailingBlankLine(b *strings.Builder) {+ s := b.String()+ switch {+ case strings.HasSuffix(s, "\n\n"):+ // already separated.+ case strings.HasSuffix(s, "\n"):+ b.WriteByte('\n')+ default:+ b.WriteString("\n\n")+ }+}+func markdownFenceMarker(line string) (byte, int, bool) {trimmed := strings.TrimLeft(line, " \t")if len(line)-len(trimmed) > 3 || len(trimmed) < 3 {@@ -236,11 +261,25 @@ func markdownTableCells(line string) []string {// top-level block. Headings, paragraphs, lists, and code blocks are// emitted with their text. When a block contains a markdown image// reference (), the image data is resolved via-// resolveMarkdownImage and the item carries `doc_type_kwd: "image"`-// together with the base64-encoded image payload. When flatten is-// true, all items are forced to doc_type_kwd="text" (mirrors Python-// parser.py:1034 flatten_media_to_text).-func walkMarkdownBlocksWithImages(doc ast.Node, rawText string, out *[]map[string]any, flatten bool) {+// findBlockImage (per-block AST walk) and the item carries+// `doc_type_kwd: "image"` together with the base64-encoded image+// payload. When flatten is true, all items are forced to+// doc_type_kwd="text" (mirrors Python parser.py:1034+// flatten_media_to_text).+//+// Tables: a GFM/HTML table is rendered inline as a single <table> HTML+// block by renderMarkdownTablesInlineText and kept as one HTML block.+// It is emitted as TWO items, mirroring Python's _markdown+// (separate_tables=False): an inlined copy in the text flow+// (doc_type_kwd:"text") and a separate structured table item+// (doc_type_kwd:"table", ck_type:"table"). The downstream chunker+// consumes doc_type_kwd:"table" to keep the table whole and attach+// table context to neighbouring chunks (chunker/token.go). Non-table+// HTML blocks (<div>, <style>, …) are emitted as ordinary text with+// no ck_type. Table items are appended after the walk so the order+// matches Python (_markdown appends tables after all sections).+func walkMarkdownBlocksWithImages(doc ast.Node, out *[]map[string]any, flatten bool) {+ var tableItems []map[string]anyfor _, child := range doc.GetChildren() {var ckType stringvar docTypeKwd string@@ -263,6 +302,33 @@ func walkMarkdownBlocksWithImages(doc ast.Node, rawText string, out *[]map[strintxt = leafText(n)ckType = "code"docTypeKwd = "text"+ case *ast.HTMLBlock:+ // An HTML block is either an inlined table (kept as a single+ // HTML block thanks to the blank lines renderMarkdownTablesInline+ // wraps around it) or a plain HTML block such as <div>/<style>.+ // Only a table is emitted as a structured table item; everything+ // else is treated as ordinary text (no ck_type).+ txt = leafText(n)+ if isTableHTML(txt) {+ // Inlined copy kept in the text flow. This is what the+ // alignment golden compares against (Python inlines the+ // rendered <table> HTML into a text section).+ *out = append(*out, map[string]any{+ "text": txt,+ "doc_type_kwd": "text",+ })+ // Separate structured table item (mirrors Python's extra+ // doc_type_kwd:"table" item). doc_type_kwd:"table" drives+ // downstream table handling.+ tableItems = append(tableItems, map[string]any{+ "text": txt,+ "doc_type_kwd": "table",+ "ck_type": "table",+ })+ continue+ }+ // Non-table HTML block: ordinary text, no ck_type.+ docTypeKwd = "text"default:txt = leafText(n)if strings.TrimSpace(txt) == "" {@@ -279,52 +345,78 @@ func walkMarkdownBlocksWithImages(doc ast.Node, rawText string, out *[]map[strinitem["ck_type"] = ckType}- // Detect markdown image references in the raw source text- // that corresponds to this block. When found, resolve the- // image data so downstream vision enhancement can describe it.- // When flatten is true, keep doc_type_kwd="text" (Python+ // Resolve markdown images from the AST node of THIS block only, so+ // the image payload (and doc_type_kwd:"image") is attached to the+ // single block that actually contains the  reference.+ // Scanning the whole document (the old approach) wrongly tagged+ // every block as an image whenever any image was present. When+ // flatten is true, keep doc_type_kwd="text" (Python// parser.py:1034: flatten_media_to_text overrides image).- if imgData, imgFound := resolveMarkdownImage(txt, rawText); imgFound && imgData != "" {- item["image"] = imgData- if !flatten {- item["doc_type_kwd"] = "image"+ if imgURL, ok := findBlockImage(child); ok {+ if imgData, resolved := resolveImageURL(imgURL); resolved && imgData != "" {+ item["image"] = imgData+ if !flatten {+ item["doc_type_kwd"] = "image"+ }}}*out = append(*out, item)}+ // Tables appended last, mirroring Python's _markdown ordering.+ *out = append(*out, tableItems...)+}++// isTableHTML reports whether block text is an outer <table> element (the+// inlined GFM/HTML table). Only such blocks are emitted as structured table+// items; other raw HTML (e.g. <div>, <style>) is plain text.+func isTableHTML(s string) bool {+ return strings.HasPrefix(strings.TrimSpace(strings.ToLower(s)), "<table")+}++// findBlockImage returns the destination URL of the first image node found+// anywhere under n. This associates an image with the specific block that+// contains it, instead of scanning the whole document (which would wrongly+// tag every block as an image when any image is present).+func findBlockImage(n ast.Node) (string, bool) {+ found := false+ var url string+ var walk func(c ast.Node)+ walk = func(c ast.Node) {+ if found || c == nil {+ return+ }+ if img, ok := c.(*ast.Image); ok {+ url = string(img.Destination)+ found = true+ return+ }+ for _, ch := range c.GetChildren() {+ walk(ch)+ }+ }+ walk(n)+ return url, found}-// resolveMarkdownImage extracts the first markdown image reference-// from the given text and returns its base64-encoded data. Supports:+// resolveImageURL resolves a markdown image URL to its base64-encoded data.+// Supports:// - data:image/... URIs → decoded directly// - http:// / https:// URLs → fetched (with basic SSRF filtering)//-// Returns (base64String, true) on success, ("", false) when no image-// is found or resolution fails.-func resolveMarkdownImage(leafText, rawFullText string) (string, bool) {- // Prefer matching against the raw full text to catch images- // whose alt-text was split across markdown rendering.- searchIn := rawFullText- if strings.TrimSpace(searchIn) == "" {- searchIn = leafText- }- matches := mdImagePattern.FindStringSubmatch(searchIn)- if len(matches) < 2 {- return "", false- }- url := matches[1]-- if strings.HasPrefix(url, dataURIPrefix) {+// Local / relative paths are not fetched (security). Returns+// (base64String, true) on success, ("", false) when resolution fails.+func resolveImageURL(imageURL string) (string, bool) {+ if strings.HasPrefix(imageURL, dataURIPrefix) {// data:image/png;base64,xxxx- idx := strings.Index(url, "base64,")+ idx := strings.Index(imageURL, "base64,")if idx < 0 {return "", false}- return url[idx+len("base64,"):], true+ return imageURL[idx+len("base64,"):], true}- if strings.HasPrefix(url, "http://") || strings.HasPrefix(url, "https://") {- b64, err := fetchImageAsBase64(url)+ if strings.HasPrefix(imageURL, "http://") || strings.HasPrefix(imageURL, "https://") {+ b64, err := fetchImageAsBase64(imageURL)if err != nil {return "", false}@@ -481,6 +573,17 @@ func walkLeaf(n ast.Node, buf *bytes.Buffer) {buf.Write(t.Literal)case *ast.Code:buf.Write(t.Literal)+ case *ast.CodeBlock:+ // finalizeCodeBlock moves the fenced body into Literal and nils+ // Content; the indented form keeps it in Content. Emit both so the+ // code text is never dropped.+ buf.Write(t.Literal)+ buf.Write(t.Content)+ case *ast.HTMLBlock:+ // Inlined tables are HTML blocks. The generic parser path stores the+ // markup in Content, the markdown-block path in Literal. Emit both.+ buf.Write(t.Literal)+ buf.Write(t.Content)default:for _, c := range n.GetChildren() {walkLeaf(c, buf)diff --git a/internal/parser/parser/markdown_parser_test.go b/internal/parser/parser/markdown_parser_test.goindex 049b9b12ad..34ab5693a3 100644--- a/internal/parser/parser/markdown_parser_test.go+++ b/internal/parser/parser/markdown_parser_test.go@@ -4,6 +4,7 @@ import ("encoding/base64""net/http""net/http/httptest"+ "os""strings""testing")@@ -97,21 +98,32 @@ func TestMarkdownParser_ParseWithResult_RendersTableInline(t *testing.T) {if res.Err != nil {t.Fatalf("ParseWithResult: %v", res.Err)}- if len(res.JSON) != 1 {- t.Fatalf("len(JSON) = %d, want 1", len(res.JSON))- }- text, _ := res.JSON[0]["text"].(string)- if got, _ := res.JSON[0]["doc_type_kwd"].(string); got != "text" {- t.Fatalf("doc_type_kwd = %q, want text", got)- }- if !strings.Contains(text, "<table>") || !strings.Contains(text, "<th>Check item</th>") {- t.Fatalf("table was not rendered inline: %q", text)+ // 方案 Y: a document containing a table must NOT collapse into a single+ // item. Each top-level block is its own item, and the table is emitted as+ // both an inlined text copy (doc_type_kwd:"text") and a separate+ // structured table item (doc_type_kwd:"table"), so the item count is > 1.+ if len(res.JSON) < 2 {+ t.Fatalf("len(JSON) = %d, want >= 2 (table must not collapse the doc)", len(res.JSON))}- if strings.Contains(text, "| Check item |") {- t.Fatalf("raw markdown table leaked into text: %q", text)+ var all strings.Builder+ for _, item := range res.JSON {+ kd, _ := item["doc_type_kwd"].(string)+ if kd != "text" && kd != "table" {+ t.Fatalf("unexpected doc_type_kwd %q; want text or table", kd)+ }+ text, _ := item["text"].(string)+ all.WriteString(text)+ all.WriteString("\n")+ if strings.Contains(text, "| Check item |") {+ t.Fatalf("raw markdown table leaked into text: %q", text)+ }}- if gap := text[strings.Index(text, "</table>"):strings.Index(text, "Note:")]; gap != "</table>\n" {- t.Fatalf("gap after table = %q, want %q", gap, "</table>\n")+ // The table is kept as raw <table>…</table> HTML (one HTML block), so the+ // cell text survives as text inside the tags — possibly duplicated across+ // the inlined copy and the table item. Check the concatenated text.+ concat := all.String()+ if !strings.Contains(concat, "Check item") || !strings.Contains(concat, "Blood routine") {+ t.Fatalf("table cell content (Check item / Blood routine) not present: %q", concat)}}@@ -183,10 +195,9 @@ func TestMarkdownParser_FlattenMediaToText(t *testing.T) {}}-func TestResolveMarkdownImage_DataURI(t *testing.T) {+func TestResolveImageURL_DataURI(t *testing.T) {b64 := base64.StdEncoding.EncodeToString([]byte("fakeimage"))- md := ""- result, found := resolveMarkdownImage("", md)+ result, found := resolveImageURL("data:image/png;base64," + b64)if !found {t.Fatal("expected image found for data URI")}@@ -195,15 +206,14 @@ func TestResolveMarkdownImage_DataURI(t *testing.T) {}}-func TestResolveMarkdownImage_NoImage(t *testing.T) {- _, found := resolveMarkdownImage("", "# Hello\nNo image here")- if found {- t.Fatal("expected no image found")+func TestResolveImageURL_LocalPathNotFetched(t *testing.T) {+ // Local / relative paths are not fetched (security); resolution fails.+ if _, found := resolveImageURL("./local/image.png"); found {+ t.Fatal("expected no image resolved for a local path")}}-func TestResolveMarkdownImage_HTTPImage(t *testing.T) {- withSSRFBypass(t)+func TestResolveImageURL_HTTPImage(t *testing.T) {// httptest servers bind loopback, which the SSRF guard rejects by// default. Allow loopback for this test so the HTTP fetch path is// exercised (production keeps ssrfAllowLoopback == false).@@ -211,15 +221,13 @@ func TestResolveMarkdownImage_HTTPImage(t *testing.T) {ssrfAllowLoopback = truedefer func() { ssrfAllowLoopback = prev }()- // Start a test HTTP server serving a fake PNGts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {w.Header().Set("Content-Type", "image/png")w.Write([]byte("fake-png-bytes"))}))defer ts.Close()- md := ""- result, found := resolveMarkdownImage("", md)+ result, found := resolveImageURL(ts.URL + "/image.png")if !found {t.Fatal("expected image found for HTTP URL")}@@ -229,6 +237,45 @@ func TestResolveMarkdownImage_HTTPImage(t *testing.T) {}}+// TestFindBlockImage resolves the image per-block from the AST so only the+// owning block is tagged (the fix for the whole-document scan bug).+func TestFindBlockImage(t *testing.T) {+ ctx := t.Context()+ p, _ := NewMarkdownParser(GoMarkdown)++ withImg := "# T\n\nText without image.\n\n\n"+ res := p.ParseWithResult(ctx, "a.md", []byte(withImg))+ if res.Err != nil {+ t.Fatalf("ParseWithResult: %v", res.Err)+ }+ var imgItems, textItems int+ for _, item := range res.JSON {+ switch kd, _ := item["doc_type_kwd"].(string); kd {+ case "image":+ imgItems+++ if _, ok := item["image"].(string); !ok {+ t.Fatal("image item missing base64 payload")+ }+ case "text":+ textItems+++ }+ }+ if imgItems != 1 {+ t.Fatalf("imgItems = %d, want 1 (only the block owning the image)", imgItems)+ }+ if textItems < 2 {+ t.Fatalf("textItems = %d, want >= 2 (other blocks must stay text, not image)", textItems)+ }++ // A document with no image must not produce any image item.+ noImg := p.ParseWithResult(ctx, "b.md", []byte("# T\n\nNo images here.\n"))+ for _, item := range noImg.JSON {+ if kd, _ := item["doc_type_kwd"].(string); kd == "image" {+ t.Fatal("unexpected image item in text-only markdown")+ }+ }+}+func TestFetchImageAsBase64_RejectsCredentials(t *testing.T) {_, err := fetchImageAsBase64("https://user:[email protected]/img.png")if err == nil {@@ -248,3 +295,265 @@ func TestFetchImageAsBase64_InvalidURL(t *testing.T) {t.Fatal("expected error for 404 response")}}++// TestMarkdownParser_TableNotCollapsed is the core regression guard for the+// markdown table fix (方案 Y): a document containing a GFM table must keep one+// item per top-level block instead of collapsing into one giant item, AND the+// table must be emitted as a single raw <table>…</table> HTML block (not+// scattered cell text), appearing both as an inlined text copy and as a+// separate doc_type_kwd:"table" item. A heading, the table, and the trailing+// paragraph must each be present.+func TestMarkdownParser_TableNotCollapsed(t *testing.T) {+ ctx := t.Context()+ p, _ := NewMarkdownParser(GoMarkdown)+ md := "# Title\n\nIntro paragraph.\n\n| A | B |\n| --- | --- |\n| x | y |\n\nTrailing note.\n"+ res := p.ParseWithResult(ctx, "test.md", []byte(md))+ if res.Err != nil {+ t.Fatalf("ParseWithResult: %v", res.Err)+ }+ // Title, Intro, inlined table copy, separate table item, Trailing = 5.+ if len(res.JSON) < 5 {+ t.Fatalf("len(JSON) = %d, want >= 5 (one per top-level block + table item)", len(res.JSON))+ }+ var all strings.Builder+ sawTitle, sawTrailing, sawTableItem, sawRawTableHTML := false, false, false, false+ for _, item := range res.JSON {+ text, _ := item["text"].(string)+ all.WriteString(text)+ all.WriteString("\n")+ if text == "Title" {+ sawTitle = true+ }+ switch kd, _ := item["doc_type_kwd"].(string); kd {+ case "text":+ // inlined copy carries the raw <table> HTML with cell text.+ if strings.Contains(text, "<table") && strings.Contains(text, "A") && strings.Contains(text, "B") {+ sawRawTableHTML = true+ }+ case "table":+ sawTableItem = true+ if !strings.Contains(text, "<table") {+ t.Fatalf("table item text is not raw <table> HTML: %q", text)+ }+ if ck, _ := item["ck_type"].(string); ck != "table" {+ t.Fatalf("table item ck_type = %q, want \"table\"", ck)+ }+ }+ }+ concat := all.String()+ if strings.Contains(concat, "Trailing note.") {+ sawTrailing = true+ }+ if !sawTitle {+ t.Fatal("heading item not emitted")+ }+ if !sawRawTableHTML {+ t.Fatal("inlined table not emitted as raw <table> HTML with cell text")+ }+ if !sawTableItem {+ t.Fatal("separate doc_type_kwd:\"table\" item not emitted")+ }+ if !sawTrailing {+ t.Fatal("trailing paragraph content not present in concatenated items")+ }+}++// TestMarkdownParser_NonTableHTMLBlockNotTable guards against the blanket+// ck_type:"table" bug: a raw HTML block that is NOT a <table> (e.g. <div>,+// <style>) must be emitted as ordinary text with no ck_type, so downstream+// consumers (chunker) do not mistake it for a table.+func TestMarkdownParser_NonTableHTMLBlockNotTable(t *testing.T) {+ ctx := t.Context()+ p, _ := NewMarkdownParser(GoMarkdown)+ md := "Before.\n\n<div class=\"note\">just a div block</div>\n\n<style>.a{color:red}</style>\n\nAfter.\n"+ res := p.ParseWithResult(ctx, "test.md", []byte(md))+ if res.Err != nil {+ t.Fatalf("ParseWithResult: %v", res.Err)+ }+ for _, item := range res.JSON {+ text, _ := item["text"].(string)+ if strings.Contains(text, "<div") || strings.Contains(text, "<style") {+ if ck, ok := item["ck_type"].(string); ok && ck == "table" {+ t.Fatalf("non-table HTML block wrongly tagged ck_type:\"table\": %q", text)+ }+ if kd, _ := item["doc_type_kwd"].(string); kd == "table" {+ t.Fatalf("non-table HTML block wrongly tagged doc_type_kwd:\"table\": %q", text)+ }+ }+ }+}++// TestMarkdownParser_TableInCodeFenceNotRendered guards against a regression+// where pipe rows INSIDE a fenced code block would be mis-identified as a GFM+// table and rewritten into <table> HTML, corrupting the code. renderMarkdownTablesInline+// tracks fence state (inFence) so the table detector must skip lines inside a fence.+func TestMarkdownParser_TableInCodeFenceNotRendered(t *testing.T) {+ ctx := t.Context()+ p, _ := NewMarkdownParser(GoMarkdown)+ md := "# Title\n\n```\n| A | B |\n| --- | --- |\n| x | y |\n```\n\nAfter fence.\n"+ res := p.ParseWithResult(ctx, "test.md", []byte(md))+ if res.Err != nil {+ t.Fatalf("ParseWithResult: %v", res.Err)+ }+ // No table item may be emitted — the pipe rows live inside a code block.+ for _, item := range res.JSON {+ if kd, _ := item["doc_type_kwd"].(string); kd == "table" {+ t.Fatalf("code-fence pipe rows wrongly emitted as table item: %q", item["text"])+ }+ }+ // The code block item must retain the raw pipe text and must NOT contain+ // any <table> markup.+ sawCode := false+ for _, item := range res.JSON {+ text, _ := item["text"].(string)+ if strings.Contains(text, "| A | B |") {+ sawCode = true+ if strings.Contains(text, "<table") {+ t.Fatalf("code block text was rewritten into table HTML: %q", text)+ }+ }+ }+ if !sawCode {+ t.Fatal("code block with pipe rows not found in output")+ }+}++// TestMarkdownParser_RawHTMLTableHandled covers a user-written raw <table> HTML+// block (not a GFM pipe table). renderMarkdownTablesInline only rewrites GFM+// pipe tables, so the raw <table> passes through and is caught by isTableHTML+// as an HTMLBlock, producing the same inlined copy + separate doc_type_kwd:"table"+// item shape as a GFM table.+func TestMarkdownParser_RawHTMLTableHandled(t *testing.T) {+ ctx := t.Context()+ p, _ := NewMarkdownParser(GoMarkdown)+ md := "Before.\n\n<table><tr><td>X</td><td>Y</td></tr></table>\n\nAfter.\n"+ res := p.ParseWithResult(ctx, "test.md", []byte(md))+ if res.Err != nil {+ t.Fatalf("ParseWithResult: %v", res.Err)+ }+ sawInlined, sawTableItem := false, false+ for _, item := range res.JSON {+ text, _ := item["text"].(string)+ switch kd, _ := item["doc_type_kwd"].(string); kd {+ case "text":+ if strings.Contains(text, "<table") && strings.Contains(text, "X") && strings.Contains(text, "Y") {+ sawInlined = true+ }+ case "table":+ sawTableItem = true+ if !strings.Contains(text, "<table") {+ t.Fatalf("raw table item text is not raw <table> HTML: %q", text)+ }+ if ck, _ := item["ck_type"].(string); ck != "table" {+ t.Fatalf("raw table item ck_type = %q, want \"table\"", ck)+ }+ }+ }+ if !sawInlined {+ t.Fatal("raw <table> HTML block not emitted as inlined copy in text flow")+ }+ if !sawTableItem {+ t.Fatal("raw <table> HTML block not emitted as separate doc_type_kwd:\"table\" item")+ }+}++// TestMarkdownParser_MultipleTablesOrdering guards the ordering contract: each+// GFM table emits an inlined copy in its original document position (among the+// surrounding text blocks) and a separate doc_type_kwd:"table" item appended at+// the end of the stream (mirroring Python's _markdown, which appends tables+// after all sections). Both tables' cell text must be present and in source order.+func TestMarkdownParser_MultipleTablesOrdering(t *testing.T) {+ ctx := t.Context()+ p, _ := NewMarkdownParser(GoMarkdown)+ md := "# Title\n\n| A | B |\n| --- | --- |\n| x | y |\n\nMiddle.\n\n| C | D |\n| --- | --- |\n| p | q |\n\nEnd.\n"+ res := p.ParseWithResult(ctx, "test.md", []byte(md))+ if res.Err != nil {+ t.Fatalf("ParseWithResult: %v", res.Err)+ }++ var inlinedTableIdx []int+ var tableItemIdx []int+ var titleIdx, middleIdx, endIdx = -1, -1, -1+ for i, item := range res.JSON {+ text, _ := item["text"].(string)+ switch kd, _ := item["doc_type_kwd"].(string); kd {+ case "text":+ switch text {+ case "Title":+ titleIdx = i+ case "Middle.":+ middleIdx = i+ case "End.":+ endIdx = i+ }+ if strings.Contains(text, "<table") {+ inlinedTableIdx = append(inlinedTableIdx, i)+ }+ case "table":+ tableItemIdx = append(tableItemIdx, i)+ }+ }++ if titleIdx < 0 || middleIdx < 0 || endIdx < 0 {+ t.Fatalf("missing anchor text item (title=%d middle=%d end=%d)", titleIdx, middleIdx, endIdx)+ }+ if len(inlinedTableIdx) != 2 {+ t.Fatalf("inlined table copies = %d, want 2", len(inlinedTableIdx))+ }+ if len(tableItemIdx) != 2 {+ t.Fatalf("separate table items = %d, want 2", len(tableItemIdx))+ }+ // Inlined copies appear in document order, bracketing "Middle.":+ // table1 before Middle, table2 between Middle and End.+ if !(inlinedTableIdx[0] < middleIdx && middleIdx < inlinedTableIdx[1] && inlinedTableIdx[1] < endIdx) {+ t.Fatalf("inlined table order wrong: tables=%v middle=%d end=%d", inlinedTableIdx, middleIdx, endIdx)+ }+ // Separate table items are appended after all text items, in source order:+ // table1 (x,y) then table2 (p,q), each strictly after the previous.+ if !(tableItemIdx[0] > endIdx && tableItemIdx[0] < tableItemIdx[1]) {+ t.Fatalf("table items not appended after text items in source order: %v end=%d", tableItemIdx, endIdx)+ }+ t1, _ := res.JSON[tableItemIdx[0]]["text"].(string)+ t2, _ := res.JSON[tableItemIdx[1]]["text"].(string)+ if !strings.Contains(t1, "x") || !strings.Contains(t1, "y") {+ t.Fatalf("first table item missing x/y cells: %q", t1)+ }+ if !strings.Contains(t2, "p") || !strings.Contains(t2, "q") {+ t.Fatalf("second table item missing p/q cells: %q", t2)+ }+}++// TestMarkdownParser_AlignmentGolden verifies Go's ParseWithResult output is+// content-equivalent to Python's _markdown on the shared sample, using the+// shared concatenation-normalization alignment tool (align_test.go). Python+// keeps raw markdown and splits on the delimiter set; Go emits clean per-block+// text. The comparison normalizes both (markdown syntax, html tags, delimiters+// stripped; whitespace collapsed) and ignores "table"/"image" items, which are+// accepted representation differences (PARSER_ALIGNMENT_HANDOFF.md §3.1).+//+// Regenerate the baseline with:+//+// .venv/bin/python internal/parser/parser/testdata/gen_markdown_golden.py+func TestMarkdownParser_AlignmentGolden(t *testing.T) {+ ctx := t.Context()+ p, _ := NewMarkdownParser(GoMarkdown)++ sample, err := os.ReadFile("testdata/markdown.sample.md")+ if err != nil {+ t.Fatalf("read sample: %v", err)+ }+ res := p.ParseWithResult(ctx, "markdown.sample.md", sample)+ if res.Err != nil {+ t.Fatalf("ParseWithResult: %v", res.Err)+ }++ golden := LoadGolden(t, "testdata/markdown.python.golden.json")++ // Ignore "table"/"image" items on both sides (accepted divergences).+ goText := FilterByDocType(res.JSON, "text")+ pyText := FilterByDocType(golden, "text")++ if ok, diff := CompareAlignment(goText, pyText, MarkdownAlignOptions(DefaultMarkdownDelimiter)); !ok {+ t.Fatalf("markdown parser not aligned with Python golden:%s", diff)+ }+}diff --git a/internal/parser/parser/testdata/gen_markdown_golden.py b/internal/parser/parser/testdata/gen_markdown_golden.pynew file mode 100644index 0000000000..fc0c113812--- /dev/null+++ b/internal/parser/parser/testdata/gen_markdown_golden.py@@ -0,0 +1,84 @@+#!/usr/bin/env python3+"""Regenerate internal/parser/parser/testdata/markdown.python.golden.json.++Drives the REAL Python markdown parser (deepdoc.parser.markdown_parser, the+same engine rag/flow/parser/parser.py:_markdown delegates to via+rag/app/naive.Markdown) so the golden is a faithful baseline rather than a+hand approximation.++Requires the project virtualenv (uv) because deepdoc needs markdown /+beartype / etc.:++ .venv/bin/python internal/parser/parser/testdata/gen_markdown_golden.py++It mirrors _markdown with separate_tables=False and the default delimiter+set, then assembles json items exactly as _markdown does:++ * each extracted section -> {"text": <raw markdown section>, "doc_type_kwd": "text"}+ * each standalone table -> {"text": <html table>, "doc_type_kwd": "table"}+ * an image section -> {"text": <alt>, "doc_type_kwd": "image"}++The Go alignment test then strips markdown syntax, html tags, and delimiters+before comparing, and ignores "table"/"image" items (those representations are+accepted divergences per PARSER_ALIGNMENT_HANDOFF.md §3.1).+"""++import json+import os+import re+import sys++# Make the repo root importable when run as a standalone script from testdata.+# Script lives at <repo>/internal/parser/parser/testdata/, so five dirname hops.+sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))))))++SAMPLE = "internal/parser/parser/testdata/markdown.sample.md"+OUT = "internal/parser/parser/testdata/markdown.python.golden.json"+DELIM = "\n!?;。;!?"+IMG_RE = re.compile(r"!\[([^\]]*)\]\(([^)]+)\)")+SENTINEL = "@@IMAGE@@"+++def main():+ from deepdoc.parser.markdown_parser import RAGFlowMarkdownParser, MarkdownElementExtractor++ with open(SAMPLE, encoding="utf-8") as f:+ raw = f.read()++ # Model _markdown's return_section_images: the image is extracted as its+ # own item (alt text only). Replace the markdown with a delimiter-free+ # sentinel so the extractor does not split it.+ alts = []++ def _repl(m):+ alts.append(m.group(1))+ return SENTINEL++ prepared = IMG_RE.sub(_repl, raw)++ parser = RAGFlowMarkdownParser()+ remainder, tables = parser.extract_tables_and_remainder(prepared + "\n", separate_tables=False)+ extractor = MarkdownElementExtractor(remainder)+ sections = extractor.extract_elements(DELIM, include_meta=True)++ items = []+ for s in sections:+ content = s["content"]+ if SENTINEL in content:+ items.append({"text": alts.pop(0), "doc_type_kwd": "image"})+ continue+ items.append({"text": content, "doc_type_kwd": "text"})++ for tbl in tables:+ # _markdown (rag/flow/parser/parser.py:1103-1111) appends each+ # extracted table as a duplicate "table" item even when inlined,+ # carrying the table's raw text (GFM source or raw <table> HTML).+ items.append({"text": tbl, "doc_type_kwd": "table"})++ with open(OUT, "w", encoding="utf-8") as f:+ json.dump(items, f, ensure_ascii=False, indent=2)+ print("wrote %d items to %s" % (len(items), OUT))+++if __name__ == "__main__":+ sys.exit(main())diff --git a/internal/parser/parser/testdata/markdown.python.golden.json b/internal/parser/parser/testdata/markdown.python.golden.jsonnew file mode 100644index 0000000000..73f5168110--- /dev/null+++ b/internal/parser/parser/testdata/markdown.python.golden.json@@ -0,0 +1,50 @@+[+ {+ "text": "# 健康检查套餐对比\n本文比较两种体检套餐,包含表格、列表与代码块",+ "doc_type_kwd": "text"+ },+ {+ "text": "## 套餐明细",+ "doc_type_kwd": "text"+ },+ {+ "text": "<table>\n<thead>\n<tr>\n<th>检查项目</th>\n<th>基础版 699 元</th>\n<th>进阶版 1299 元</th>\n</tr>\n</thead>\n<tbody>\n<tr>\n<td>血常规 / 尿常规</td>\n<td>包含</td>\n<td>包含</td>\n</tr>\n<tr>\n<td>心电图</td>\n<td>不包含</td>\n<td>包含</td>\n</tr>\n</tbody>\n</table>",+ "doc_type_kwd": "text"+ },+ {+ "text": "注意:所有套餐均需空腹",+ "doc_type_kwd": "text"+ },+ {+ "text": "## 注意事项",+ "doc_type_kwd": "text"+ },+ {+ "text": "- 体检前三天清淡饮食",+ "doc_type_kwd": "text"+ },+ {+ "text": "- 避免剧烈运动",+ "doc_type_kwd": "text"+ },+ {+ "text": "下面是示例配置:",+ "doc_type_kwd": "text"+ },+ {+ "text": "```yaml\nname: health-check\nversion: 1\n```",+ "doc_type_kwd": "text"+ },+ {+ "text": "示意图",+ "doc_type_kwd": "image"+ },+ {+ "text": "\n| 检查项目 | 基础版 699 元 | 进阶版 1299 元 |\n| --- | --- | --- |\n| 血常规 / 尿常规 | 包含 | 包含 |\n| 心电图 | 不包含 | 包含 |\n",+ "doc_type_kwd": "table"+ },+ {+ "text": "\n<table>\n<thead>\n<tr>\n<th>检查项目</th>\n<th>基础版 699 元</th>\n<th>进阶版 1299 元</th>\n</tr>\n</thead>\n<tbody>\n<tr>\n<td>血常规 / 尿常规</td>\n<td>包含</td>\n<td>包含</td>\n</tr>\n<tr>\n<td>心电图</td>\n<td>不包含</td>\n<td>包含</td>\n</tr>\n</tbody>\n</table>\n",+ "doc_type_kwd": "table"+ }+]diff --git a/internal/parser/parser/testdata/markdown.sample.md b/internal/parser/parser/testdata/markdown.sample.mdnew file mode 100644index 0000000000..7aec22ee26--- /dev/null+++ b/internal/parser/parser/testdata/markdown.sample.md@@ -0,0 +1,26 @@+# 健康检查套餐对比++本文比较两种体检套餐,包含表格、列表与代码块。++## 套餐明细++| 检查项目 | 基础版 699 元 | 进阶版 1299 元 |+| --- | --- | --- |+| 血常规 / 尿常规 | 包含 | 包含 |+| 心电图 | 不包含 | 包含 |++注意:所有套餐均需空腹。++## 注意事项++- 体检前三天清淡饮食。+- 避免剧烈运动!++下面是示例配置:++```yaml+name: health-check+version: 1+```++