Skip to content

Open-source LLM knowledge platform: turn raw documents into a queryable RAG, an autonomous reasoning agent, and a self-maintaining Wiki.

fix(chunking): 修复内联 HTML 表格与保护模式导致的超长 chunk (#3404)

neoczr473c8726bbdd
diff --git a/internal/application/service/html_table_normalize_wiring_test.go b/internal/application/service/html_table_normalize_wiring_test.go
new file mode 100644
index 000000000..295776c1e
--- /dev/null
+++ b/internal/application/service/html_table_normalize_wiring_test.go
@@ -0,0 +1,107 @@
+package service
+
+import (
+ "os"
+ "strings"
+ "testing"
+
+ "github.com/Tencent/WeKnora/internal/infrastructure/docparser"
+)
+
+// Regression guard for the central "normalize inline HTML tables before
+// chunking" wiring. The bug this guards: normalization was only wired into the
+// paddleocr-vl converters, so tables emitted by other engines (e.g. MinerU)
+// reached the chunker as raw single-line <table> blocks and were force-cut at
+// the absolute size limit.
+func TestHTMLEmbeddedTableWiring_ProducesChunkableOutput(t *testing.T) {
+ input := strings.Join([]string{
+ "# 检测报告",
+ "",
+ "以下是正文段落,用于确认归一化不会破坏普通 markdown。",
+ "",
+ // colspan=1: redundant spans must still be convertible to GFM.
+ `<table><tr><td colspan="1" style="text-align:left">项目</td>` +
+ `<td colspan="1">结果</td></tr>` +
+ `<tr><td>拉伸强度</td><td>合格</td></tr></table>`,
+ "",
+ // colspan=6: a real merge -> cannot be GFM, but must stay splittable.
+ `<table><tr><td colspan="6" style="text-align:center">汇总</td></tr>` +
+ `<tr><td colspan="6">备注</td></tr></table>`,
+ "",
+ "收尾段落。",
+ }, "\n")
+
+ got := docparser.NormalizeHTMLTables(input)
+
+ if strings.Contains(got, `colspan="1"`) {
+ t.Fatalf("redundant colspan=1 table was not normalized:\n%s", got)
+ }
+ if !strings.Contains(got, "|") || !strings.Contains(got, "拉伸强度") {
+ t.Fatalf("expected colspan=1 table to become GFM, got:\n%s", got)
+ }
+ if !strings.Contains(got, "<table") {
+ t.Fatalf("expected colspan=6 table to remain HTML, got:\n%s", got)
+ }
+ for i := 0; ; {
+ idx := strings.Index(got[i:], "<tr")
+ if idx < 0 {
+ break
+ }
+ abs := i + idx
+ if abs == 0 || got[abs-1] != '\n' {
+ t.Fatalf("remaining <tr> at offset %d is not preceded by a newline:\n%s", abs, got)
+ }
+ i = abs + 1
+ }
+}
+
+func lastCallBefore(src, call, anchor string) int {
+ idx := strings.Index(src, anchor)
+ if idx < 0 {
+ return -1
+ }
+ return strings.LastIndex(src[:idx], call)
+}
+
+// TestHTMLEmbeddedTableWiring_CallSitesPresent is a lightweight static guard:
+// it fails if the central normalization call is removed from a pre-chunk
+// (or pre-image-resolution) location.
+func TestHTMLEmbeddedTableWiring_CallSitesPresent(t *testing.T) {
+ call := "docparser.NormalizeHTMLTables("
+
+ processSrc, err := os.ReadFile("knowledge_process.go")
+ if err != nil {
+ t.Fatalf("read knowledge_process.go: %v", err)
+ }
+ process := string(processSrc)
+ if lastCallBefore(process, call, "s.imageResolver.ResolveAndStore") < 0 {
+ t.Fatalf("knowledge_process.go: %s must run before image resolution", call)
+ }
+ if lastCallBefore(process, call, "chunkCfg := buildSplitterConfigFromChunking") < 0 {
+ t.Fatalf("knowledge_process.go: %s must run before the chunking step", call)
+ }
+
+ createSrc, err := os.ReadFile("knowledge_create.go")
+ if err != nil {
+ t.Fatalf("read knowledge_create.go: %v", err)
+ }
+ if lastCallBefore(string(createSrc), call, "chunkCfg := buildSplitterConfigFromChunking") < 0 {
+ t.Fatalf("knowledge_create.go: %s must run before buildSplitterConfigFromChunking", call)
+ }
+
+ tempSrc, err := os.ReadFile("temporary_document.go")
+ if err != nil {
+ t.Fatalf("read temporary_document.go: %v", err)
+ }
+ if lastCallBefore(string(tempSrc), call, "chunker.Split(") < 0 {
+ t.Fatalf("temporary_document.go: %s must run before chunker.Split", call)
+ }
+
+ previewSrc, err := os.ReadFile("../../handler/chunker_debug.go")
+ if err != nil {
+ t.Fatalf("read chunker_debug.go: %v", err)
+ }
+ if !strings.Contains(string(previewSrc), call) {
+ t.Fatalf("chunker_debug.go is missing the preview %s call", call)
+ }
+}
diff --git a/internal/application/service/knowledge_create.go b/internal/application/service/knowledge_create.go
index df4f46985..1b19f5a6b 100644
--- a/internal/application/service/knowledge_create.go
+++ b/internal/application/service/knowledge_create.go
@@ -1276,6 +1276,11 @@ func (s *knowledgeService) triggerManualProcessing(ctx context.Context,
processOverrides, _ := knowledge.ProcessOverrides()
eff := ResolveProcessConfig(kb, processOverrides)
+ // Normalize inline HTML tables before chunking, for the same reason as the
+ // file-processing path: parser/OCR output may embed raw <table> blocks that
+ // the chunker cannot split. Fenced code examples are left untouched.
+ clean = docparser.NormalizeHTMLTables(clean)
+
// Manual content is markdown - chunk directly with Go chunker
chunkCfg := buildSplitterConfigFromChunking(eff.ChunkingConfig)
diff --git a/internal/application/service/knowledge_process.go b/internal/application/service/knowledge_process.go
index 8cce86850..648593473 100644
--- a/internal/application/service/knowledge_process.go
+++ b/internal/application/service/knowledge_process.go
@@ -3689,6 +3689,22 @@ func (s *knowledgeService) ProcessDocument(ctx context.Context, t *asynq.Task) e
convertResult.AudioData = nil
}
+ // Normalize inline HTML tables from any parser engine (MinerU, builtin,
+ // markitdown, paddleocr-vl, ...) into GFM, or row-newline-separated HTML
+ // for merged / non-GFM tables, so the chunker can split them.
+ // Runs before image resolution so <img> inside HTML tables becomes
+ // markdown the resolver can persist, instead of rewriting data-URI
+ // <img> tags to `![...](...)` and then escaping them in a later
+ // HTML→GFM pass. Line endings are normalized first so injected row
+ // breaks are plain LF.
+ sanitizeReadResult(convertResult)
+ if convertResult != nil {
+ convertResult.MarkdownContent = chunker.NormalizeLineEndings(convertResult.MarkdownContent)
+ if convertResult.MarkdownContent != "" {
+ convertResult.MarkdownContent = docparser.NormalizeHTMLTables(convertResult.MarkdownContent)
+ }
+ }
+
// Step 2: Store images and update markdown references
var storedImages []docparser.StoredImage
@@ -3719,11 +3735,8 @@ func (s *knowledgeService) ProcessDocument(ctx context.Context, t *asynq.Task) e
logger.Infof(ctx, "Resolved %d total images for knowledge %s", len(storedImages), knowledge.ID)
}
- // Step 3: Split into chunks using Go chunker. Browser textareas normalize
- // pasted content to LF, so normalize uploaded source text before calculating
- // chunk boundaries as well.
- sanitizeReadResult(convertResult)
- convertResult.MarkdownContent = chunker.NormalizeLineEndings(convertResult.MarkdownContent)
+ // Step 3: Split into chunks using Go chunker. Line endings and inline
+ // HTML tables were normalized before image resolution above.
chunkCfg := buildSplitterConfigFromChunking(eff.ChunkingConfig)
processOpts := ProcessChunksOptions{
diff --git a/internal/application/service/ocr_sanitizer.go b/internal/application/service/ocr_sanitizer.go
index dfa5fb6e6..fba14b033 100644
--- a/internal/application/service/ocr_sanitizer.go
+++ b/internal/application/service/ocr_sanitizer.go
@@ -5,6 +5,7 @@ import (
"strings"
htmltomd "github.com/JohannesKaufmann/html-to-markdown/v2"
+ "github.com/Tencent/WeKnora/internal/infrastructure/docparser"
)
var (
@@ -49,6 +50,13 @@ func sanitizeOCRText(raw string) string {
}
}
+ // Convert inline HTML <table> blocks to GFM tables. This is a safe no-op
+ // when no <table> is present, so it runs unconditionally: a markdown body
+ // with an embedded HTML table does not satisfy looksLikeHTML (it neither
+ // starts with a tag nor is dominated by tag characters), yet leaving the
+ // raw markup in place makes the chunker split inside table rows.
+ text = docparser.NormalizeHTMLTables(text)
+
if isKnownEmptyReply(text) {
return ""
}
diff --git a/internal/application/service/ocr_sanitizer_test.go b/internal/application/service/ocr_sanitizer_test.go
index b6a91c9c0..1281bc28a 100644
--- a/internal/application/service/ocr_sanitizer_test.go
+++ b/internal/application/service/ocr_sanitizer_test.go
@@ -1,6 +1,10 @@
package service
-import "testing"
+import (
+ "regexp"
+ "strings"
+ "testing"
+)
func TestSanitizeOCRText(t *testing.T) {
tests := []struct {
@@ -187,6 +191,46 @@ func TestLooksLikeHTML(t *testing.T) {
}
}
+// TestSanitizeOCRText_ConvertsInlineHTMLTable guards against a regression where
+// an HTML <table> embedded in a markdown body survived sanitization untouched:
+// the mixed content does not satisfy looksLikeHTML, so the old HTML-to-markdown
+// path never ran and the raw <table> markup reached the chunker.
+func TestSanitizeOCRText_ConvertsInlineHTMLTable(t *testing.T) {
+ para := "本季度公司整体经营情况保持稳定,营收与利润两项核心指标均实现同比增长。" +
+ "为了便于管理层快速了解经营成果,下表汇总了报告期内的主要财务数据," +
+ "供后续经营分析、预算编制以及年度考核等工作参考使用," +
+ "请结合实际业务情况综合判断,切勿脱离业务背景单独解读其中任何一项数字。"
+ tail := "以上数据均来自财务部门审核后的正式报表,统计口径与上一报告期保持一致," +
+ "未发生会计政策变更或追溯调整。如有疑问请与财务部门联系确认," +
+ "最终解释权归公司财务部门所有。"
+ input := "# 报告\n\n" + para + "\n\n" +
+ `<table><tr><th>指标</th><th>数值</th></tr>` +
+ `<tr><td>营收</td><td>10亿</td></tr>` +
+ `<tr><td>利润</td><td>2.3亿</td></tr></table>` +
+ "\n\n" + tail + "\n\n"
+
+ if looksLikeHTML(input) {
+ t.Fatalf("test input unexpectedly looks like HTML; the regression test must exercise the mixed-content path")
+ }
+
+ got := sanitizeOCRText(input)
+
+ if strings.Contains(got, "<table") {
+ t.Fatalf("expected inline HTML table to be converted, got:\n%s", got)
+ }
+ if !strings.Contains(got, "# 报告") || !strings.Contains(got, "最终解释权归公司财务部门所有。") {
+ t.Fatalf("expected surrounding markdown to be preserved, got:\n%s", got)
+ }
+ for _, want := range []string{"指标", "数值", "营收", "10亿", "利润", "2.3亿"} {
+ if !strings.Contains(got, want) {
+ t.Fatalf("expected converted table to retain %q, got:\n%s", want, got)
+ }
+ }
+ if !regexp.MustCompile(`\|[-:]{3,}\|`).MatchString(got) {
+ t.Fatalf("expected a GFM table separator row, got:\n%s", got)
+ }
+}
+
func TestIsKnownEmptyReply(t *testing.T) {
tests := []struct {
input string
diff --git a/internal/application/service/temporary_document.go b/internal/application/service/temporary_document.go
index 4a0976d27..2c478d3f6 100644
--- a/internal/application/service/temporary_document.go
+++ b/internal/application/service/temporary_document.go
@@ -383,6 +383,8 @@ func (s *temporaryDocumentService) Process(ctx context.Context, task *asynq.Task
return nil
}
content = common.CleanInvalidUTF8(content)
+ content = chunker.NormalizeLineEndings(content)
+ content = docparser.NormalizeHTMLTables(content)
lang := chunker.DetectLanguage(content)
cfg := chunker.DefaultConfig()
cfg.Strategy = chunker.StrategyAuto
@@ -463,6 +465,10 @@ func (s *temporaryDocumentService) parse(ctx context.Context, document *types.Te
if err != nil {
return "", nil, nil, fmt.Errorf("parse document: %w", err)
}
+ if result != nil && result.MarkdownContent != "" {
+ result.MarkdownContent = chunker.NormalizeLineEndings(result.MarkdownContent)
+ result.MarkdownContent = docparser.NormalizeHTMLTables(result.MarkdownContent)
+ }
// Capture raw page-image bytes before ResolveAndStore stores/rewrites them,
// so the VLM OCR fallback for scanned documents has bytes to work with.
maxOCRPages := temporaryDocumentImageOCRMaxPages()
diff --git a/internal/handler/chunker_debug.go b/internal/handler/chunker_debug.go
index 3e12d2496..6d045a917 100644
--- a/internal/handler/chunker_debug.go
+++ b/internal/handler/chunker_debug.go
@@ -13,6 +13,7 @@ import (
"unicode/utf8"
"github.com/Tencent/WeKnora/internal/infrastructure/chunker"
+ "github.com/Tencent/WeKnora/internal/infrastructure/docparser"
"github.com/Tencent/WeKnora/internal/logger"
"github.com/gin-gonic/gin"
)
@@ -147,6 +148,7 @@ func PreviewChunking(c *gin.Context) {
return
}
text := chunker.NormalizeLineEndings(req.Text)
+ text = docparser.NormalizeHTMLTables(text)
cfg := chunker.NormalizeSplitterConfig(chunker.SplitterConfig{
ChunkSize: req.ChunkingConfig.ChunkSize,
diff --git a/internal/infrastructure/chunker/protected_spans_test.go b/internal/infrastructure/chunker/protected_spans_test.go
new file mode 100644
index 000000000..3fe2cda0c
--- /dev/null
+++ b/internal/infrastructure/chunker/protected_spans_test.go
@@ -0,0 +1,135 @@
+package chunker
+
+import (
+ "strconv"
+ "strings"
+ "testing"
+)
+
+// buildOCRStrayBracketDoc models the OCR failure mode behind this regression:
+// a full-width "【公章】" whose opening bracket was mis-read as the half-width
+// '[' (closing bracket lost), followed later by an unrelated half-width
+// "](...)" fragment. CommonMark forbids a link's text/destination from
+// spanning a blank line, but the previous unbounded
+// `\[[^\]]*\]\([^)]+\)` joined the stray '[' to the distant '](' and
+// swallowed the entire document as one protected atomic span.
+func buildOCRStrayBracketDoc() string {
+ var sb strings.Builder
+ sb.WriteString("[公章】\n\n")
+ for i := 0; i < 60; i++ {
+ sb.WriteString("这是第")
+ sb.WriteString(strconv.Itoa(i))
+ sb.WriteString("段用于测试切分的中文内容,包含常见标点符号。\n\n")
+ }
+ sb.WriteString("第三段(北京)\n\n")
+ sb.WriteString("[引用](来源)")
+ return sb.String()
+}
+
+func maxChunkRunes(chunks []Chunk) int {
+ best := 0
+ for _, c := range chunks {
+ if n := len([]rune(c.Content)); n > best {
+ best = n
+ }
+ }
+ return best
+}
+
+// TestSplitText_OCRStrayBracketDoesNotSwallowDocument is the core regression:
+// before the fix the malformed '[' turned the whole document into a single
+// protected span (~1700 runes) and SplitText returned one oversized chunk.
+func TestSplitText_OCRStrayBracketDoesNotSwallowDocument(t *testing.T) {
+ doc := buildOCRStrayBracketDoc()
+ if !strings.Contains(doc, "[公章】") || !strings.Contains(doc, "](来源)") {
+ t.Fatalf("fixture missing malformed-bracket markers")
+ }
+ const chunkSize = 128
+ cfg := SplitterConfig{ChunkSize: chunkSize, ChunkOverlap: 0, Separators: []string{"\n\n", "\n", "。"}}
+
+ chunks := SplitText(doc, cfg)
+ t.Logf("doc=%d runes chunkSize=%d -> chunks=%d max=%d runes",
+ len([]rune(doc)), chunkSize, len(chunks), maxChunkRunes(chunks))
+ if len(chunks) < 2 {
+ t.Fatalf("expected multiple chunks, got %d (max=%d runes)", len(chunks), maxChunkRunes(chunks))
+ }
+ if got := maxChunkRunes(chunks); got > 2*chunkSize {
+ t.Fatalf("max chunk %d runes exceeds 2*chunkSize=%d: stray '[' swallowed the document", got, 2*chunkSize)
+ }
+}
+
+// TestSplit_OCRStrayBracketDoesNotSwallowDocument exercises the strategy-aware
+// entry point with the legacy tier pinned, so the protected-pattern behaviour
+// is what is under test rather than the profiler's tier selection.
+func TestSplit_OCRStrayBracketDoesNotSwallowDocument(t *testing.T) {
+ doc := buildOCRStrayBracketDoc()
+ const chunkSize = 128
+ cfg := SplitterConfig{
+ ChunkSize: chunkSize,
+ ChunkOverlap: 0,
+ Separators: []string{"\n\n", "\n", "。"},
+ Strategy: StrategyLegacy,
+ }
+
+ chunks := Split(doc, cfg)
+ if len(chunks) < 2 {
+ t.Fatalf("expected multiple chunks, got %d (max=%d runes)", len(chunks), maxChunkRunes(chunks))
+ }
+ if got := maxChunkRunes(chunks); got > 2*chunkSize {
+ t.Fatalf("max chunk %d runes exceeds 2*chunkSize=%d: stray '[' swallowed the document", got, 2*chunkSize)
+ }
+}
+
+// TestProtectedSpans_NoCrossLineLinkOrImage asserts no protected span may
+// contain a newline for the malformed document: link text and destination
+// cannot span a blank line (CommonMark).
+func TestProtectedSpans_NoCrossLineLinkOrImage(t *testing.T) {
+ doc := buildOCRStrayBracketDoc()
+ for _, s := range protectedSpans(doc) {
+ seg := doc[s.start:s.end]
+ if strings.Contains(seg, "\n") {
+ t.Fatalf("protected span spans a newline (%d runes): %q", len([]rune(seg)), seg)
+ }
+ }
+}
+
+// TestProtectedSpans_MarkdownLinksAndImagesStillProtected is the positive
+// guard: the newline-bounded patterns must keep recognising well-formed
+// single-line Markdown links and images.
+func TestProtectedSpans_MarkdownLinksAndImagesStillProtected(t *testing.T) {
+ cases := []string{
+ "![alt](http://x/y.png)",
+ "[文字](http://x.com)",
+ `[link](url "title")`,
+ "![](resource://abc)",
+ }
+ for _, c := range cases {
+ spans := protectedSpans(c)
+ if len(spans) == 0 {
+ t.Errorf("expected %q to be protected, got no span", c)
+ continue
+ }
+ if spans[0].start != 0 || spans[0].end != len(c) {
+ t.Errorf("%q: protected span = [%d,%d), want [0,%d)", c, spans[0].start, spans[0].end, len(c))
+ }
+ }
+}
+
+// TestProtectedSpans_LiteralCrossLineExample documents that an isolated
+// malformed bracket pair with NO complete '](...)' construct is never treated
+// as a protected link/image region.
+func TestProtectedSpans_MultilineLinkIntentionallyUnprotected(t *testing.T) {
+ // CommonMark allows a single soft line break in link text. The protected
+ // patterns still refuse it so a stray OCR '[' cannot swallow a paragraph.
+ doc := "[link\ntext](http://example.com)"
+ if spans := protectedSpans(doc); len(spans) != 0 {
+ t.Fatalf("expected multiline link to stay unprotected, got %v", spans)
+ }
+}
+
+func TestProtectedSpans_LiteralCrossLineExample(t *testing.T) {
+ doc := "[公章】\n\n第一段。\n\n第二段。\n\n第三段(北京)"
+ if spans := protectedSpans(doc); len(spans) != 0 {
+ t.Fatalf("expected no protected span, got %v", spans)
+ }
+}
diff --git a/internal/infrastructure/chunker/splitter.go b/internal/infrastructure/chunker/splitter.go
index ff415eed6..69ff38e43 100644
--- a/internal/infrastructure/chunker/splitter.go
+++ b/internal/infrastructure/chunker/splitter.go
@@ -114,10 +114,18 @@ func DefaultConfig() SplitterConfig {
}
// protectedPatterns are regex patterns for content that must not be split.
+//
+// The Markdown link/image patterns deliberately exclude '\n' and bound the
+// link text / destination length: CommonMark forbids them from spanning a
+// blank line, and the previous unbounded [^\]]* / [^)]+ let a stray '[' left
+// behind by OCR swallow whole paragraphs as one "protected" atomic span,
+// defeating chunking entirely.
var protectedPatterns = []*regexp.Regexp{
- regexp.MustCompile(`(?s)\$\$.*?\$\$`), // LaTeX block math
- regexp.MustCompile(`!\[[^\]]*\]\([^)]+\)`), // Markdown images
- regexp.MustCompile(`\[[^\]]*\]\([^)]+\)`), // Markdown links
+ regexp.MustCompile(`(?s)\$\$.*?\$\$`), // LaTeX block math
+ // Markdown images / links: single line, bounded so a stray OCR '['
+ // cannot swallow a paragraph (CommonMark forbids blank-line spans).
+ regexp.MustCompile(`!\[[^\]\n]{0,200}\]\([^)\n]{1,500}\)`),
+ regexp.MustCompile(`\[[^\]\n]{1,200}\]\([^)\n]{1,500}\)`),
regexp.MustCompile("(?m)[ ]*(?:\\|[^|\\n]*)+\\|[\\r\\n]+\\s*(?:\\|\\s*:?-{3,}:?\\s*)+\\|[\\r\\n]+"), // Table header+separator
regexp.MustCompile("(?m)[ ]*(?:\\|[^|\\n]*)+\\|[\\r\\n]+"), // Table rows
regexp.MustCompile("(?s)```(?:\\w+)?[\\r\\n].*?```"), // Fenced code blocks
diff --git a/internal/infrastructure/docparser/html_table_normalizer.go b/internal/infrastructure/docparser/html_table_normalizer.go
index ad55b1967..8851c2ba7 100644
--- a/internal/infrastructure/docparser/html_table_normalizer.go
+++ b/internal/infrastructure/docparser/html_table_normalizer.go
@@ -15,6 +15,10 @@ var (
// block, the form OCR/layout engines such as PaddleOCR-VL emit tables in.
htmlTableBlockPattern = regexp.MustCompile(`(?is)<table\b[^>]*>.*?</table>`)
+ // fencedCodeBlockPattern matches CommonMark fenced code blocks so HTML
+ // <table> examples inside them are left untouched.
+ fencedCodeBlockPattern = regexp.MustCompile("(?s)(?:```|~~~)[^\\n]*\\n.*?(?:```|~~~)")
+
// htmlLayoutAttrPattern matches presentational HTML attributes that carry
// no semantic value (text-align styles, CSS classes, sizing). Structural
// attributes like rowspan/colspan are intentionally excluded.
@@ -22,28 +26,45 @@ var (
`(?is)\s+(?:style|class|align|valign|width|height|bgcolor)\s*=\s*(?:"[^"]*"|'[^']*'|[^\s>]+)`,
)
- // htmlSpanAttrPattern detects rowspan/colspan, which Markdown tables cannot
- // represent; such tables keep their HTML form (attributes stripped) instead.
- htmlSpanAttrPattern = regexp.MustCompile(`(?i)\b(?:row|col)span\b`)
+ // htmlSpanAttrPattern detects rowspan/colspan values greater than 1, which
+ // Markdown tables cannot represent; such tables keep their HTML form
+ // (attributes stripped) instead. Span values of 1 (and the invalid 0) do
+ // not merge anything and must stay convertible.
+ htmlSpanAttrPattern = regexp.MustCompile(`(?i)\b(?:row|col)span\s*=\s*["']?(?:[2-9]|\d{2,})`)
+
+ // htmlTableRowPattern matches the opening <tr> tag of each table row, used
+ // to put every row on its own line so the chunker can split degraded HTML
+ // tables at "\n" boundaries.
+ htmlTableRowPattern = regexp.MustCompile(`(?i)<tr\b`)
- // markdownTableSeparatorPattern matches the |---|---| delimiter row that a
- // valid GFM table must contain.
- markdownTableSeparatorPattern = regexp.MustCompile(`(?m)^\s*\|?\s*:?-+:?\s*(?:\|\s*:?-+:?\s*)+\|?\s*$`)
+ // markdownTableSeparatorPattern matches the |---| delimiter row that a
+ // valid GFM table must contain. The repeating column group is optional so
+ // single-column tables (|---|) are accepted; requiring two or more columns
+ // discarded successful conversions of MinerU figure/TOC tables.
+ markdownTableSeparatorPattern = regexp.MustCompile(`(?m)^\s*\|?\s*:?-+:?\s*(?:\|\s*:?-+:?\s*)*\|?\s*$`)
)
-// normalizeHTMLTables rewrites inline HTML <table> blocks embedded in OCR
+// NormalizeHTMLTables rewrites inline HTML <table> blocks embedded in OCR
// markdown output. PaddleOCR-VL emits tables as HTML with per-cell text-align
// styles, which (1) waste tokens on layout markup and (2) are not recognized
// by the chunker's table-protection logic, so large tables get split mid-row.
//
// Each table block is converted to a GFM Markdown table when possible. Tables
-// that use rowspan/colspan (which Markdown cannot express) fall back to having
-// their presentational attributes stripped so they stay intact as HTML.
-func normalizeHTMLTables(md string) string {
+// that use rowspan/colspan (which Markdown cannot express), or that the
+// converter cannot turn into a valid GFM table (cell <br>, nested lists, …),
+// fall back to presentational-attribute stripping plus one row per line so
+// the chunker can still split on "\n". Tables inside fenced code blocks are
+// left unchanged.
+func NormalizeHTMLTables(md string) string {
if !strings.Contains(strings.ToLower(md), "<table") {
return md
}
+ locs := htmlTableBlockPattern.FindAllStringIndex(md, -1)
+ if len(locs) == 0 {
+ return md
+ }
+
conv := converter.NewConverter(
converter.WithPlugins(
base.NewBasePlugin(),
@@ -51,23 +72,60 @@ func normalizeHTMLTables(md string) string {
table.NewTablePlugin(),
),
)
+ fences := fencedCodeBlockPattern.FindAllStringIndex(md, -1)
- return htmlTableBlockPattern.ReplaceAllStringFunc(md, func(block string) string {
- if htmlSpanAttrPattern.MatchString(block) {
- return stripHTMLLayoutAttrs(block)
+ var b strings.Builder
+ last := 0
+ for _, loc := range locs {
+ b.WriteString(md[last:loc[0]])
+ block := md[loc[0]:loc[1]]
+ if indexInsideSpans(loc[0], fences) {
+ b.WriteString(block)
+ last = loc[1]
+ continue
}
- converted, err := conv.ConvertString(block)
- if err != nil {
- return stripHTMLLayoutAttrs(block)
+ // Isolate the rewritten table with blank lines without stacking pads
+ // on repeated NormalizeHTMLTables calls.
+ out := strings.TrimRight(b.String(), "\n")
+ b.Reset()
+ b.WriteString(out)
+ b.WriteString("\n\n")
+ b.WriteString(normalizeOneHTMLTable(conv, block))
+ last = loc[1]
+ for last < len(md) && md[last] == '\n' {
+ last++
}
- converted = strings.TrimSpace(converted)
- if converted == "" || !markdownTableSeparatorPattern.MatchString(converted) {
- return stripHTMLLayoutAttrs(block)
+ b.WriteString("\n\n")
+ }
+ b.WriteString(md[last:])
+ return b.String()
+}
+
+func normalizeOneHTMLTable(conv *converter.Converter, block string) string {
+ fallback := func() string {
+ return splitHTMLTableRows(stripHTMLLayoutAttrs(block))
+ }
+ if htmlSpanAttrPattern.MatchString(block) {
+ return fallback()
+ }
+ converted, err := conv.ConvertString(block)
+ if err != nil {
+ return fallback()
+ }
+ converted = unescapeMarkdownImageSyntax(strings.TrimSpace(converted))
+ if converted == "" || !markdownTableSeparatorPattern.MatchString(converted) {
+ return fallback()
+ }
+ return converted
+}
+
+func indexInsideSpans(pos int, spans [][]int) bool {
+ for _, s := range spans {
+ if pos >= s[0] && pos < s[1] {
+ return true
}
- // Pad with blank lines so the Markdown table is a standalone block that
- // the chunker recognizes (and protects) as a table.
- return "\n\n" + converted + "\n\n"
- })
+ }
+ return false
}
// stripHTMLLayoutAttrs removes presentational attributes from an HTML fragment
@@ -75,3 +133,27 @@ func normalizeHTMLTables(md string) string {
func stripHTMLLayoutAttrs(html string) string {
return htmlLayoutAttrPattern.ReplaceAllString(html, "")
}
+
+// splitHTMLTableRows puts each table row on its own line. The chunker splits
+// on "\n", so a single-line HTML table would otherwise be unsplittable and get
+// force-cut at the absolute max size. Whitespace between tags is insignificant
+// in HTML, and newlines are only inserted before <tr> (never inside a tag).
+// A <tr> that already starts on its own line is left alone.
+func splitHTMLTableRows(block string) string {
+ locs := htmlTableRowPattern.FindAllStringIndex(block, -1)
+ if len(locs) == 0 {
+ return strings.TrimSpace(block)
+ }
+ var b strings.Builder
+ last := 0
+ for _, loc := range locs {
+ b.WriteString(block[last:loc[0]])
+ if loc[0] == 0 || block[loc[0]-1] != '\n' {
+ b.WriteByte('\n')
+ }
+ b.WriteString(block[loc[0]:loc[1]])
+ last = loc[1]
+ }
+ b.WriteString(block[last:])
+ return strings.TrimSpace(b.String())
+}
diff --git a/internal/infrastructure/docparser/html_table_normalizer_test.go b/internal/infrastructure/docparser/html_table_normalizer_test.go
index 8ae31d591..19b036bbc 100644
--- a/internal/infrastructure/docparser/html_table_normalizer_test.go
+++ b/internal/infrastructure/docparser/html_table_normalizer_test.go
@@ -16,7 +16,7 @@ func TestNormalizeHTMLTables_ConvertsStyledTableToMarkdown(t *testing.T) {
结尾。`
- got := normalizeHTMLTables(input)
+ got := NormalizeHTMLTables(input)
if strings.Contains(got, "<table") {
t.Fatalf("expected HTML table to be converted away, got:\n%s", got)
@@ -37,13 +37,92 @@ func TestNormalizeHTMLTables_ConvertsStyledTableToMarkdown(t *testing.T) {
}
}
+func TestNormalizeHTMLTables_NoTableUnchanged(t *testing.T) {
+ input := "# 标题\n\n普通段落,没有表格。\n\n| a | b |\n| --- | --- |\n| 1 | 2 |"
+ if got := NormalizeHTMLTables(input); got != input {
+ t.Fatalf("expected content without HTML tables to be unchanged, got:\n%s", got)
+ }
+}
+
+func TestNormalizeHTMLTables_SpanValueOneIsConvertible(t *testing.T) {
+ // rowspan=1 / colspan=1 merge nothing, so the table must still become GFM.
+ // Covers quoted, unquoted and space-padded attribute forms.
+ input := `<table><tr>` +
+ `<td rowspan="1" colspan="1" style="text-align:center;">指标</td>` +
+ `<td rowspan=1 colspan=1>数值</td></tr>` +
+ `<tr><td rowspan="1" colspan='1'>营收</td>` +
+ `<td rowspan = 1 colspan = 1>10亿</td></tr></table>`
+
+ got := NormalizeHTMLTables(input)
+
+ if strings.Contains(got, "<table") {
+ t.Fatalf("expected span=1 table to be converted away, got:\n%s", got)
+ }
+ if !markdownTableSeparatorPattern.MatchString(got) {
+ t.Fatalf("expected a Markdown table separator row, got:\n%s", got)
+ }
+ for _, want := range []string{"指标", "数值", "营收", "10亿"} {
+ if !strings.Contains(got, want) {
+ t.Fatalf("expected converted table to retain %q, got:\n%s", want, got)
+ }
+ }
+}
+
+func TestNormalizeHTMLTables_RealSpanKeepsHTMLWithRowNewlines(t *testing.T) {
+ // Genuine merges (colspan>1 / rowspan>1) cannot be GFM, so the table stays
+ // HTML — but each row must start on its own line so the chunker can split it.
+ input := `<table><tr><td colspan="6" style="text-align:center;">合计</td></tr>` +
+ `<tr><td rowspan="3">A</td><td>1</td></tr>` +
+ `<tr><td rowspan = 2>B</td><td>2</td></tr></table>`
+
+ got := NormalizeHTMLTables(input)
+
+ if !strings.Contains(got, "<table") {
+ t.Fatalf("expected real span table to remain HTML, got:\n%s", got)
+ }
+ if !strings.Contains(got, `colspan="6"`) || !strings.Contains(got, `rowspan="3"`) {
+ t.Fatalf("expected structural span attrs preserved, got:\n%s", got)
+ }
+ if !strings.Contains(got, "\n<tr") {
+ t.Fatalf("expected each <tr> to start on a new line, got:\n%q", got)
+ }
+ if !strings.Contains(got, "\n\n<table") {
+ t.Fatalf("expected block to be padded at the head with blank lines, got:\n%q", got)
+ }
+ if !strings.HasSuffix(got, "</table>\n\n") {
+ t.Fatalf("expected block to be padded at the tail with blank lines, got:\n%q", got)
+ }
+ if n := strings.Count(got, "\n<tr"); n != 3 {
+ t.Fatalf("expected 3 rows each preceded by a newline, got %d in:\n%q", n, got)
+ }
+}
+
+func assertHTMLRowsSplittable(t *testing.T, got string) {
+ t.Helper()
+ if !strings.Contains(got, "<tr") {
+ t.Fatalf("expected remaining HTML table rows, got:\n%s", got)
+ }
+ for i := 0; ; {
+ idx := strings.Index(got[i:], "<tr")
+ if idx < 0 {
+ return
+ }
+ abs := i + idx
+ if abs == 0 || got[abs-1] != '\n' {
+ t.Fatalf("remaining <tr> at offset %d is not preceded by a newline:\n%s", abs, got)
+ }
+ i = abs + 1
+ }
+}
+
func TestNormalizeHTMLTables_StripsAttrsOnSpanTables(t *testing.T) {
// rowspan/colspan cannot be expressed in Markdown, so the table stays HTML
- // but its presentational attributes are stripped.
- input := `<table><tr><td colspan="2" style="text-align:center;" class="hdr">合计</td></tr>` +
+ // but its presentational attributes are stripped and rows become splittable.
+ input := `<table><tr>` +
+ `<td colspan="2" style="text-align:center;" class="hdr">合计</td></tr>` +
`<tr><td style="text-align:left;">A</td><td width="80">B</td></tr></table>`
- got := normalizeHTMLTables(input)
+ got := NormalizeHTMLTables(input)
if !strings.Contains(got, "<table") {
t.Fatalf("expected span table to remain HTML, got:\n%s", got)
@@ -56,11 +135,99 @@ func TestNormalizeHTMLTables_StripsAttrsOnSpanTables(t *testing.T) {
t.Fatalf("expected %q to be stripped, got:\n%s", banned, got)
}
}
+ assertHTMLRowsSplittable(t, got)
}
-func TestNormalizeHTMLTables_NoTableUnchanged(t *testing.T) {
- input := "# 标题\n\n普通段落,没有表格。\n\n| a | b |\n| --- | --- |\n| 1 | 2 |"
- if got := normalizeHTMLTables(input); got != input {
- t.Fatalf("expected content without HTML tables to be unchanged, got:\n%s", got)
+func TestNormalizeHTMLTables_BRCellsStaySplittableHTML(t *testing.T) {
+ // <br> inside a cell makes the table plugin give up (it emits a newline).
+ // The HTML fallback must still put each <tr> on its own line.
+ input := `<table><tr><td>指标<br/>名称</td><td>数值<br/>单位</td></tr>` +
+ `<tr><td>拉伸强度<br/>MPa</td><td>合格<br/>A级</td></tr></table>`
+
+ got := NormalizeHTMLTables(input)
+
+ if !strings.Contains(got, "<table") {
+ t.Fatalf("expected <br> table to remain HTML rather than flattened text, got:\n%s", got)
+ }
+ assertHTMLRowsSplittable(t, got)
+ for _, want := range []string{"指标", "名称", "拉伸强度", "合格"} {
+ if !strings.Contains(got, want) {
+ t.Fatalf("expected fallback HTML to retain %q, got:\n%s", want, got)
+ }
+ }
+}
+
+func TestNormalizeHTMLTables_ListInCellFallsBackToSplittableHTML(t *testing.T) {
+ input := `<table><tr><td><ul><li>a</li><li>b</li></ul></td><td>x</td></tr>` +
+ `<tr><td>c</td><td>d</td></tr></table>`
+
+ got := NormalizeHTMLTables(input)
+
+ if !strings.Contains(got, "<table") {
+ t.Fatalf("expected list-in-cell table to remain HTML, got:\n%s", got)
+ }
+ assertHTMLRowsSplittable(t, got)
+}
+
+func TestNormalizeHTMLTables_SingleColumnConvertsToGFM(t *testing.T) {
+ input := `<table><tr><td>目录</td></tr><tr><td>第一章</td></tr><tr><td>第二章</td></tr></table>`
+
+ got := NormalizeHTMLTables(input)
+
+ if strings.Contains(got, "<table") {
+ t.Fatalf("expected single-column table to convert to GFM, got:\n%s", got)
+ }
+ if !markdownTableSeparatorPattern.MatchString(got) {
+ t.Fatalf("expected a Markdown table separator row, got:\n%s", got)
+ }
+ for _, want := range []string{"目录", "第一章", "第二章"} {
+ if !strings.Contains(got, want) {
+ t.Fatalf("expected converted table to retain %q, got:\n%s", want, got)
+ }
+ }
+}
+
+func TestNormalizeHTMLTables_ImgInSingleColumnBecomesGFM(t *testing.T) {
+ input := `<table><tr><td><img src="local://images/profile.png" alt="profile"/></td></tr></table>`
+
+ got := NormalizeHTMLTables(input)
+
+ if strings.Contains(got, "<table") {
+ t.Fatalf("expected single-column image table to convert to GFM, got:\n%s", got)
+ }
+ if !strings.Contains(got, "![profile](local://images/profile.png)") {
+ t.Fatalf("expected markdown image to be preserved, got:\n%s", got)
+ }
+}
+
+func TestNormalizeHTMLTables_MarkdownImageInCellNotEscaped(t *testing.T) {
+ input := `<table><tr><td>![cover](local://images/cover.png)</td><td>说明</td></tr></table>`
+
+ got := NormalizeHTMLTables(input)
+
+ if strings.Contains(got, `!\[`) {
+ t.Fatalf("markdown image syntax escaped:\n%s", got)
+ }
+ if !strings.Contains(got, "![cover](local://images/cover.png)") {
+ t.Fatalf("expected markdown image to survive conversion, got:\n%s", got)
+ }
+}
+
+func TestNormalizeHTMLTables_CodeFenceTableLeftAlone(t *testing.T) {
+ input := "示例:\n\n```html\n<table><tr><td>a</td><td>b</td></tr></table>\n```\n"
+
+ got := NormalizeHTMLTables(input)
+
+ if got != input {
+ t.Fatalf("expected fenced HTML table to be unchanged, got:\n%s", got)
+ }
+}
+
+func TestNormalizeHTMLTables_SpanTableIdempotent(t *testing.T) {
+ input := `<table><tr><td colspan="6">汇总</td></tr><tr><td colspan="6">备注</td></tr></table>`
+ once := NormalizeHTMLTables(input)
+ twice := NormalizeHTMLTables(once)
+ if once != twice {
+ t.Fatalf("NormalizeHTMLTables is not idempotent for span tables:\n1=%q\n2=%q", once, twice)
}
}
diff --git a/internal/infrastructure/docparser/paddleocr_vl_cloud_converter.go b/internal/infrastructure/docparser/paddleocr_vl_cloud_converter.go
index 98cfc0133..c05c9561a 100644
--- a/internal/infrastructure/docparser/paddleocr_vl_cloud_converter.go
+++ b/internal/infrastructure/docparser/paddleocr_vl_cloud_converter.go
@@ -78,7 +78,7 @@ func (c *PaddleOCRVLCloudReader) Read(ctx context.Context, req *types.ReadReques
return nil, fmt.Errorf("PaddleOCR-VL Cloud fetch results: %w", err)
}
- mdContent = normalizeHTMLTables(mdContent)
+ mdContent = NormalizeHTMLTables(mdContent)
imageRefs := c.downloadImages(mdContent, imagesURL)
mdContent, imageRefs = ensureOriginalImageRef(req, mdContent, imageRefs)
diff --git a/internal/infrastructure/docparser/paddleocr_vl_converter.go b/internal/infrastructure/docparser/paddleocr_vl_converter.go
index 313d3b74e..f49e1df58 100644
--- a/internal/infrastructure/docparser/paddleocr_vl_converter.go
+++ b/internal/infrastructure/docparser/paddleocr_vl_converter.go
@@ -65,7 +65,7 @@ func (c *PaddleOCRVLReader) Read(ctx context.Context, req *types.ReadRequest) (*
// wastes tokens and defeats the chunker's table-protection logic. Convert
// them to Markdown tables (or strip layout attributes when conversion is
// not possible) before downstream processing.
- mdContent = normalizeHTMLTables(mdContent)
+ mdContent = NormalizeHTMLTables(mdContent)
imageRefs, mdContent := c.processImages(mdContent, imagesB64)
mdContent, imageRefs = ensureOriginalImageRef(req, mdContent, imageRefs)