Open-source LLM knowledge platform: turn raw documents into a queryable RAG, an autonomous reasoning agent, and a self-maintaining Wiki.
Go agentagenticaichatbotdsh-pluginembeddingsevaluationgenerative-aigolangknowledge-basellmmulti-tenantollamaopenaiquestion-answeringragrerankingsemantic-searchvector-searchwiki
fix(chunking): 修复内联 HTML 表格与保护模式导致的超长 chunk (#3404)
neoczr473c8726bbdd
diff --git a/internal/application/service/html_table_normalize_wiring_test.go b/internal/application/service/html_table_normalize_wiring_test.gonew file mode 100644index 000000000..295776c1e--- /dev/null+++ b/internal/application/service/html_table_normalize_wiring_test.go@@ -0,0 +1,107 @@+package service++import (+ "os"+ "strings"+ "testing"++ "github.com/Tencent/WeKnora/internal/infrastructure/docparser"+)++// Regression guard for the central "normalize inline HTML tables before+// chunking" wiring. The bug this guards: normalization was only wired into the+// paddleocr-vl converters, so tables emitted by other engines (e.g. MinerU)+// reached the chunker as raw single-line <table> blocks and were force-cut at+// the absolute size limit.+func TestHTMLEmbeddedTableWiring_ProducesChunkableOutput(t *testing.T) {+ input := strings.Join([]string{+ "# 检测报告",+ "",+ "以下是正文段落,用于确认归一化不会破坏普通 markdown。",+ "",+ // colspan=1: redundant spans must still be convertible to GFM.+ `<table><tr><td colspan="1" style="text-align:left">项目</td>` ++ `<td colspan="1">结果</td></tr>` ++ `<tr><td>拉伸强度</td><td>合格</td></tr></table>`,+ "",+ // colspan=6: a real merge -> cannot be GFM, but must stay splittable.+ `<table><tr><td colspan="6" style="text-align:center">汇总</td></tr>` ++ `<tr><td colspan="6">备注</td></tr></table>`,+ "",+ "收尾段落。",+ }, "\n")++ got := docparser.NormalizeHTMLTables(input)++ if strings.Contains(got, `colspan="1"`) {+ t.Fatalf("redundant colspan=1 table was not normalized:\n%s", got)+ }+ if !strings.Contains(got, "|") || !strings.Contains(got, "拉伸强度") {+ t.Fatalf("expected colspan=1 table to become GFM, got:\n%s", got)+ }+ if !strings.Contains(got, "<table") {+ t.Fatalf("expected colspan=6 table to remain HTML, got:\n%s", got)+ }+ for i := 0; ; {+ idx := strings.Index(got[i:], "<tr")+ if idx < 0 {+ break+ }+ abs := i + idx+ if abs == 0 || got[abs-1] != '\n' {+ t.Fatalf("remaining <tr> at offset %d is not preceded by a newline:\n%s", abs, got)+ }+ i = abs + 1+ }+}++func lastCallBefore(src, call, anchor string) int {+ idx := strings.Index(src, anchor)+ if idx < 0 {+ return -1+ }+ return strings.LastIndex(src[:idx], call)+}++// TestHTMLEmbeddedTableWiring_CallSitesPresent is a lightweight static guard:+// it fails if the central normalization call is removed from a pre-chunk+// (or pre-image-resolution) location.+func TestHTMLEmbeddedTableWiring_CallSitesPresent(t *testing.T) {+ call := "docparser.NormalizeHTMLTables("++ processSrc, err := os.ReadFile("knowledge_process.go")+ if err != nil {+ t.Fatalf("read knowledge_process.go: %v", err)+ }+ process := string(processSrc)+ if lastCallBefore(process, call, "s.imageResolver.ResolveAndStore") < 0 {+ t.Fatalf("knowledge_process.go: %s must run before image resolution", call)+ }+ if lastCallBefore(process, call, "chunkCfg := buildSplitterConfigFromChunking") < 0 {+ t.Fatalf("knowledge_process.go: %s must run before the chunking step", call)+ }++ createSrc, err := os.ReadFile("knowledge_create.go")+ if err != nil {+ t.Fatalf("read knowledge_create.go: %v", err)+ }+ if lastCallBefore(string(createSrc), call, "chunkCfg := buildSplitterConfigFromChunking") < 0 {+ t.Fatalf("knowledge_create.go: %s must run before buildSplitterConfigFromChunking", call)+ }++ tempSrc, err := os.ReadFile("temporary_document.go")+ if err != nil {+ t.Fatalf("read temporary_document.go: %v", err)+ }+ if lastCallBefore(string(tempSrc), call, "chunker.Split(") < 0 {+ t.Fatalf("temporary_document.go: %s must run before chunker.Split", call)+ }++ previewSrc, err := os.ReadFile("../../handler/chunker_debug.go")+ if err != nil {+ t.Fatalf("read chunker_debug.go: %v", err)+ }+ if !strings.Contains(string(previewSrc), call) {+ t.Fatalf("chunker_debug.go is missing the preview %s call", call)+ }+}diff --git a/internal/application/service/knowledge_create.go b/internal/application/service/knowledge_create.goindex df4f46985..1b19f5a6b 100644--- a/internal/application/service/knowledge_create.go+++ b/internal/application/service/knowledge_create.go@@ -1276,6 +1276,11 @@ func (s *knowledgeService) triggerManualProcessing(ctx context.Context,processOverrides, _ := knowledge.ProcessOverrides()eff := ResolveProcessConfig(kb, processOverrides)+ // Normalize inline HTML tables before chunking, for the same reason as the+ // file-processing path: parser/OCR output may embed raw <table> blocks that+ // the chunker cannot split. Fenced code examples are left untouched.+ clean = docparser.NormalizeHTMLTables(clean)+// Manual content is markdown - chunk directly with Go chunkerchunkCfg := buildSplitterConfigFromChunking(eff.ChunkingConfig)diff --git a/internal/application/service/knowledge_process.go b/internal/application/service/knowledge_process.goindex 8cce86850..648593473 100644--- a/internal/application/service/knowledge_process.go+++ b/internal/application/service/knowledge_process.go@@ -3689,6 +3689,22 @@ func (s *knowledgeService) ProcessDocument(ctx context.Context, t *asynq.Task) econvertResult.AudioData = nil}+ // Normalize inline HTML tables from any parser engine (MinerU, builtin,+ // markitdown, paddleocr-vl, ...) into GFM, or row-newline-separated HTML+ // for merged / non-GFM tables, so the chunker can split them.+ // Runs before image resolution so <img> inside HTML tables becomes+ // markdown the resolver can persist, instead of rewriting data-URI+ // <img> tags to `` and then escaping them in a later+ // HTML→GFM pass. Line endings are normalized first so injected row+ // breaks are plain LF.+ sanitizeReadResult(convertResult)+ if convertResult != nil {+ convertResult.MarkdownContent = chunker.NormalizeLineEndings(convertResult.MarkdownContent)+ if convertResult.MarkdownContent != "" {+ convertResult.MarkdownContent = docparser.NormalizeHTMLTables(convertResult.MarkdownContent)+ }+ }+// Step 2: Store images and update markdown referencesvar storedImages []docparser.StoredImage@@ -3719,11 +3735,8 @@ func (s *knowledgeService) ProcessDocument(ctx context.Context, t *asynq.Task) elogger.Infof(ctx, "Resolved %d total images for knowledge %s", len(storedImages), knowledge.ID)}- // Step 3: Split into chunks using Go chunker. Browser textareas normalize- // pasted content to LF, so normalize uploaded source text before calculating- // chunk boundaries as well.- sanitizeReadResult(convertResult)- convertResult.MarkdownContent = chunker.NormalizeLineEndings(convertResult.MarkdownContent)+ // Step 3: Split into chunks using Go chunker. Line endings and inline+ // HTML tables were normalized before image resolution above.chunkCfg := buildSplitterConfigFromChunking(eff.ChunkingConfig)processOpts := ProcessChunksOptions{diff --git a/internal/application/service/ocr_sanitizer.go b/internal/application/service/ocr_sanitizer.goindex dfa5fb6e6..fba14b033 100644--- a/internal/application/service/ocr_sanitizer.go+++ b/internal/application/service/ocr_sanitizer.go@@ -5,6 +5,7 @@ import ("strings"htmltomd "github.com/JohannesKaufmann/html-to-markdown/v2"+ "github.com/Tencent/WeKnora/internal/infrastructure/docparser")var (@@ -49,6 +50,13 @@ func sanitizeOCRText(raw string) string {}}+ // Convert inline HTML <table> blocks to GFM tables. This is a safe no-op+ // when no <table> is present, so it runs unconditionally: a markdown body+ // with an embedded HTML table does not satisfy looksLikeHTML (it neither+ // starts with a tag nor is dominated by tag characters), yet leaving the+ // raw markup in place makes the chunker split inside table rows.+ text = docparser.NormalizeHTMLTables(text)+if isKnownEmptyReply(text) {return ""}diff --git a/internal/application/service/ocr_sanitizer_test.go b/internal/application/service/ocr_sanitizer_test.goindex b6a91c9c0..1281bc28a 100644--- a/internal/application/service/ocr_sanitizer_test.go+++ b/internal/application/service/ocr_sanitizer_test.go@@ -1,6 +1,10 @@package service-import "testing"+import (+ "regexp"+ "strings"+ "testing"+)func TestSanitizeOCRText(t *testing.T) {tests := []struct {@@ -187,6 +191,46 @@ func TestLooksLikeHTML(t *testing.T) {}}+// TestSanitizeOCRText_ConvertsInlineHTMLTable guards against a regression where+// an HTML <table> embedded in a markdown body survived sanitization untouched:+// the mixed content does not satisfy looksLikeHTML, so the old HTML-to-markdown+// path never ran and the raw <table> markup reached the chunker.+func TestSanitizeOCRText_ConvertsInlineHTMLTable(t *testing.T) {+ para := "本季度公司整体经营情况保持稳定,营收与利润两项核心指标均实现同比增长。" ++ "为了便于管理层快速了解经营成果,下表汇总了报告期内的主要财务数据," ++ "供后续经营分析、预算编制以及年度考核等工作参考使用," ++ "请结合实际业务情况综合判断,切勿脱离业务背景单独解读其中任何一项数字。"+ tail := "以上数据均来自财务部门审核后的正式报表,统计口径与上一报告期保持一致," ++ "未发生会计政策变更或追溯调整。如有疑问请与财务部门联系确认," ++ "最终解释权归公司财务部门所有。"+ input := "# 报告\n\n" + para + "\n\n" ++ `<table><tr><th>指标</th><th>数值</th></tr>` ++ `<tr><td>营收</td><td>10亿</td></tr>` ++ `<tr><td>利润</td><td>2.3亿</td></tr></table>` ++ "\n\n" + tail + "\n\n"++ if looksLikeHTML(input) {+ t.Fatalf("test input unexpectedly looks like HTML; the regression test must exercise the mixed-content path")+ }++ got := sanitizeOCRText(input)++ if strings.Contains(got, "<table") {+ t.Fatalf("expected inline HTML table to be converted, got:\n%s", got)+ }+ if !strings.Contains(got, "# 报告") || !strings.Contains(got, "最终解释权归公司财务部门所有。") {+ t.Fatalf("expected surrounding markdown to be preserved, got:\n%s", got)+ }+ for _, want := range []string{"指标", "数值", "营收", "10亿", "利润", "2.3亿"} {+ if !strings.Contains(got, want) {+ t.Fatalf("expected converted table to retain %q, got:\n%s", want, got)+ }+ }+ if !regexp.MustCompile(`\|[-:]{3,}\|`).MatchString(got) {+ t.Fatalf("expected a GFM table separator row, got:\n%s", got)+ }+}+func TestIsKnownEmptyReply(t *testing.T) {tests := []struct {input stringdiff --git a/internal/application/service/temporary_document.go b/internal/application/service/temporary_document.goindex 4a0976d27..2c478d3f6 100644--- a/internal/application/service/temporary_document.go+++ b/internal/application/service/temporary_document.go@@ -383,6 +383,8 @@ func (s *temporaryDocumentService) Process(ctx context.Context, task *asynq.Taskreturn nil}content = common.CleanInvalidUTF8(content)+ content = chunker.NormalizeLineEndings(content)+ content = docparser.NormalizeHTMLTables(content)lang := chunker.DetectLanguage(content)cfg := chunker.DefaultConfig()cfg.Strategy = chunker.StrategyAuto@@ -463,6 +465,10 @@ func (s *temporaryDocumentService) parse(ctx context.Context, document *types.Teif err != nil {return "", nil, nil, fmt.Errorf("parse document: %w", err)}+ if result != nil && result.MarkdownContent != "" {+ result.MarkdownContent = chunker.NormalizeLineEndings(result.MarkdownContent)+ result.MarkdownContent = docparser.NormalizeHTMLTables(result.MarkdownContent)+ }// Capture raw page-image bytes before ResolveAndStore stores/rewrites them,// so the VLM OCR fallback for scanned documents has bytes to work with.maxOCRPages := temporaryDocumentImageOCRMaxPages()diff --git a/internal/handler/chunker_debug.go b/internal/handler/chunker_debug.goindex 3e12d2496..6d045a917 100644--- a/internal/handler/chunker_debug.go+++ b/internal/handler/chunker_debug.go@@ -13,6 +13,7 @@ import ("unicode/utf8""github.com/Tencent/WeKnora/internal/infrastructure/chunker"+ "github.com/Tencent/WeKnora/internal/infrastructure/docparser""github.com/Tencent/WeKnora/internal/logger""github.com/gin-gonic/gin")@@ -147,6 +148,7 @@ func PreviewChunking(c *gin.Context) {return}text := chunker.NormalizeLineEndings(req.Text)+ text = docparser.NormalizeHTMLTables(text)cfg := chunker.NormalizeSplitterConfig(chunker.SplitterConfig{ChunkSize: req.ChunkingConfig.ChunkSize,diff --git a/internal/infrastructure/chunker/protected_spans_test.go b/internal/infrastructure/chunker/protected_spans_test.gonew file mode 100644index 000000000..3fe2cda0c--- /dev/null+++ b/internal/infrastructure/chunker/protected_spans_test.go@@ -0,0 +1,135 @@+package chunker++import (+ "strconv"+ "strings"+ "testing"+)++// buildOCRStrayBracketDoc models the OCR failure mode behind this regression:+// a full-width "【公章】" whose opening bracket was mis-read as the half-width+// '[' (closing bracket lost), followed later by an unrelated half-width+// "](...)" fragment. CommonMark forbids a link's text/destination from+// spanning a blank line, but the previous unbounded+// `\[[^\]]*\]\([^)]+\)` joined the stray '[' to the distant '](' and+// swallowed the entire document as one protected atomic span.+func buildOCRStrayBracketDoc() string {+ var sb strings.Builder+ sb.WriteString("[公章】\n\n")+ for i := 0; i < 60; i++ {+ sb.WriteString("这是第")+ sb.WriteString(strconv.Itoa(i))+ sb.WriteString("段用于测试切分的中文内容,包含常见标点符号。\n\n")+ }+ sb.WriteString("第三段(北京)\n\n")+ sb.WriteString("[引用](来源)")+ return sb.String()+}++func maxChunkRunes(chunks []Chunk) int {+ best := 0+ for _, c := range chunks {+ if n := len([]rune(c.Content)); n > best {+ best = n+ }+ }+ return best+}++// TestSplitText_OCRStrayBracketDoesNotSwallowDocument is the core regression:+// before the fix the malformed '[' turned the whole document into a single+// protected span (~1700 runes) and SplitText returned one oversized chunk.+func TestSplitText_OCRStrayBracketDoesNotSwallowDocument(t *testing.T) {+ doc := buildOCRStrayBracketDoc()+ if !strings.Contains(doc, "[公章】") || !strings.Contains(doc, "](来源)") {+ t.Fatalf("fixture missing malformed-bracket markers")+ }+ const chunkSize = 128+ cfg := SplitterConfig{ChunkSize: chunkSize, ChunkOverlap: 0, Separators: []string{"\n\n", "\n", "。"}}++ chunks := SplitText(doc, cfg)+ t.Logf("doc=%d runes chunkSize=%d -> chunks=%d max=%d runes",+ len([]rune(doc)), chunkSize, len(chunks), maxChunkRunes(chunks))+ if len(chunks) < 2 {+ t.Fatalf("expected multiple chunks, got %d (max=%d runes)", len(chunks), maxChunkRunes(chunks))+ }+ if got := maxChunkRunes(chunks); got > 2*chunkSize {+ t.Fatalf("max chunk %d runes exceeds 2*chunkSize=%d: stray '[' swallowed the document", got, 2*chunkSize)+ }+}++// TestSplit_OCRStrayBracketDoesNotSwallowDocument exercises the strategy-aware+// entry point with the legacy tier pinned, so the protected-pattern behaviour+// is what is under test rather than the profiler's tier selection.+func TestSplit_OCRStrayBracketDoesNotSwallowDocument(t *testing.T) {+ doc := buildOCRStrayBracketDoc()+ const chunkSize = 128+ cfg := SplitterConfig{+ ChunkSize: chunkSize,+ ChunkOverlap: 0,+ Separators: []string{"\n\n", "\n", "。"},+ Strategy: StrategyLegacy,+ }++ chunks := Split(doc, cfg)+ if len(chunks) < 2 {+ t.Fatalf("expected multiple chunks, got %d (max=%d runes)", len(chunks), maxChunkRunes(chunks))+ }+ if got := maxChunkRunes(chunks); got > 2*chunkSize {+ t.Fatalf("max chunk %d runes exceeds 2*chunkSize=%d: stray '[' swallowed the document", got, 2*chunkSize)+ }+}++// TestProtectedSpans_NoCrossLineLinkOrImage asserts no protected span may+// contain a newline for the malformed document: link text and destination+// cannot span a blank line (CommonMark).+func TestProtectedSpans_NoCrossLineLinkOrImage(t *testing.T) {+ doc := buildOCRStrayBracketDoc()+ for _, s := range protectedSpans(doc) {+ seg := doc[s.start:s.end]+ if strings.Contains(seg, "\n") {+ t.Fatalf("protected span spans a newline (%d runes): %q", len([]rune(seg)), seg)+ }+ }+}++// TestProtectedSpans_MarkdownLinksAndImagesStillProtected is the positive+// guard: the newline-bounded patterns must keep recognising well-formed+// single-line Markdown links and images.+func TestProtectedSpans_MarkdownLinksAndImagesStillProtected(t *testing.T) {+ cases := []string{+ "",+ "[文字](http://x.com)",+ `[link](url "title")`,+ "",+ }+ for _, c := range cases {+ spans := protectedSpans(c)+ if len(spans) == 0 {+ t.Errorf("expected %q to be protected, got no span", c)+ continue+ }+ if spans[0].start != 0 || spans[0].end != len(c) {+ t.Errorf("%q: protected span = [%d,%d), want [0,%d)", c, spans[0].start, spans[0].end, len(c))+ }+ }+}++// TestProtectedSpans_LiteralCrossLineExample documents that an isolated+// malformed bracket pair with NO complete '](...)' construct is never treated+// as a protected link/image region.+func TestProtectedSpans_MultilineLinkIntentionallyUnprotected(t *testing.T) {+ // CommonMark allows a single soft line break in link text. The protected+ // patterns still refuse it so a stray OCR '[' cannot swallow a paragraph.+ doc := "[link\ntext](http://example.com)"+ if spans := protectedSpans(doc); len(spans) != 0 {+ t.Fatalf("expected multiline link to stay unprotected, got %v", spans)+ }+}++func TestProtectedSpans_LiteralCrossLineExample(t *testing.T) {+ doc := "[公章】\n\n第一段。\n\n第二段。\n\n第三段(北京)"+ if spans := protectedSpans(doc); len(spans) != 0 {+ t.Fatalf("expected no protected span, got %v", spans)+ }+}diff --git a/internal/infrastructure/chunker/splitter.go b/internal/infrastructure/chunker/splitter.goindex ff415eed6..69ff38e43 100644--- a/internal/infrastructure/chunker/splitter.go+++ b/internal/infrastructure/chunker/splitter.go@@ -114,10 +114,18 @@ func DefaultConfig() SplitterConfig {}// protectedPatterns are regex patterns for content that must not be split.+//+// The Markdown link/image patterns deliberately exclude '\n' and bound the+// link text / destination length: CommonMark forbids them from spanning a+// blank line, and the previous unbounded [^\]]* / [^)]+ let a stray '[' left+// behind by OCR swallow whole paragraphs as one "protected" atomic span,+// defeating chunking entirely.var protectedPatterns = []*regexp.Regexp{- regexp.MustCompile(`(?s)\$\$.*?\$\$`), // LaTeX block math- regexp.MustCompile(`!\[[^\]]*\]\([^)]+\)`), // Markdown images- regexp.MustCompile(`\[[^\]]*\]\([^)]+\)`), // Markdown links+ regexp.MustCompile(`(?s)\$\$.*?\$\$`), // LaTeX block math+ // Markdown images / links: single line, bounded so a stray OCR '['+ // cannot swallow a paragraph (CommonMark forbids blank-line spans).+ regexp.MustCompile(`!\[[^\]\n]{0,200}\]\([^)\n]{1,500}\)`),+ regexp.MustCompile(`\[[^\]\n]{1,200}\]\([^)\n]{1,500}\)`),regexp.MustCompile("(?m)[ ]*(?:\\|[^|\\n]*)+\\|[\\r\\n]+\\s*(?:\\|\\s*:?-{3,}:?\\s*)+\\|[\\r\\n]+"), // Table header+separatorregexp.MustCompile("(?m)[ ]*(?:\\|[^|\\n]*)+\\|[\\r\\n]+"), // Table rowsregexp.MustCompile("(?s)```(?:\\w+)?[\\r\\n].*?```"), // Fenced code blocksdiff --git a/internal/infrastructure/docparser/html_table_normalizer.go b/internal/infrastructure/docparser/html_table_normalizer.goindex ad55b1967..8851c2ba7 100644--- a/internal/infrastructure/docparser/html_table_normalizer.go+++ b/internal/infrastructure/docparser/html_table_normalizer.go@@ -15,6 +15,10 @@ var (// block, the form OCR/layout engines such as PaddleOCR-VL emit tables in.htmlTableBlockPattern = regexp.MustCompile(`(?is)<table\b[^>]*>.*?</table>`)+ // fencedCodeBlockPattern matches CommonMark fenced code blocks so HTML+ // <table> examples inside them are left untouched.+ fencedCodeBlockPattern = regexp.MustCompile("(?s)(?:```|~~~)[^\\n]*\\n.*?(?:```|~~~)")+// htmlLayoutAttrPattern matches presentational HTML attributes that carry// no semantic value (text-align styles, CSS classes, sizing). Structural// attributes like rowspan/colspan are intentionally excluded.@@ -22,28 +26,45 @@ var (`(?is)\s+(?:style|class|align|valign|width|height|bgcolor)\s*=\s*(?:"[^"]*"|'[^']*'|[^\s>]+)`,)- // htmlSpanAttrPattern detects rowspan/colspan, which Markdown tables cannot- // represent; such tables keep their HTML form (attributes stripped) instead.- htmlSpanAttrPattern = regexp.MustCompile(`(?i)\b(?:row|col)span\b`)+ // htmlSpanAttrPattern detects rowspan/colspan values greater than 1, which+ // Markdown tables cannot represent; such tables keep their HTML form+ // (attributes stripped) instead. Span values of 1 (and the invalid 0) do+ // not merge anything and must stay convertible.+ htmlSpanAttrPattern = regexp.MustCompile(`(?i)\b(?:row|col)span\s*=\s*["']?(?:[2-9]|\d{2,})`)++ // htmlTableRowPattern matches the opening <tr> tag of each table row, used+ // to put every row on its own line so the chunker can split degraded HTML+ // tables at "\n" boundaries.+ htmlTableRowPattern = regexp.MustCompile(`(?i)<tr\b`)- // markdownTableSeparatorPattern matches the |---|---| delimiter row that a- // valid GFM table must contain.- markdownTableSeparatorPattern = regexp.MustCompile(`(?m)^\s*\|?\s*:?-+:?\s*(?:\|\s*:?-+:?\s*)+\|?\s*$`)+ // markdownTableSeparatorPattern matches the |---| delimiter row that a+ // valid GFM table must contain. The repeating column group is optional so+ // single-column tables (|---|) are accepted; requiring two or more columns+ // discarded successful conversions of MinerU figure/TOC tables.+ markdownTableSeparatorPattern = regexp.MustCompile(`(?m)^\s*\|?\s*:?-+:?\s*(?:\|\s*:?-+:?\s*)*\|?\s*$`))-// normalizeHTMLTables rewrites inline HTML <table> blocks embedded in OCR+// NormalizeHTMLTables rewrites inline HTML <table> blocks embedded in OCR// markdown output. PaddleOCR-VL emits tables as HTML with per-cell text-align// styles, which (1) waste tokens on layout markup and (2) are not recognized// by the chunker's table-protection logic, so large tables get split mid-row.//// Each table block is converted to a GFM Markdown table when possible. Tables-// that use rowspan/colspan (which Markdown cannot express) fall back to having-// their presentational attributes stripped so they stay intact as HTML.-func normalizeHTMLTables(md string) string {+// that use rowspan/colspan (which Markdown cannot express), or that the+// converter cannot turn into a valid GFM table (cell <br>, nested lists, …),+// fall back to presentational-attribute stripping plus one row per line so+// the chunker can still split on "\n". Tables inside fenced code blocks are+// left unchanged.+func NormalizeHTMLTables(md string) string {if !strings.Contains(strings.ToLower(md), "<table") {return md}+ locs := htmlTableBlockPattern.FindAllStringIndex(md, -1)+ if len(locs) == 0 {+ return md+ }+conv := converter.NewConverter(converter.WithPlugins(base.NewBasePlugin(),@@ -51,23 +72,60 @@ func normalizeHTMLTables(md string) string {table.NewTablePlugin(),),)+ fences := fencedCodeBlockPattern.FindAllStringIndex(md, -1)- return htmlTableBlockPattern.ReplaceAllStringFunc(md, func(block string) string {- if htmlSpanAttrPattern.MatchString(block) {- return stripHTMLLayoutAttrs(block)+ var b strings.Builder+ last := 0+ for _, loc := range locs {+ b.WriteString(md[last:loc[0]])+ block := md[loc[0]:loc[1]]+ if indexInsideSpans(loc[0], fences) {+ b.WriteString(block)+ last = loc[1]+ continue}- converted, err := conv.ConvertString(block)- if err != nil {- return stripHTMLLayoutAttrs(block)+ // Isolate the rewritten table with blank lines without stacking pads+ // on repeated NormalizeHTMLTables calls.+ out := strings.TrimRight(b.String(), "\n")+ b.Reset()+ b.WriteString(out)+ b.WriteString("\n\n")+ b.WriteString(normalizeOneHTMLTable(conv, block))+ last = loc[1]+ for last < len(md) && md[last] == '\n' {+ last++}- converted = strings.TrimSpace(converted)- if converted == "" || !markdownTableSeparatorPattern.MatchString(converted) {- return stripHTMLLayoutAttrs(block)+ b.WriteString("\n\n")+ }+ b.WriteString(md[last:])+ return b.String()+}++func normalizeOneHTMLTable(conv *converter.Converter, block string) string {+ fallback := func() string {+ return splitHTMLTableRows(stripHTMLLayoutAttrs(block))+ }+ if htmlSpanAttrPattern.MatchString(block) {+ return fallback()+ }+ converted, err := conv.ConvertString(block)+ if err != nil {+ return fallback()+ }+ converted = unescapeMarkdownImageSyntax(strings.TrimSpace(converted))+ if converted == "" || !markdownTableSeparatorPattern.MatchString(converted) {+ return fallback()+ }+ return converted+}++func indexInsideSpans(pos int, spans [][]int) bool {+ for _, s := range spans {+ if pos >= s[0] && pos < s[1] {+ return true}- // Pad with blank lines so the Markdown table is a standalone block that- // the chunker recognizes (and protects) as a table.- return "\n\n" + converted + "\n\n"- })+ }+ return false}// stripHTMLLayoutAttrs removes presentational attributes from an HTML fragment@@ -75,3 +133,27 @@ func normalizeHTMLTables(md string) string {func stripHTMLLayoutAttrs(html string) string {return htmlLayoutAttrPattern.ReplaceAllString(html, "")}++// splitHTMLTableRows puts each table row on its own line. The chunker splits+// on "\n", so a single-line HTML table would otherwise be unsplittable and get+// force-cut at the absolute max size. Whitespace between tags is insignificant+// in HTML, and newlines are only inserted before <tr> (never inside a tag).+// A <tr> that already starts on its own line is left alone.+func splitHTMLTableRows(block string) string {+ locs := htmlTableRowPattern.FindAllStringIndex(block, -1)+ if len(locs) == 0 {+ return strings.TrimSpace(block)+ }+ var b strings.Builder+ last := 0+ for _, loc := range locs {+ b.WriteString(block[last:loc[0]])+ if loc[0] == 0 || block[loc[0]-1] != '\n' {+ b.WriteByte('\n')+ }+ b.WriteString(block[loc[0]:loc[1]])+ last = loc[1]+ }+ b.WriteString(block[last:])+ return strings.TrimSpace(b.String())+}diff --git a/internal/infrastructure/docparser/html_table_normalizer_test.go b/internal/infrastructure/docparser/html_table_normalizer_test.goindex 8ae31d591..19b036bbc 100644--- a/internal/infrastructure/docparser/html_table_normalizer_test.go+++ b/internal/infrastructure/docparser/html_table_normalizer_test.go@@ -16,7 +16,7 @@ func TestNormalizeHTMLTables_ConvertsStyledTableToMarkdown(t *testing.T) {结尾。`- got := normalizeHTMLTables(input)+ got := NormalizeHTMLTables(input)if strings.Contains(got, "<table") {t.Fatalf("expected HTML table to be converted away, got:\n%s", got)@@ -37,13 +37,92 @@ func TestNormalizeHTMLTables_ConvertsStyledTableToMarkdown(t *testing.T) {}}+func TestNormalizeHTMLTables_NoTableUnchanged(t *testing.T) {+ input := "# 标题\n\n普通段落,没有表格。\n\n| a | b |\n| --- | --- |\n| 1 | 2 |"+ if got := NormalizeHTMLTables(input); got != input {+ t.Fatalf("expected content without HTML tables to be unchanged, got:\n%s", got)+ }+}++func TestNormalizeHTMLTables_SpanValueOneIsConvertible(t *testing.T) {+ // rowspan=1 / colspan=1 merge nothing, so the table must still become GFM.+ // Covers quoted, unquoted and space-padded attribute forms.+ input := `<table><tr>` ++ `<td rowspan="1" colspan="1" style="text-align:center;">指标</td>` ++ `<td rowspan=1 colspan=1>数值</td></tr>` ++ `<tr><td rowspan="1" colspan='1'>营收</td>` ++ `<td rowspan = 1 colspan = 1>10亿</td></tr></table>`++ got := NormalizeHTMLTables(input)++ if strings.Contains(got, "<table") {+ t.Fatalf("expected span=1 table to be converted away, got:\n%s", got)+ }+ if !markdownTableSeparatorPattern.MatchString(got) {+ t.Fatalf("expected a Markdown table separator row, got:\n%s", got)+ }+ for _, want := range []string{"指标", "数值", "营收", "10亿"} {+ if !strings.Contains(got, want) {+ t.Fatalf("expected converted table to retain %q, got:\n%s", want, got)+ }+ }+}++func TestNormalizeHTMLTables_RealSpanKeepsHTMLWithRowNewlines(t *testing.T) {+ // Genuine merges (colspan>1 / rowspan>1) cannot be GFM, so the table stays+ // HTML — but each row must start on its own line so the chunker can split it.+ input := `<table><tr><td colspan="6" style="text-align:center;">合计</td></tr>` ++ `<tr><td rowspan="3">A</td><td>1</td></tr>` ++ `<tr><td rowspan = 2>B</td><td>2</td></tr></table>`++ got := NormalizeHTMLTables(input)++ if !strings.Contains(got, "<table") {+ t.Fatalf("expected real span table to remain HTML, got:\n%s", got)+ }+ if !strings.Contains(got, `colspan="6"`) || !strings.Contains(got, `rowspan="3"`) {+ t.Fatalf("expected structural span attrs preserved, got:\n%s", got)+ }+ if !strings.Contains(got, "\n<tr") {+ t.Fatalf("expected each <tr> to start on a new line, got:\n%q", got)+ }+ if !strings.Contains(got, "\n\n<table") {+ t.Fatalf("expected block to be padded at the head with blank lines, got:\n%q", got)+ }+ if !strings.HasSuffix(got, "</table>\n\n") {+ t.Fatalf("expected block to be padded at the tail with blank lines, got:\n%q", got)+ }+ if n := strings.Count(got, "\n<tr"); n != 3 {+ t.Fatalf("expected 3 rows each preceded by a newline, got %d in:\n%q", n, got)+ }+}++func assertHTMLRowsSplittable(t *testing.T, got string) {+ t.Helper()+ if !strings.Contains(got, "<tr") {+ t.Fatalf("expected remaining HTML table rows, got:\n%s", got)+ }+ for i := 0; ; {+ idx := strings.Index(got[i:], "<tr")+ if idx < 0 {+ return+ }+ abs := i + idx+ if abs == 0 || got[abs-1] != '\n' {+ t.Fatalf("remaining <tr> at offset %d is not preceded by a newline:\n%s", abs, got)+ }+ i = abs + 1+ }+}+func TestNormalizeHTMLTables_StripsAttrsOnSpanTables(t *testing.T) {// rowspan/colspan cannot be expressed in Markdown, so the table stays HTML- // but its presentational attributes are stripped.- input := `<table><tr><td colspan="2" style="text-align:center;" class="hdr">合计</td></tr>` ++ // but its presentational attributes are stripped and rows become splittable.+ input := `<table><tr>` ++ `<td colspan="2" style="text-align:center;" class="hdr">合计</td></tr>` +`<tr><td style="text-align:left;">A</td><td width="80">B</td></tr></table>`- got := normalizeHTMLTables(input)+ got := NormalizeHTMLTables(input)if !strings.Contains(got, "<table") {t.Fatalf("expected span table to remain HTML, got:\n%s", got)@@ -56,11 +135,99 @@ func TestNormalizeHTMLTables_StripsAttrsOnSpanTables(t *testing.T) {t.Fatalf("expected %q to be stripped, got:\n%s", banned, got)}}+ assertHTMLRowsSplittable(t, got)}-func TestNormalizeHTMLTables_NoTableUnchanged(t *testing.T) {- input := "# 标题\n\n普通段落,没有表格。\n\n| a | b |\n| --- | --- |\n| 1 | 2 |"- if got := normalizeHTMLTables(input); got != input {- t.Fatalf("expected content without HTML tables to be unchanged, got:\n%s", got)+func TestNormalizeHTMLTables_BRCellsStaySplittableHTML(t *testing.T) {+ // <br> inside a cell makes the table plugin give up (it emits a newline).+ // The HTML fallback must still put each <tr> on its own line.+ input := `<table><tr><td>指标<br/>名称</td><td>数值<br/>单位</td></tr>` ++ `<tr><td>拉伸强度<br/>MPa</td><td>合格<br/>A级</td></tr></table>`++ got := NormalizeHTMLTables(input)++ if !strings.Contains(got, "<table") {+ t.Fatalf("expected <br> table to remain HTML rather than flattened text, got:\n%s", got)+ }+ assertHTMLRowsSplittable(t, got)+ for _, want := range []string{"指标", "名称", "拉伸强度", "合格"} {+ if !strings.Contains(got, want) {+ t.Fatalf("expected fallback HTML to retain %q, got:\n%s", want, got)+ }+ }+}++func TestNormalizeHTMLTables_ListInCellFallsBackToSplittableHTML(t *testing.T) {+ input := `<table><tr><td><ul><li>a</li><li>b</li></ul></td><td>x</td></tr>` ++ `<tr><td>c</td><td>d</td></tr></table>`++ got := NormalizeHTMLTables(input)++ if !strings.Contains(got, "<table") {+ t.Fatalf("expected list-in-cell table to remain HTML, got:\n%s", got)+ }+ assertHTMLRowsSplittable(t, got)+}++func TestNormalizeHTMLTables_SingleColumnConvertsToGFM(t *testing.T) {+ input := `<table><tr><td>目录</td></tr><tr><td>第一章</td></tr><tr><td>第二章</td></tr></table>`++ got := NormalizeHTMLTables(input)++ if strings.Contains(got, "<table") {+ t.Fatalf("expected single-column table to convert to GFM, got:\n%s", got)+ }+ if !markdownTableSeparatorPattern.MatchString(got) {+ t.Fatalf("expected a Markdown table separator row, got:\n%s", got)+ }+ for _, want := range []string{"目录", "第一章", "第二章"} {+ if !strings.Contains(got, want) {+ t.Fatalf("expected converted table to retain %q, got:\n%s", want, got)+ }+ }+}++func TestNormalizeHTMLTables_ImgInSingleColumnBecomesGFM(t *testing.T) {+ input := `<table><tr><td><img src="local://images/profile.png" alt="profile"/></td></tr></table>`++ got := NormalizeHTMLTables(input)++ if strings.Contains(got, "<table") {+ t.Fatalf("expected single-column image table to convert to GFM, got:\n%s", got)+ }+ if !strings.Contains(got, "") {+ t.Fatalf("expected markdown image to be preserved, got:\n%s", got)+ }+}++func TestNormalizeHTMLTables_MarkdownImageInCellNotEscaped(t *testing.T) {+ input := `<table><tr><td></td><td>说明</td></tr></table>`++ got := NormalizeHTMLTables(input)++ if strings.Contains(got, `!\[`) {+ t.Fatalf("markdown image syntax escaped:\n%s", got)+ }+ if !strings.Contains(got, "") {+ t.Fatalf("expected markdown image to survive conversion, got:\n%s", got)+ }+}++func TestNormalizeHTMLTables_CodeFenceTableLeftAlone(t *testing.T) {+ input := "示例:\n\n```html\n<table><tr><td>a</td><td>b</td></tr></table>\n```\n"++ got := NormalizeHTMLTables(input)++ if got != input {+ t.Fatalf("expected fenced HTML table to be unchanged, got:\n%s", got)+ }+}++func TestNormalizeHTMLTables_SpanTableIdempotent(t *testing.T) {+ input := `<table><tr><td colspan="6">汇总</td></tr><tr><td colspan="6">备注</td></tr></table>`+ once := NormalizeHTMLTables(input)+ twice := NormalizeHTMLTables(once)+ if once != twice {+ t.Fatalf("NormalizeHTMLTables is not idempotent for span tables:\n1=%q\n2=%q", once, twice)}}diff --git a/internal/infrastructure/docparser/paddleocr_vl_cloud_converter.go b/internal/infrastructure/docparser/paddleocr_vl_cloud_converter.goindex 98cfc0133..c05c9561a 100644--- a/internal/infrastructure/docparser/paddleocr_vl_cloud_converter.go+++ b/internal/infrastructure/docparser/paddleocr_vl_cloud_converter.go@@ -78,7 +78,7 @@ func (c *PaddleOCRVLCloudReader) Read(ctx context.Context, req *types.ReadRequesreturn nil, fmt.Errorf("PaddleOCR-VL Cloud fetch results: %w", err)}- mdContent = normalizeHTMLTables(mdContent)+ mdContent = NormalizeHTMLTables(mdContent)imageRefs := c.downloadImages(mdContent, imagesURL)mdContent, imageRefs = ensureOriginalImageRef(req, mdContent, imageRefs)diff --git a/internal/infrastructure/docparser/paddleocr_vl_converter.go b/internal/infrastructure/docparser/paddleocr_vl_converter.goindex 313d3b74e..f49e1df58 100644--- a/internal/infrastructure/docparser/paddleocr_vl_converter.go+++ b/internal/infrastructure/docparser/paddleocr_vl_converter.go@@ -65,7 +65,7 @@ func (c *PaddleOCRVLReader) Read(ctx context.Context, req *types.ReadRequest) (*// wastes tokens and defeats the chunker's table-protection logic. Convert// them to Markdown tables (or strip layout attributes when conversion is// not possible) before downstream processing.- mdContent = normalizeHTMLTables(mdContent)+ mdContent = NormalizeHTMLTables(mdContent)imageRefs, mdContent := c.processImages(mdContent, imagesB64)mdContent, imageRefs = ensureOriginalImageRef(req, mdContent, imageRefs)