RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.
Go agent-harnessagent-native-databaseagentic-aiagentic-retrievalagentic-searchaiai-agentscontext-enginecontext-engineeringcontext-managementharness-engineeringknowledge-compilationragretrieval-augmented-generationretrieval-harnesssearch-harness
fix(deepdoc): keep body sentences that cite Table N out of table captions (#21453)
CCLUCKYbaf314d58877
diff --git a/internal/deepdoc/parser/pdf/table/caption_kind_test.go b/internal/deepdoc/parser/pdf/table/caption_kind_test.goindex 54d1b72e7a..318b683370 100644--- a/internal/deepdoc/parser/pdf/table/caption_kind_test.go+++ b/internal/deepdoc/parser/pdf/table/caption_kind_test.go@@ -12,7 +12,7 @@ import (// MENTIONS "Table 1" mid-sentence must NOT be classified as a table caption.// Python's is_caption uses re.match (start-anchored), so it keeps such a// paragraph as a normal text section; Go must match that. Before the fix,-// reTableCaptionText matched "Table N" anywhere in the string, so the+// table-caption detection matched "Table N" anywhere in the string, so the// paragraph was mislabeled a caption and silently dropped by MergeCaptions.func TestCaptionKind_MidSentenceTableMentionNotCaption(t *testing.T) {para := "The following analysis compares product performance across different market segments. Table 1 shows revenuebycategory."@@ -36,8 +36,9 @@ func TestCaptionKind_StartsWithTableIsCaption(t *testing.T) {}}-// TestIsCaptionBox_MidSentenceTableMention confirms IsCaptionBox (via reCaption)-// also keeps mid-sentence "Table N" as non-caption, mirroring Python re.match.+// TestIsCaptionBox_MidSentenceTableMention confirms IsCaptionBox (via the+// shared caption detection) also keeps mid-sentence "Table N" as non-caption,+// mirroring Python re.match.func TestIsCaptionBox_MidSentenceTableMention(t *testing.T) {para := "The following analysis compares product performance across different market segments. Table 1 shows revenuebycategory."if IsCaptionBox(para, pdf.LayoutTypeText) {@@ -113,6 +114,131 @@ func TestCaptionKind_CJKCaptionMarkersStillMatch(t *testing.T) {}}+// TestCaptionKind_EnglishTableCrossReferenceNotCaption locks the fix for+// #20672: a body sentence that STARTS with a table cross-reference is not a+// caption. Python's pattern-only is_caption classifies these as captions and+// swallows them into the table; this Go path requires a caption boundary after+// the number, so the sentence stays a standalone text section.+//+// IsCaptionBox deliberately keeps the pattern-only verdict: it decides cell+// filling, no-merge marking, and caption-row stripping, all of which must keep+// matching Python's is_caption.+func TestCaptionKind_EnglishTableCrossReferenceNotCaption(t *testing.T) {+ for _, text := range []string{+ "Table 1, we present the F1 scores and success rates for KNOWAGENT.",+ "Table 9 reports the adaptive expansion ablation, as shown in Figure 5.",+ "Table 1 shows revenue by category for the fiscal year.",+ "Table 2 (see Appendix A) reports the remaining configurations.",+ "Table 1's results show a 12% improvement over the baseline.",+ "Table 1.5 shows the ablation without retrieval.",+ } {+ if got := CaptionKind(pdf.Section{Text: text, LayoutType: pdf.LayoutTypeText}); got != "" {+ t.Errorf("CaptionKind(%q) = %q, want \"\" (cross-reference is body text)", text, got)+ }+ if !IsCaptionBox(text, pdf.LayoutTypeText) {+ t.Errorf("IsCaptionBox(%q) = false, want true (pattern-only contract)", text)+ }+ }+}++// TestCaptionKind_EnglishTableCaptionBoundaries pins the positive side of the+// boundary rule: separators, title-like words, end of line, letter-suffixed+// numbers, and parenthesised continuations/panel labels still classify as table+// captions.+func TestCaptionKind_EnglishTableCaptionBoundaries(t *testing.T) {+ for _, text := range []string{+ "Table 2: Comparative Experiment of ReAct",+ "Table 1. Data statistics",+ "Table 1",+ "Table 2 Performance comparison",+ "Table 2 (continued)",+ "Table 2 (cont'd)",+ "Table 2 (a) Confusion matrix",+ "Table 1a. Results",+ "Table 1b: Results",+ "Table 1 (iv) Confusion matrix",+ "Table 1 (12) Results",+ "Table 1.5: Ablation results",+ "Table 1\nRevenue by category",+ "Table 1\u00a0Revenue by category",+ "Table 1\u3000Revenue by category",+ } {+ if got := CaptionKind(pdf.Section{Text: text, LayoutType: pdf.LayoutTypeText}); got != pdf.LayoutTypeTable {+ t.Errorf("CaptionKind(%q) = %q, want %q", text, got, pdf.LayoutTypeTable)+ }+ if !IsCaptionBox(text, pdf.LayoutTypeText) {+ t.Errorf("IsCaptionBox(%q) = false, want true", text)+ }+ }+}++// TestCaptionKind_LabelKeepsDeclaredKind pins label precedence: a layout label+// names the parent kind even when its text carries no marker (a DLA caption+// such as "Quarterly revenue breakdown"), including non-canonical labels such as+// "table_caption".+func TestCaptionKind_LabelKeepsDeclaredKind(t *testing.T) {+ for _, tt := range []struct{ text, label, want string }{+ {"Quarterly revenue breakdown", pdf.DLALabelTableCaption, pdf.LayoutTypeTable},+ {"Quarterly revenue breakdown", "table_caption", pdf.LayoutTypeTable},+ {"Table 9: Fine-grained retrieval recall.", pdf.DLALabelTableCaption, pdf.LayoutTypeTable},+ {"Figure 1 shows the architecture.", "figure_caption", pdf.LayoutTypeFigure},+ {"Figure 1 shows the architecture.", pdf.DLALabelFigureCaption, pdf.LayoutTypeFigure},+ } {+ if got := CaptionKind(pdf.Section{Text: tt.text, LayoutType: tt.label}); got != tt.want {+ t.Errorf("CaptionKind(%q, label %q) = %q, want %q", tt.text, tt.label, got, tt.want)+ }+ }+}++// TestCaptionKind_LabeledCrossReferenceNotCaption covers the DLA half of+// #20672: when the layout labels a CROSS-REFERENCE sentence as a caption, the+// label must not override the boundary rule, or MergeCaptions injects the prose+// into the table's <caption> (or deletes it when no table is near).+func TestCaptionKind_LabeledCrossReferenceNotCaption(t *testing.T) {+ const prose = "Table 1, we present the F1 scores and success rates."+ for _, label := range []string{+ pdf.DLALabelTableCaption,+ "table_caption",+ pdf.DLALabelFigureCaption,+ "figure_caption",+ "caption",+ } {+ if got := CaptionKind(pdf.Section{Text: prose, LayoutType: label}); got != "" {+ t.Errorf("CaptionKind(labeled %q, %q) = %q, want \"\" (prose is not a caption)", prose, label, got)+ }+ }+}++// TestCaptionKind_FigureBoundaryUnchanged documents the scope of #20672:+// sentence-style FIGURE captions ("Figure 1 shows …") remain captions, because+// that form is a common legitimate caption style and the issue covers tables.+func TestCaptionKind_FigureBoundaryUnchanged(t *testing.T) {+ text := "Figure 1 shows the overall architecture of the system."+ if got := CaptionKind(pdf.Section{Text: text, LayoutType: pdf.LayoutTypeText}); got != pdf.LayoutTypeFigure {+ t.Errorf("CaptionKind(%q) = %q, want %q", text, got, pdf.LayoutTypeFigure)+ }+}++// TestCaptionKind_ProperNounContinuationIsCaption pins the KNOWN LIMITATION of+// the boundary rule: it keys on the first character after "Table N", so a+// cross-reference that continues with a proper noun classifies as a caption+// ("Table 3 Smith et al. report …"), exactly as a real caption title would+// ("Table 3 Smith-Kappa-Rosenberg parameters"). Nothing in the string separates+// them without part-of-speech tagging, so the rule keeps the caption reading.+// The consequence is placement, not deletion: MergeCaptions moves the sentence+// into the nearest table's <caption>, so it stays indexed+// (TestMergeCaptions_ProperNounCrossReferenceLandsInCaption).+func TestCaptionKind_ProperNounContinuationIsCaption(t *testing.T) {+ for _, text := range []string{+ "Table 3 Smith et al. report the F1 scores.",+ "Table 3 Gaussian noise dominates above 40 dB.",+ } {+ if got := CaptionKind(pdf.Section{Text: text, LayoutType: pdf.LayoutTypeText}); got != pdf.LayoutTypeTable {+ t.Errorf("CaptionKind(%q) = %q, want %q (documented limitation)", text, got, pdf.LayoutTypeTable)+ }+ }+}+// hasBox reports whether positions contains a box with the given top and// bottom edges. Tests use it to assert that a merged caption's box reached the// section it was merged into.diff --git a/internal/deepdoc/parser/pdf/table/merge_captions_caption_test.go b/internal/deepdoc/parser/pdf/table/merge_captions_caption_test.goindex 77da354921..12a9c51189 100644--- a/internal/deepdoc/parser/pdf/table/merge_captions_caption_test.go+++ b/internal/deepdoc/parser/pdf/table/merge_captions_caption_test.go@@ -407,12 +407,12 @@ func TestMergeCaptions_FigureCaptionKeptWithMarkerFigure(t *testing.T) {// table OR figure (nearest(tables)/nearest(figures), `elif fk`) and drops it// only when BOTH searches fail. Go used to search tables only and delete the// caption when none was reachable — losing text Python keeps, for English-// body text AND for correctly-DLA-labeled "table caption" sections alike+// text captions AND for correctly-DLA-labeled "table caption" sections alike// (language-independent; the figure here is deliberately neutral-text so the// figure-self-deletion bug cannot mask this path).func TestMergeCaptions_TableCaptionFallsBackToFigure(t *testing.T) {for _, caption := range []pdf.Section{- {Text: "Table 1 shows revenue by category.", LayoutType: pdf.LayoutTypeText,+ {Text: "Table 1: revenue by category.", LayoutType: pdf.LayoutTypeText,Positions: []pdf.Position{{PageNumbers: []int{0}, Left: 100, Right: 500, Top: 510, Bottom: 530}}},{Text: "Quarterly revenue breakdown", LayoutType: pdf.DLALabelTableCaption,Positions: []pdf.Position{{PageNumbers: []int{0}, Left: 100, Right: 500, Top: 510, Bottom: 530}}},@@ -436,27 +436,148 @@ func TestMergeCaptions_TableCaptionFallsBackToFigure(t *testing.T) {}}-// TestMergeCaptions_EnglishTable1OpeningParagraphSwallowed pins the SHARED-// (Python-identical) heuristic boundary of the CJK fix: a body paragraph that-// STARTS with "Table N" is classified a caption by BOTH implementations-// (Python's re.match is start-anchored too) and consumed into the table's-// <caption>. The text is retained — misplacement parity with Python, not a-// Go regression; only the CJK false-positive class was fixed.-func TestMergeCaptions_EnglishTable1OpeningParagraphSwallowed(t *testing.T) {+// TestMergeCaptions_EnglishTableCrossReferenceStaysWithRealCaption models the+// RAG-paper sample from #20672: a real "Table 9: …" caption and a body sentence+// "Table 9 reports …" both sit near the table. Only the real caption may join+// the <caption>; the body sentence stays its own section, and the table's+// highlight geometry covers the caption only.+func TestMergeCaptions_EnglishTableCrossReferenceStaysWithRealCaption(t *testing.T) {+ const (+ caption = "Table 9: Fine-grained retrieval recall for the adaptive expansion ablation."+ body = "Table 9 reports the adaptive expansion ablation, as shown in Figure 5."+ )sections := []pdf.Section{{Text: "<table><tr><td>data</td></tr></table>", LayoutType: pdf.LayoutTypeTable,Positions: []pdf.Position{{PageNumbers: []int{0}, Left: 100, Right: 500, Top: 200, Bottom: 400}}},- {Text: "Table 1 shows revenue by category for the fiscal year.", LayoutType: pdf.LayoutTypeText,+ {Text: caption, LayoutType: pdf.LayoutTypeText,+ Positions: []pdf.Position{{PageNumbers: []int{0}, Left: 100, Right: 500, Top: 170, Bottom: 190}}},+ {Text: body, LayoutType: pdf.LayoutTypeText,+ Positions: []pdf.Position{{PageNumbers: []int{0}, Left: 100, Right: 500, Top: 410, Bottom: 430}}},+ }+ result := MergeCaptions(sections)+ if len(result) != 2 {+ t.Fatalf("expected table + body sentence, got %d sections: %v", len(result), textsOf(result))+ }+ var table *pdf.Section+ foundBody := false+ for i := range result {+ if result[i].LayoutType == pdf.LayoutTypeTable {+ table = &result[i]+ }+ if result[i].LayoutType == pdf.LayoutTypeText && result[i].Text == body {+ foundBody = true+ }+ }+ if table == nil || !strings.Contains(table.Text, "<caption>"+caption+"</caption>") {+ t.Fatalf("real caption not attached to the table: %v", textsOf(result))+ }+ if !foundBody {+ t.Errorf("cross-reference body sentence dropped or rewritten: %v", textsOf(result))+ }+ if len(table.Positions) != 2 || !hasBox(table.Positions, 170, 190) {+ t.Errorf("table geometry must cover the caption box only, got %+v", table.Positions)+ }+ if hasBox(table.Positions, 410, 430) {+ t.Errorf("body sentence box leaked into the table's highlight geometry: %+v", table.Positions)+ }+}++// TestMergeCaptions_EnglishTable1OpeningParagraphKept locks the #20672+// divergence from Python. Start-anchored pattern matching alone classified+// "Table 1 shows revenue …" as a caption, and MergeCaptions then moved the body+// sentence into the table's <caption> and deleted it as a standalone section.+// The sentence now stays body text and the table receives no caption.+func TestMergeCaptions_EnglishTable1OpeningParagraphKept(t *testing.T) {+ const body = "Table 1 shows revenue by category for the fiscal year."+ sections := []pdf.Section{+ {Text: "<table><tr><td>data</td></tr></table>", LayoutType: pdf.LayoutTypeTable,+ Positions: []pdf.Position{{PageNumbers: []int{0}, Left: 100, Right: 500, Top: 200, Bottom: 400}}},+ {Text: body, LayoutType: pdf.LayoutTypeText,Positions: []pdf.Position{{PageNumbers: []int{0}, Left: 100, Right: 500, Top: 170, Bottom: 195}}},}result := MergeCaptions(sections)+ if len(result) != 2 {+ t.Fatalf("expected the body sentence and the table as separate sections, got %d: %v", len(result), textsOf(result))+ }+ found := falsefor _, s := range result {- if s.LayoutType == pdf.LayoutTypeTable && strings.Contains(s.Text, "<caption>Table 1 shows revenue") {- return+ if s.LayoutType == pdf.LayoutTypeTable && strings.Contains(s.Text, "<caption>") {+ t.Errorf("body sentence consumed into <caption>: %v", textsOf(result))+ }+ if s.LayoutType == pdf.LayoutTypeText && s.Text == body {+ found = true+ }+ }+ if !found {+ t.Errorf("body sentence dropped or rewritten; sections = %v", textsOf(result))+ }+}++// TestMergeCaptions_LabeledCrossReferenceStaysBodyText covers the DLA half of+// #20672: a caption LABEL must not override the boundary rule, or MergeCaptions+// injects the prose into the table's <caption> (sibling table present) or+// deletes it outright (no table near).+func TestMergeCaptions_LabeledCrossReferenceStaysBodyText(t *testing.T) {+ const prose = "Table 1, we present the F1 scores and success rates for KNOWAGENT."+ for _, label := range []string{pdf.DLALabelTableCaption, "table_caption"} {+ withTable := []pdf.Section{+ {Text: "<table><tr><td>data</td></tr></table>", LayoutType: pdf.LayoutTypeTable,+ Positions: []pdf.Position{{PageNumbers: []int{0}, Left: 100, Right: 500, Top: 200, Bottom: 400}}},+ {Text: prose, LayoutType: label,+ Positions: []pdf.Position{{PageNumbers: []int{0}, Left: 100, Right: 500, Top: 410, Bottom: 430}}},+ }+ result := MergeCaptions(withTable)+ if len(result) != 2 {+ t.Fatalf("label %q: expected table + prose, got %d sections: %v", label, len(result), textsOf(result))+ }+ for _, s := range result {+ if strings.Contains(s.Text, "<caption>") {+ t.Errorf("label %q: prose injected into <caption>: %v", label, textsOf(result))+ }+ }++ noTable := []pdf.Section{+ {Text: prose, LayoutType: label,+ Positions: []pdf.Position{{PageNumbers: []int{0}, Left: 100, Right: 500, Top: 410, Bottom: 430}}},+ }+ result = MergeCaptions(noTable)+ if len(result) != 1 || result[0].Text != prose {+ t.Errorf("label %q: prose deleted when no table is near: %v", label, textsOf(result))}}- t.Errorf("expected the start-anchored English paragraph consumed into <caption> (Python parity); sections = %v",- textsOf(result))+}++// TestMergeCaptions_ProperNounCrossReferenceLandsInCaption measures the cost of+// the boundary rule's known limitation (see+// TestCaptionKind_ProperNounContinuationIsCaption): the sentence is misread as a+// caption and merged into the table's <caption> rather than dropped, so its+// text stays in the output and indexed with the table. Pinning this keeps the+// limitation honest: if a later change makes the no-target branch delete+// captions again, this test turns RED and the trade-off has to be re-argued.+func TestMergeCaptions_ProperNounCrossReferenceLandsInCaption(t *testing.T) {+ const sentence = "Table 3 Smith et al. report the F1 scores."+ sections := []pdf.Section{+ {Text: "<table><tr><td>data</td></tr></table>", LayoutType: pdf.LayoutTypeTable,+ Positions: []pdf.Position{{PageNumbers: []int{0}, Left: 100, Right: 500, Top: 200, Bottom: 400}}},+ {Text: sentence, LayoutType: pdf.LayoutTypeText,+ Positions: []pdf.Position{{PageNumbers: []int{0}, Left: 100, Right: 500, Top: 170, Bottom: 195}}},+ }+ result := MergeCaptions(sections)+ inCaption := false+ for _, s := range result {+ if s.LayoutType == pdf.LayoutTypeTable {+ start := strings.Index(s.Text, "<caption>")+ end := strings.Index(s.Text, "</caption>")+ if start >= 0 && end > start && strings.Contains(s.Text[start+len("<caption>"):end], sentence) {+ inCaption = true+ }+ } else if strings.Contains(s.Text, sentence) {+ t.Errorf("misclassified sentence survived as a standalone section: %v", textsOf(result))+ }+ }+ if !inCaption {+ t.Errorf("misclassified sentence not placed in the table's <caption>; sections = %v", textsOf(result))+ }}// TestMergeCaptions_FigureCaptionOtherPageDoesNotAttach locks the figure-sidediff --git a/internal/deepdoc/parser/pdf/table/table_annotate_test.go b/internal/deepdoc/parser/pdf/table/table_annotate_test.goindex dbc01b983f..a025847827 100644--- a/internal/deepdoc/parser/pdf/table/table_annotate_test.go+++ b/internal/deepdoc/parser/pdf/table/table_annotate_test.go@@ -735,3 +735,22 @@ func TestMarkNoMergeTables_EmptyInputs(t *testing.T) {MarkNoMergeTables(nil, nil)MarkNoMergeTables([]pdf.TextBox{}, []pdf.TableItem{})}++// TestGarbageKeepFeat_ReferenceKeepsCrossReference pins text retention: a+// reference-labelled box is dropped unless it looks like a caption. IsCaptionBox+// stays pattern-only, so a distinguishing noun phrase that starts with "Table N"+// ("Table 1, we present …") is still kept as text, not dropped as a reference.+func TestGarbageKeepFeat_ReferenceKeepsCrossReference(t *testing.T) {+ for _, text := range []string{+ "Table 1, we present the F1 scores and success rates.",+ "Table 9 reports the adaptive expansion ablation.",+ "Table 2: Comparative Experiment of ReAct",+ } {+ if !garbageKeepFeat(pdf.LayoutTypeReference, pdf.TextBox{Text: text}, 1000) {+ t.Errorf("garbageKeepFeat(reference, %q) = false, want true (text would be dropped)", text)+ }+ }+ if garbageKeepFeat(pdf.LayoutTypeReference, pdf.TextBox{Text: "plain reference text"}, 1000) {+ t.Error("garbageKeepFeat kept a plain reference box")+ }+}diff --git a/internal/deepdoc/parser/pdf/table/table_cells.go b/internal/deepdoc/parser/pdf/table/table_cells.goindex 9b85b18d4d..d2ba9e4cfc 100644--- a/internal/deepdoc/parser/pdf/table/table_cells.go+++ b/internal/deepdoc/parser/pdf/table/table_cells.go@@ -5,6 +5,8 @@ import ("regexp""sort""strings"+ "unicode"+ "unicode/utf8""go.uber.org/zap"@@ -414,61 +416,151 @@ func matchRowStrip(rowStrips []pdf.TSRCell, y0 float64) (float64, float64, bool)return 0, 0, false}-// isCaptionBox checks if a text box is a table/figure caption,-// matching Python is_caption(). Captions should not enter table cells.-// reCaption backs IsCaptionBox and must mirror Python's start-anchored-// is_caption: only a line BEGINNING with a caption marker counts. The English-// alternatives are start-anchored with ^ for the same reason as-// reTableCaptionText/reFigureCaptionText.-var reCaption = regexp.MustCompile(`^[图表]+[ 0-9::]{2,}|(?i)^Fig\.?\s*\d+|(?i)^Figure\s+\d+|(?i)^Table\s+\d+`)-+// IsCaptionBox checks if a text box is a table/figure caption. Captions should+// not enter table cells. Start-anchored detection is shared with Python's+// is_caption. IsCaptionBox stays pattern-only on purpose: its other callers fill+// table cells, mark cross-page no-merge tables, and strip caption rows, and none+// of them may start treating a citing sentence as data. The boundary rule that+// keeps cross-references out of <caption> belongs to CaptionKind.func IsCaptionBox(text string, layoutType string) bool {if strings.Contains(layoutType, "caption") {return true}- return reCaption.MatchString(strings.TrimSpace(text))+ return captionMarkerPrefix(strings.TrimSpace(text))}-// reTableCaptionText matches text patterns that indicate a table caption-// (as opposed to a figure caption). Python is_caption uses re.match, which is-// start-anchored, so a body paragraph that merely MENTIONS "Table N"-// mid-sentence is NOT a caption. The English alternatives below are therefore-// start-anchored with ^: an unanchored alternative wrongly classifies such a-// paragraph as a caption and MergeCaptions then drops it (go_bug-// table-text-interleaved-paragraph-dropped).-//-// The CJK branch mirrors Python's `[图表]+[ 0-9::]{2,}`: the 图/表 run must be-// followed by ≥2 chars from [ 0-9::], so a body paragraph that merely STARTS-// with 表/图 (表格是一种…, 表現方法…, 图中所示…) is not a caption. The bare-// `^表` previously classified any 表-prefixed CJK/Japanese body paragraph as a-// caption that MergeCaptions either swallowed into <caption> or dropped —-// content loss Python never has (go_bug cjk-caption-false-positive). Kind is-// split by the run's first character because findNearestParent needs a-// table/figure kind to pick which parent to search; Python has a single-// pattern and picks the nearest parent of either kind instead.-var reTableCaptionText = regexp.MustCompile(`^表[图表]*[ 0-9::]{2,}|(?i)^Table\s+\d+`)--// reFigureCaptionText matches text patterns that indicate a figure caption —-// the figure-kind split of the same Python pattern (see reTableCaptionText).-var reFigureCaptionText = regexp.MustCompile(`^图[图表]*[ 0-9::]{2,}|(?i)^Fig\.?\s*\d+|(?i)^Figure\s+\d+`)--// captionKind returns "table" if the section is a table caption,-// "figure" if a figure caption, or "" if not a caption.-// Matches Python's is_caption check: text patterns OR layout_type containing "caption".+// reCJKTableCaptionText/reCJKFigureCaptionText mirror Python's+// `[图表]+[ 0-9::]{2,}`. The 图/表 run must be followed by ≥2 chars from+// [ 0-9::], so a body paragraph that merely STARTS with 表/图 (表格是一种…,+// 表現方法…, 图中所示…) is not a caption. The bare `^表` previously classified+// any 表-prefixed CJK/Japanese body paragraph as a caption that MergeCaptions+// either swallowed into <caption> or dropped — content loss Python never has+// (go_bug cjk-caption-false-positive). Kind is split by the run's first+// character because findNearestParent needs a table/figure kind to pick which+// parent to search; Python has a single pattern and picks the nearest parent+// of either kind instead.+var (+ reCJKTableCaptionText = regexp.MustCompile(`^表[图表]*[ 0-9::]{2,}`)+ reCJKFigureCaptionText = regexp.MustCompile(`^图[图表]*[ 0-9::]{2,}`)+)++// reEnglishTableCaptionPrefix/reEnglishFigureCaptionPrefix are start-anchored+// like Python's re.match, so a paragraph that merely MENTIONS "Table N"+// mid-sentence is not a caption (go_bug+// table-text-interleaved-paragraph-dropped). Start-anchoring alone still+// misclassifies a body sentence that BEGINS with a cross-reference+// ("Table 1, we present …", "Table 9 reports …"): MergeCaptions then moves the+// sentence into the table's <caption> and deletes it as a standalone section.+// The English table marker therefore also requires a caption boundary. The+// optional letter suffix keeps numbered captions ("Table 1a. Results") that the+// pattern-only detection recognized.+var (+ reEnglishTableCaptionPrefix = regexp.MustCompile(`(?i)^Table\s+\d+[A-Za-z]?(?:\.\d+)*`)+ reEnglishFigureCaptionPrefix = regexp.MustCompile(`(?i)^(?:Fig\.?\s*|Figure\s+)\d+`)+)++// reCaptionContinuation matches the parenthesised continuations that extend a+// caption rather than cite it.+var reCaptionContinuation = regexp.MustCompile(`(?i)^\(\s*(?:continued(?:\s+on\s+next\s+page)?|cont(?:['’]?d|\.)?|续表|续)\s*\)`)++// rePanelCaption matches a panel label that introduces the panel's title+// ("Table 2 (a) Confusion matrix", "Table 1 (iv) Confusion matrix",+// "Table 1 (12) Results"). Panels are single letters, Roman numerals, short+// numbers, or a letter+number combination.+var rePanelCaption = regexp.MustCompile(`^\(\s*(?:[A-Za-z]|[ivxlcdmIVXLCDM]{2,4}|\d{1,3}|[A-Za-z]\d{1,2})\s*\)`)++// captionBoundary reports whether the text following an English "Table N"+// marker starts a caption rather than continuing the sentence that cited it.+// A caption continues with a separator (": Revenue", ". Data statistics"), a+// title-like word ("Table 2 Performance comparison"), a parenthesised+// continuation or panel label, or nothing; a citation continues with a comma,+// a possessive/quote, or a lowercase word ("Table 1, we present", "Table 9+// reports"). Rejecting the latter is a deliberate divergence from Python's+// pattern-only is_caption: Python swallows the sentence, this path keeps it as+// body text.+func captionBoundary(rest string) bool {+ rest = strings.TrimLeftFunc(rest, unicode.IsSpace)+ if rest == "" {+ return true+ }+ r, _ := utf8.DecodeRuneInString(rest)+ switch r {+ case ':', ':', '.', '.', '。', '|', '-', '–', '—', '“', '‘':+ return true+ case '(':+ return reCaptionContinuation.MatchString(rest) || rePanelCaption.MatchString(rest)+ }+ if unicode.IsUpper(r) || unicode.IsDigit(r) {+ return true+ }+ return unicode.Is(unicode.Han, r) || unicode.Is(unicode.Hiragana, r) ||+ unicode.Is(unicode.Katakana, r) || unicode.Is(unicode.Hangul, r)+}++// captionMarkerPrefix reports whether text starts with a caption marker. It is+// the pattern-only half of caption detection, shared by IsCaptionBox.+func captionMarkerPrefix(t string) bool {+ return reCJKTableCaptionText.MatchString(t) || reCJKFigureCaptionText.MatchString(t) ||+ reEnglishTableCaptionPrefix.MatchString(t) || reEnglishFigureCaptionPrefix.MatchString(t)+}++// tableCaptionText classifies a standalone line as a table caption. The English+// branch requires captionBoundary; the CJK branch is unchanged.+func tableCaptionText(t string) bool {+ if reCJKTableCaptionText.MatchString(t) {+ return true+ }+ loc := reEnglishTableCaptionPrefix.FindStringIndex(t)+ return loc != nil && captionBoundary(t[loc[1]:])+}++// figureCaptionText classifies a standalone line as a figure caption. Figure+// captions keep the historical pattern-only rule: sentence-style captions+// ("Figure 1 shows the architecture") are common and this issue does not cover+// them.+func figureCaptionText(t string) bool {+ return reCJKFigureCaptionText.MatchString(t) || reEnglishFigureCaptionPrefix.MatchString(t)+}++// tableCaptionProse reports whether text starts with an English "Table N"+// marker whose continuation makes it a sentence that CITES the table rather+// than a caption ("Table 1, we present …").+func tableCaptionProse(t string) bool {+ loc := reEnglishTableCaptionPrefix.FindStringIndex(t)+ return loc != nil && !captionBoundary(t[loc[1]:])+}++// CaptionKind returns "table" if the section is a table caption, "figure" if a+// figure caption, or "" if not a caption. Layout-type hints keep precedence;+// text patterns decide unlabeled boxes.func CaptionKind(s pdf.Section) string {lt := s.LayoutType- if lt == pdf.DLALabelTableCaption || (strings.Contains(lt, "caption") && reTableCaptionText.MatchString(strings.TrimSpace(s.Text))) {- return pdf.LayoutTypeTable- }- if lt == pdf.DLALabelFigureCaption || strings.Contains(lt, "caption") {+ t := strings.TrimSpace(s.Text)+ // A citing sentence stays body text even when the layout labelled it a+ // caption: MergeCaptions would otherwise inject the prose into the nearest+ // table's <caption> (#20672) or delete it when no table is near. The label+ // still decides for real captions, including ones without a marker.+ if tableCaptionProse(t) {+ return ""+ }+ // A layout label decides the kind even when the text does not verify, so+ // MergeCaptions still searches the labeled parent kind first. A+ // non-canonical label such as "table_caption" keeps the kind its name+ // declares instead of falling through to the figure default.+ if strings.Contains(lt, "caption") {+ if lt == pdf.DLALabelFigureCaption || strings.Contains(lt, "figure") {+ return pdf.LayoutTypeFigure+ }+ if lt == pdf.DLALabelTableCaption || strings.Contains(lt, "table") || tableCaptionText(t) {+ return pdf.LayoutTypeTable+ }return pdf.LayoutTypeFigure}// DLA may label captions as "text" or other types — check text patterns.- t := strings.TrimSpace(s.Text)- if reTableCaptionText.MatchString(t) {+ if tableCaptionText(t) {return pdf.LayoutTypeTable}- if reFigureCaptionText.MatchString(t) {+ if figureCaptionText(t) {return pdf.LayoutTypeFigure}return ""diff --git a/internal/deepdoc/parser/pdf/table/testdata/parity/known_diffs.json b/internal/deepdoc/parser/pdf/table/testdata/parity/known_diffs.jsonindex bf2d3ba525..b3b706b046 100644--- a/internal/deepdoc/parser/pdf/table/testdata/parity/known_diffs.json+++ b/internal/deepdoc/parser/pdf/table/testdata/parity/known_diffs.json@@ -1,6 +1,21 @@{"version": 1,"rules": [+ {+ "id": "english-table-caption-cross-reference-kept",+ "tag": "go_intentional",+ "kind": "classification",+ "applies_to": [+ "*"+ ],+ "fields": [+ "CaptionKind",+ "IsCaptionBox",+ "MergeCaptions"+ ],+ "permanent": true,+ "reason": "Python's is_caption uses the start-anchored pattern (?i)^Table\\s+\\d+ alone, so a body sentence that BEGINS with a table cross-reference ('Table 1, we present ...', 'Table 9 reports ...') is classified as a caption and _extract_table_figure moves it into the nearest table's <caption>, deleting it as a standalone section. Go requires a caption boundary after the number (table_cells.go captionBoundary): punctuation separator, an uppercase/digit/CJK start, a parenthesised continuation or panel label, or end of line. A comma, possessive, or lowercase sentence continuation keeps the line as body text. Real captions without a colon ('Table 2 Performance comparison'), with a letter suffix ('Table 1a. Results'), or with a Roman/multi-digit panel label ('Table 1 (iv) ...', 'Table 1 (12) ...') still classify. Deliberate go_intentional divergence, locked by TestCaptionKind_EnglishTableCrossReferenceNotCaption, TestCaptionKind_EnglishTableCaptionBoundaries and TestMergeCaptions_EnglishTable1OpeningParagraphKept. Figure captions are unchanged (sentence-style figure captions remain captions). A layout-type hint still names the kind for captions without a marker ('Quarterly revenue breakdown'), but it cannot override the boundary rule for a line that starts with 'Table N' followed by prose; such a box stays body text (TestCaptionKind_LabeledCrossReferenceNotCaption, TestMergeCaptions_LabeledCrossReferenceStaysBodyText). IsCaptionBox keeps the pattern-only verdict because cell filling, no-merge marking, and caption-row stripping depend on it. KNOWN LIMITATION: the boundary keys on the first character, so a cross-reference that continues with a proper noun ('Table 3 Smith et al. report ...') classifies as a caption, indistinguishable from a caption title that starts with one ('Table 3 Smith-Kappa-Rosenberg parameters') without part-of-speech tagging; the sentence is then merged into the nearest table's <caption>, so its text survives in the output rather than being dropped. Pinned by TestCaptionKind_ProperNounContinuationIsCaption and TestMergeCaptions_ProperNounCrossReferenceLandsInCaption."+ },{"id": "table-cell-fill-filled-threshold-0.85","tag": "go_intentional",@@ -88,15 +103,15 @@],"fields": ["CaptionKind",- "reTableCaptionText",- "reCaption",+ "tableCaptionText",+ "IsCaptionBox","MergeCaptions"],"owner_fix_side": "go","status": "resolved",- "resolution": "reTableCaptionText and reCaption now start-anchor their English 'Table N' / 'Figure N' / 'Fig N' alternatives (^), mirroring Python's is_caption which uses re.match (start-anchored). A body paragraph that only MENTIONS 'Table 1' mid-sentence ('...Table 1 shows revenuebycategory.') is no longer misclassified as a caption, so MergeCaptions no longer drops it; 14 now emits title + paragraph + 2 tables (textSim 81.7% -> 90.0%, boxes ...->4). Regression: TestCaptionKind_MidSentenceTableMentionNotCaption, TestIsCaptionBox_MidSentenceTableMention, TestMergeCaptions_KeepsInterleavedParagraph, plus the end-to-end TestPipelineParity14InterleavedParagraphDropped (now GREEN).",+ "resolution": "tableCaptionText/IsCaptionBox now start-anchor their English 'Table N' / 'Figure N' / 'Fig N' alternatives (^), aligning with Python's re.match for mid-sentence mentions. A body paragraph that only MENTIONS 'Table 1' mid-sentence ('...Table 1 shows revenuebycategory.') is no longer misclassified as a caption, so MergeCaptions no longer drops it; 14 now emits title + paragraph + 2 tables (textSim 81.7% -> 90.0%, boxes ...->4). A later change also rejects line-initial cross-references ('Table 1, we present …') via the caption boundary; see rule english-table-caption-cross-reference-kept. Regression: TestCaptionKind_MidSentenceTableMentionNotCaption, TestIsCaptionBox_MidSentenceTableMention, TestMergeCaptions_KeepsInterleavedParagraph, plus the end-to-end TestPipelineParity14InterleavedParagraphDropped (now GREEN).","tracking": "Original fields guessed the loss in processPageBoxes/NaiveVerticalMerge/BoxesToSections. Step-through of buildLayout proved the paragraph box survives every stage (Dedup, AssignColumn, TextMerge, FinalReadingOrderMerge, NaiveVerticalMerge, ExtractTableAndReplace, ConsolidateFigures, BoxesToSections) and is only dropped in MergeCaptions: CaptionKind mislabeled it a table caption (unanchored 'Table N' match), findNearestParent returned -1 (nearest section is the title, not a table), so the caption-without-target branch removed it. Root cause is caption DETECTION, not assembly.",- "reason": "14_text_table_interleaved.pdf has a body paragraph ('The following analysis compares product performance across different market segments. Table 1 shows revenuebycategory.') interleaved BEFORE Table 1. Python extracts it as a separate text section; Go's pipeline dropped it entirely (Go output had only the title + the two tables). This was a real CONTENT loss, not an HTML-emission/format difference: the text was absent from Go's sections, and gridSim=100% still held because the paragraph is outside any table cell. RESOLVED: the drop was caused by unanchored 'Table N' caption detection (reTableCaptionText/reCaption matched 'Table 1' anywhere in the string), mislabeling the paragraph a caption that MergeCaptions then removed; anchoring to the start aligns Go with Python's re.match is_caption and retains the paragraph. 14's RESIDUAL gap is now caption emission only (rule table-html-emission-format). Originally exposed by TestPipelineParity14InterleavedParagraphDropped (was RED, now GREEN)."+ "reason": "14_text_table_interleaved.pdf has a body paragraph ('The following analysis compares product performance across different market segments. Table 1 shows revenuebycategory.') interleaved BEFORE Table 1. Python extracts it as a separate text section; Go's pipeline dropped it entirely (Go output had only the title + the two tables). This was a real CONTENT loss, not an HTML-emission/format difference: the text was absent from Go's sections, and gridSim=100% still held because the paragraph is outside any table cell. RESOLVED: the drop was caused by unanchored 'Table N' caption detection (the table-caption detection matched 'Table 1' anywhere in the string), mislabeling the paragraph a caption that MergeCaptions then removed; anchoring to the start retains the paragraph. Line-initial cross-references ('Table 1, we present …') are now also kept as body text via the caption boundary. 14's RESIDUAL gap is now caption emission only (rule table-html-emission-format). Originally exposed by TestPipelineParity14InterleavedParagraphDropped (was RED, now GREEN)."},{"id": "english-page-ocr-source-divergence",