Skip to content

RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.

fix(deepdoc): keep body sentences that cite Table N out of table captions (#21453)

CCLUCKYbaf314d58877
diff --git a/internal/deepdoc/parser/pdf/table/caption_kind_test.go b/internal/deepdoc/parser/pdf/table/caption_kind_test.go
index 54d1b72e7a..318b683370 100644
--- a/internal/deepdoc/parser/pdf/table/caption_kind_test.go
+++ b/internal/deepdoc/parser/pdf/table/caption_kind_test.go
@@ -12,7 +12,7 @@ import (
// MENTIONS "Table 1" mid-sentence must NOT be classified as a table caption.
// Python's is_caption uses re.match (start-anchored), so it keeps such a
// paragraph as a normal text section; Go must match that. Before the fix,
-// reTableCaptionText matched "Table N" anywhere in the string, so the
+// table-caption detection matched "Table N" anywhere in the string, so the
// paragraph was mislabeled a caption and silently dropped by MergeCaptions.
func TestCaptionKind_MidSentenceTableMentionNotCaption(t *testing.T) {
para := "The following analysis compares product performance across different market segments. Table 1 shows revenuebycategory."
@@ -36,8 +36,9 @@ func TestCaptionKind_StartsWithTableIsCaption(t *testing.T) {
}
}
-// TestIsCaptionBox_MidSentenceTableMention confirms IsCaptionBox (via reCaption)
-// also keeps mid-sentence "Table N" as non-caption, mirroring Python re.match.
+// TestIsCaptionBox_MidSentenceTableMention confirms IsCaptionBox (via the
+// shared caption detection) also keeps mid-sentence "Table N" as non-caption,
+// mirroring Python re.match.
func TestIsCaptionBox_MidSentenceTableMention(t *testing.T) {
para := "The following analysis compares product performance across different market segments. Table 1 shows revenuebycategory."
if IsCaptionBox(para, pdf.LayoutTypeText) {
@@ -113,6 +114,131 @@ func TestCaptionKind_CJKCaptionMarkersStillMatch(t *testing.T) {
}
}
+// TestCaptionKind_EnglishTableCrossReferenceNotCaption locks the fix for
+// #20672: a body sentence that STARTS with a table cross-reference is not a
+// caption. Python's pattern-only is_caption classifies these as captions and
+// swallows them into the table; this Go path requires a caption boundary after
+// the number, so the sentence stays a standalone text section.
+//
+// IsCaptionBox deliberately keeps the pattern-only verdict: it decides cell
+// filling, no-merge marking, and caption-row stripping, all of which must keep
+// matching Python's is_caption.
+func TestCaptionKind_EnglishTableCrossReferenceNotCaption(t *testing.T) {
+ for _, text := range []string{
+ "Table 1, we present the F1 scores and success rates for KNOWAGENT.",
+ "Table 9 reports the adaptive expansion ablation, as shown in Figure 5.",
+ "Table 1 shows revenue by category for the fiscal year.",
+ "Table 2 (see Appendix A) reports the remaining configurations.",
+ "Table 1's results show a 12% improvement over the baseline.",
+ "Table 1.5 shows the ablation without retrieval.",
+ } {
+ if got := CaptionKind(pdf.Section{Text: text, LayoutType: pdf.LayoutTypeText}); got != "" {
+ t.Errorf("CaptionKind(%q) = %q, want \"\" (cross-reference is body text)", text, got)
+ }
+ if !IsCaptionBox(text, pdf.LayoutTypeText) {
+ t.Errorf("IsCaptionBox(%q) = false, want true (pattern-only contract)", text)
+ }
+ }
+}
+
+// TestCaptionKind_EnglishTableCaptionBoundaries pins the positive side of the
+// boundary rule: separators, title-like words, end of line, letter-suffixed
+// numbers, and parenthesised continuations/panel labels still classify as table
+// captions.
+func TestCaptionKind_EnglishTableCaptionBoundaries(t *testing.T) {
+ for _, text := range []string{
+ "Table 2: Comparative Experiment of ReAct",
+ "Table 1. Data statistics",
+ "Table 1",
+ "Table 2 Performance comparison",
+ "Table 2 (continued)",
+ "Table 2 (cont'd)",
+ "Table 2 (a) Confusion matrix",
+ "Table 1a. Results",
+ "Table 1b: Results",
+ "Table 1 (iv) Confusion matrix",
+ "Table 1 (12) Results",
+ "Table 1.5: Ablation results",
+ "Table 1\nRevenue by category",
+ "Table 1\u00a0Revenue by category",
+ "Table 1\u3000Revenue by category",
+ } {
+ if got := CaptionKind(pdf.Section{Text: text, LayoutType: pdf.LayoutTypeText}); got != pdf.LayoutTypeTable {
+ t.Errorf("CaptionKind(%q) = %q, want %q", text, got, pdf.LayoutTypeTable)
+ }
+ if !IsCaptionBox(text, pdf.LayoutTypeText) {
+ t.Errorf("IsCaptionBox(%q) = false, want true", text)
+ }
+ }
+}
+
+// TestCaptionKind_LabelKeepsDeclaredKind pins label precedence: a layout label
+// names the parent kind even when its text carries no marker (a DLA caption
+// such as "Quarterly revenue breakdown"), including non-canonical labels such as
+// "table_caption".
+func TestCaptionKind_LabelKeepsDeclaredKind(t *testing.T) {
+ for _, tt := range []struct{ text, label, want string }{
+ {"Quarterly revenue breakdown", pdf.DLALabelTableCaption, pdf.LayoutTypeTable},
+ {"Quarterly revenue breakdown", "table_caption", pdf.LayoutTypeTable},
+ {"Table 9: Fine-grained retrieval recall.", pdf.DLALabelTableCaption, pdf.LayoutTypeTable},
+ {"Figure 1 shows the architecture.", "figure_caption", pdf.LayoutTypeFigure},
+ {"Figure 1 shows the architecture.", pdf.DLALabelFigureCaption, pdf.LayoutTypeFigure},
+ } {
+ if got := CaptionKind(pdf.Section{Text: tt.text, LayoutType: tt.label}); got != tt.want {
+ t.Errorf("CaptionKind(%q, label %q) = %q, want %q", tt.text, tt.label, got, tt.want)
+ }
+ }
+}
+
+// TestCaptionKind_LabeledCrossReferenceNotCaption covers the DLA half of
+// #20672: when the layout labels a CROSS-REFERENCE sentence as a caption, the
+// label must not override the boundary rule, or MergeCaptions injects the prose
+// into the table's <caption> (or deletes it when no table is near).
+func TestCaptionKind_LabeledCrossReferenceNotCaption(t *testing.T) {
+ const prose = "Table 1, we present the F1 scores and success rates."
+ for _, label := range []string{
+ pdf.DLALabelTableCaption,
+ "table_caption",
+ pdf.DLALabelFigureCaption,
+ "figure_caption",
+ "caption",
+ } {
+ if got := CaptionKind(pdf.Section{Text: prose, LayoutType: label}); got != "" {
+ t.Errorf("CaptionKind(labeled %q, %q) = %q, want \"\" (prose is not a caption)", prose, label, got)
+ }
+ }
+}
+
+// TestCaptionKind_FigureBoundaryUnchanged documents the scope of #20672:
+// sentence-style FIGURE captions ("Figure 1 shows …") remain captions, because
+// that form is a common legitimate caption style and the issue covers tables.
+func TestCaptionKind_FigureBoundaryUnchanged(t *testing.T) {
+ text := "Figure 1 shows the overall architecture of the system."
+ if got := CaptionKind(pdf.Section{Text: text, LayoutType: pdf.LayoutTypeText}); got != pdf.LayoutTypeFigure {
+ t.Errorf("CaptionKind(%q) = %q, want %q", text, got, pdf.LayoutTypeFigure)
+ }
+}
+
+// TestCaptionKind_ProperNounContinuationIsCaption pins the KNOWN LIMITATION of
+// the boundary rule: it keys on the first character after "Table N", so a
+// cross-reference that continues with a proper noun classifies as a caption
+// ("Table 3 Smith et al. report …"), exactly as a real caption title would
+// ("Table 3 Smith-Kappa-Rosenberg parameters"). Nothing in the string separates
+// them without part-of-speech tagging, so the rule keeps the caption reading.
+// The consequence is placement, not deletion: MergeCaptions moves the sentence
+// into the nearest table's <caption>, so it stays indexed
+// (TestMergeCaptions_ProperNounCrossReferenceLandsInCaption).
+func TestCaptionKind_ProperNounContinuationIsCaption(t *testing.T) {
+ for _, text := range []string{
+ "Table 3 Smith et al. report the F1 scores.",
+ "Table 3 Gaussian noise dominates above 40 dB.",
+ } {
+ if got := CaptionKind(pdf.Section{Text: text, LayoutType: pdf.LayoutTypeText}); got != pdf.LayoutTypeTable {
+ t.Errorf("CaptionKind(%q) = %q, want %q (documented limitation)", text, got, pdf.LayoutTypeTable)
+ }
+ }
+}
+
// hasBox reports whether positions contains a box with the given top and
// bottom edges. Tests use it to assert that a merged caption's box reached the
// section it was merged into.
diff --git a/internal/deepdoc/parser/pdf/table/merge_captions_caption_test.go b/internal/deepdoc/parser/pdf/table/merge_captions_caption_test.go
index 77da354921..12a9c51189 100644
--- a/internal/deepdoc/parser/pdf/table/merge_captions_caption_test.go
+++ b/internal/deepdoc/parser/pdf/table/merge_captions_caption_test.go
@@ -407,12 +407,12 @@ func TestMergeCaptions_FigureCaptionKeptWithMarkerFigure(t *testing.T) {
// table OR figure (nearest(tables)/nearest(figures), `elif fk`) and drops it
// only when BOTH searches fail. Go used to search tables only and delete the
// caption when none was reachable — losing text Python keeps, for English
-// body text AND for correctly-DLA-labeled "table caption" sections alike
+// text captions AND for correctly-DLA-labeled "table caption" sections alike
// (language-independent; the figure here is deliberately neutral-text so the
// figure-self-deletion bug cannot mask this path).
func TestMergeCaptions_TableCaptionFallsBackToFigure(t *testing.T) {
for _, caption := range []pdf.Section{
- {Text: "Table 1 shows revenue by category.", LayoutType: pdf.LayoutTypeText,
+ {Text: "Table 1: revenue by category.", LayoutType: pdf.LayoutTypeText,
Positions: []pdf.Position{{PageNumbers: []int{0}, Left: 100, Right: 500, Top: 510, Bottom: 530}}},
{Text: "Quarterly revenue breakdown", LayoutType: pdf.DLALabelTableCaption,
Positions: []pdf.Position{{PageNumbers: []int{0}, Left: 100, Right: 500, Top: 510, Bottom: 530}}},
@@ -436,27 +436,148 @@ func TestMergeCaptions_TableCaptionFallsBackToFigure(t *testing.T) {
}
}
-// TestMergeCaptions_EnglishTable1OpeningParagraphSwallowed pins the SHARED
-// (Python-identical) heuristic boundary of the CJK fix: a body paragraph that
-// STARTS with "Table N" is classified a caption by BOTH implementations
-// (Python's re.match is start-anchored too) and consumed into the table's
-// <caption>. The text is retained — misplacement parity with Python, not a
-// Go regression; only the CJK false-positive class was fixed.
-func TestMergeCaptions_EnglishTable1OpeningParagraphSwallowed(t *testing.T) {
+// TestMergeCaptions_EnglishTableCrossReferenceStaysWithRealCaption models the
+// RAG-paper sample from #20672: a real "Table 9: …" caption and a body sentence
+// "Table 9 reports …" both sit near the table. Only the real caption may join
+// the <caption>; the body sentence stays its own section, and the table's
+// highlight geometry covers the caption only.
+func TestMergeCaptions_EnglishTableCrossReferenceStaysWithRealCaption(t *testing.T) {
+ const (
+ caption = "Table 9: Fine-grained retrieval recall for the adaptive expansion ablation."
+ body = "Table 9 reports the adaptive expansion ablation, as shown in Figure 5."
+ )
sections := []pdf.Section{
{Text: "<table><tr><td>data</td></tr></table>", LayoutType: pdf.LayoutTypeTable,
Positions: []pdf.Position{{PageNumbers: []int{0}, Left: 100, Right: 500, Top: 200, Bottom: 400}}},
- {Text: "Table 1 shows revenue by category for the fiscal year.", LayoutType: pdf.LayoutTypeText,
+ {Text: caption, LayoutType: pdf.LayoutTypeText,
+ Positions: []pdf.Position{{PageNumbers: []int{0}, Left: 100, Right: 500, Top: 170, Bottom: 190}}},
+ {Text: body, LayoutType: pdf.LayoutTypeText,
+ Positions: []pdf.Position{{PageNumbers: []int{0}, Left: 100, Right: 500, Top: 410, Bottom: 430}}},
+ }
+ result := MergeCaptions(sections)
+ if len(result) != 2 {
+ t.Fatalf("expected table + body sentence, got %d sections: %v", len(result), textsOf(result))
+ }
+ var table *pdf.Section
+ foundBody := false
+ for i := range result {
+ if result[i].LayoutType == pdf.LayoutTypeTable {
+ table = &result[i]
+ }
+ if result[i].LayoutType == pdf.LayoutTypeText && result[i].Text == body {
+ foundBody = true
+ }
+ }
+ if table == nil || !strings.Contains(table.Text, "<caption>"+caption+"</caption>") {
+ t.Fatalf("real caption not attached to the table: %v", textsOf(result))
+ }
+ if !foundBody {
+ t.Errorf("cross-reference body sentence dropped or rewritten: %v", textsOf(result))
+ }
+ if len(table.Positions) != 2 || !hasBox(table.Positions, 170, 190) {
+ t.Errorf("table geometry must cover the caption box only, got %+v", table.Positions)
+ }
+ if hasBox(table.Positions, 410, 430) {
+ t.Errorf("body sentence box leaked into the table's highlight geometry: %+v", table.Positions)
+ }
+}
+
+// TestMergeCaptions_EnglishTable1OpeningParagraphKept locks the #20672
+// divergence from Python. Start-anchored pattern matching alone classified
+// "Table 1 shows revenue …" as a caption, and MergeCaptions then moved the body
+// sentence into the table's <caption> and deleted it as a standalone section.
+// The sentence now stays body text and the table receives no caption.
+func TestMergeCaptions_EnglishTable1OpeningParagraphKept(t *testing.T) {
+ const body = "Table 1 shows revenue by category for the fiscal year."
+ sections := []pdf.Section{
+ {Text: "<table><tr><td>data</td></tr></table>", LayoutType: pdf.LayoutTypeTable,
+ Positions: []pdf.Position{{PageNumbers: []int{0}, Left: 100, Right: 500, Top: 200, Bottom: 400}}},
+ {Text: body, LayoutType: pdf.LayoutTypeText,
Positions: []pdf.Position{{PageNumbers: []int{0}, Left: 100, Right: 500, Top: 170, Bottom: 195}}},
}
result := MergeCaptions(sections)
+ if len(result) != 2 {
+ t.Fatalf("expected the body sentence and the table as separate sections, got %d: %v", len(result), textsOf(result))
+ }
+ found := false
for _, s := range result {
- if s.LayoutType == pdf.LayoutTypeTable && strings.Contains(s.Text, "<caption>Table 1 shows revenue") {
- return
+ if s.LayoutType == pdf.LayoutTypeTable && strings.Contains(s.Text, "<caption>") {
+ t.Errorf("body sentence consumed into <caption>: %v", textsOf(result))
+ }
+ if s.LayoutType == pdf.LayoutTypeText && s.Text == body {
+ found = true
+ }
+ }
+ if !found {
+ t.Errorf("body sentence dropped or rewritten; sections = %v", textsOf(result))
+ }
+}
+
+// TestMergeCaptions_LabeledCrossReferenceStaysBodyText covers the DLA half of
+// #20672: a caption LABEL must not override the boundary rule, or MergeCaptions
+// injects the prose into the table's <caption> (sibling table present) or
+// deletes it outright (no table near).
+func TestMergeCaptions_LabeledCrossReferenceStaysBodyText(t *testing.T) {
+ const prose = "Table 1, we present the F1 scores and success rates for KNOWAGENT."
+ for _, label := range []string{pdf.DLALabelTableCaption, "table_caption"} {
+ withTable := []pdf.Section{
+ {Text: "<table><tr><td>data</td></tr></table>", LayoutType: pdf.LayoutTypeTable,
+ Positions: []pdf.Position{{PageNumbers: []int{0}, Left: 100, Right: 500, Top: 200, Bottom: 400}}},
+ {Text: prose, LayoutType: label,
+ Positions: []pdf.Position{{PageNumbers: []int{0}, Left: 100, Right: 500, Top: 410, Bottom: 430}}},
+ }
+ result := MergeCaptions(withTable)
+ if len(result) != 2 {
+ t.Fatalf("label %q: expected table + prose, got %d sections: %v", label, len(result), textsOf(result))
+ }
+ for _, s := range result {
+ if strings.Contains(s.Text, "<caption>") {
+ t.Errorf("label %q: prose injected into <caption>: %v", label, textsOf(result))
+ }
+ }
+
+ noTable := []pdf.Section{
+ {Text: prose, LayoutType: label,
+ Positions: []pdf.Position{{PageNumbers: []int{0}, Left: 100, Right: 500, Top: 410, Bottom: 430}}},
+ }
+ result = MergeCaptions(noTable)
+ if len(result) != 1 || result[0].Text != prose {
+ t.Errorf("label %q: prose deleted when no table is near: %v", label, textsOf(result))
}
}
- t.Errorf("expected the start-anchored English paragraph consumed into <caption> (Python parity); sections = %v",
- textsOf(result))
+}
+
+// TestMergeCaptions_ProperNounCrossReferenceLandsInCaption measures the cost of
+// the boundary rule's known limitation (see
+// TestCaptionKind_ProperNounContinuationIsCaption): the sentence is misread as a
+// caption and merged into the table's <caption> rather than dropped, so its
+// text stays in the output and indexed with the table. Pinning this keeps the
+// limitation honest: if a later change makes the no-target branch delete
+// captions again, this test turns RED and the trade-off has to be re-argued.
+func TestMergeCaptions_ProperNounCrossReferenceLandsInCaption(t *testing.T) {
+ const sentence = "Table 3 Smith et al. report the F1 scores."
+ sections := []pdf.Section{
+ {Text: "<table><tr><td>data</td></tr></table>", LayoutType: pdf.LayoutTypeTable,
+ Positions: []pdf.Position{{PageNumbers: []int{0}, Left: 100, Right: 500, Top: 200, Bottom: 400}}},
+ {Text: sentence, LayoutType: pdf.LayoutTypeText,
+ Positions: []pdf.Position{{PageNumbers: []int{0}, Left: 100, Right: 500, Top: 170, Bottom: 195}}},
+ }
+ result := MergeCaptions(sections)
+ inCaption := false
+ for _, s := range result {
+ if s.LayoutType == pdf.LayoutTypeTable {
+ start := strings.Index(s.Text, "<caption>")
+ end := strings.Index(s.Text, "</caption>")
+ if start >= 0 && end > start && strings.Contains(s.Text[start+len("<caption>"):end], sentence) {
+ inCaption = true
+ }
+ } else if strings.Contains(s.Text, sentence) {
+ t.Errorf("misclassified sentence survived as a standalone section: %v", textsOf(result))
+ }
+ }
+ if !inCaption {
+ t.Errorf("misclassified sentence not placed in the table's <caption>; sections = %v", textsOf(result))
+ }
}
// TestMergeCaptions_FigureCaptionOtherPageDoesNotAttach locks the figure-side
diff --git a/internal/deepdoc/parser/pdf/table/table_annotate_test.go b/internal/deepdoc/parser/pdf/table/table_annotate_test.go
index dbc01b983f..a025847827 100644
--- a/internal/deepdoc/parser/pdf/table/table_annotate_test.go
+++ b/internal/deepdoc/parser/pdf/table/table_annotate_test.go
@@ -735,3 +735,22 @@ func TestMarkNoMergeTables_EmptyInputs(t *testing.T) {
MarkNoMergeTables(nil, nil)
MarkNoMergeTables([]pdf.TextBox{}, []pdf.TableItem{})
}
+
+// TestGarbageKeepFeat_ReferenceKeepsCrossReference pins text retention: a
+// reference-labelled box is dropped unless it looks like a caption. IsCaptionBox
+// stays pattern-only, so a distinguishing noun phrase that starts with "Table N"
+// ("Table 1, we present …") is still kept as text, not dropped as a reference.
+func TestGarbageKeepFeat_ReferenceKeepsCrossReference(t *testing.T) {
+ for _, text := range []string{
+ "Table 1, we present the F1 scores and success rates.",
+ "Table 9 reports the adaptive expansion ablation.",
+ "Table 2: Comparative Experiment of ReAct",
+ } {
+ if !garbageKeepFeat(pdf.LayoutTypeReference, pdf.TextBox{Text: text}, 1000) {
+ t.Errorf("garbageKeepFeat(reference, %q) = false, want true (text would be dropped)", text)
+ }
+ }
+ if garbageKeepFeat(pdf.LayoutTypeReference, pdf.TextBox{Text: "plain reference text"}, 1000) {
+ t.Error("garbageKeepFeat kept a plain reference box")
+ }
+}
diff --git a/internal/deepdoc/parser/pdf/table/table_cells.go b/internal/deepdoc/parser/pdf/table/table_cells.go
index 9b85b18d4d..d2ba9e4cfc 100644
--- a/internal/deepdoc/parser/pdf/table/table_cells.go
+++ b/internal/deepdoc/parser/pdf/table/table_cells.go
@@ -5,6 +5,8 @@ import (
"regexp"
"sort"
"strings"
+ "unicode"
+ "unicode/utf8"
"go.uber.org/zap"
@@ -414,61 +416,151 @@ func matchRowStrip(rowStrips []pdf.TSRCell, y0 float64) (float64, float64, bool)
return 0, 0, false
}
-// isCaptionBox checks if a text box is a table/figure caption,
-// matching Python is_caption(). Captions should not enter table cells.
-// reCaption backs IsCaptionBox and must mirror Python's start-anchored
-// is_caption: only a line BEGINNING with a caption marker counts. The English
-// alternatives are start-anchored with ^ for the same reason as
-// reTableCaptionText/reFigureCaptionText.
-var reCaption = regexp.MustCompile(`^[图表]+[ 0-9::]{2,}|(?i)^Fig\.?\s*\d+|(?i)^Figure\s+\d+|(?i)^Table\s+\d+`)
-
+// IsCaptionBox checks if a text box is a table/figure caption. Captions should
+// not enter table cells. Start-anchored detection is shared with Python's
+// is_caption. IsCaptionBox stays pattern-only on purpose: its other callers fill
+// table cells, mark cross-page no-merge tables, and strip caption rows, and none
+// of them may start treating a citing sentence as data. The boundary rule that
+// keeps cross-references out of <caption> belongs to CaptionKind.
func IsCaptionBox(text string, layoutType string) bool {
if strings.Contains(layoutType, "caption") {
return true
}
- return reCaption.MatchString(strings.TrimSpace(text))
+ return captionMarkerPrefix(strings.TrimSpace(text))
}
-// reTableCaptionText matches text patterns that indicate a table caption
-// (as opposed to a figure caption). Python is_caption uses re.match, which is
-// start-anchored, so a body paragraph that merely MENTIONS "Table N"
-// mid-sentence is NOT a caption. The English alternatives below are therefore
-// start-anchored with ^: an unanchored alternative wrongly classifies such a
-// paragraph as a caption and MergeCaptions then drops it (go_bug
-// table-text-interleaved-paragraph-dropped).
-//
-// The CJK branch mirrors Python's `[图表]+[ 0-9::]{2,}`: the 图/表 run must be
-// followed by ≥2 chars from [ 0-9::], so a body paragraph that merely STARTS
-// with 表/图 (表格是一种…, 表現方法…, 图中所示…) is not a caption. The bare
-// `^表` previously classified any 表-prefixed CJK/Japanese body paragraph as a
-// caption that MergeCaptions either swallowed into <caption> or dropped —
-// content loss Python never has (go_bug cjk-caption-false-positive). Kind is
-// split by the run's first character because findNearestParent needs a
-// table/figure kind to pick which parent to search; Python has a single
-// pattern and picks the nearest parent of either kind instead.
-var reTableCaptionText = regexp.MustCompile(`^表[图表]*[ 0-9::]{2,}|(?i)^Table\s+\d+`)
-
-// reFigureCaptionText matches text patterns that indicate a figure caption —
-// the figure-kind split of the same Python pattern (see reTableCaptionText).
-var reFigureCaptionText = regexp.MustCompile(`^图[图表]*[ 0-9::]{2,}|(?i)^Fig\.?\s*\d+|(?i)^Figure\s+\d+`)
-
-// captionKind returns "table" if the section is a table caption,
-// "figure" if a figure caption, or "" if not a caption.
-// Matches Python's is_caption check: text patterns OR layout_type containing "caption".
+// reCJKTableCaptionText/reCJKFigureCaptionText mirror Python's
+// `[图表]+[ 0-9::]{2,}`. The 图/表 run must be followed by ≥2 chars from
+// [ 0-9::], so a body paragraph that merely STARTS with 表/图 (表格是一种…,
+// 表現方法…, 图中所示…) is not a caption. The bare `^表` previously classified
+// any 表-prefixed CJK/Japanese body paragraph as a caption that MergeCaptions
+// either swallowed into <caption> or dropped — content loss Python never has
+// (go_bug cjk-caption-false-positive). Kind is split by the run's first
+// character because findNearestParent needs a table/figure kind to pick which
+// parent to search; Python has a single pattern and picks the nearest parent
+// of either kind instead.
+var (
+ reCJKTableCaptionText = regexp.MustCompile(`^表[图表]*[ 0-9::]{2,}`)
+ reCJKFigureCaptionText = regexp.MustCompile(`^图[图表]*[ 0-9::]{2,}`)
+)
+
+// reEnglishTableCaptionPrefix/reEnglishFigureCaptionPrefix are start-anchored
+// like Python's re.match, so a paragraph that merely MENTIONS "Table N"
+// mid-sentence is not a caption (go_bug
+// table-text-interleaved-paragraph-dropped). Start-anchoring alone still
+// misclassifies a body sentence that BEGINS with a cross-reference
+// ("Table 1, we present …", "Table 9 reports …"): MergeCaptions then moves the
+// sentence into the table's <caption> and deletes it as a standalone section.
+// The English table marker therefore also requires a caption boundary. The
+// optional letter suffix keeps numbered captions ("Table 1a. Results") that the
+// pattern-only detection recognized.
+var (
+ reEnglishTableCaptionPrefix = regexp.MustCompile(`(?i)^Table\s+\d+[A-Za-z]?(?:\.\d+)*`)
+ reEnglishFigureCaptionPrefix = regexp.MustCompile(`(?i)^(?:Fig\.?\s*|Figure\s+)\d+`)
+)
+
+// reCaptionContinuation matches the parenthesised continuations that extend a
+// caption rather than cite it.
+var reCaptionContinuation = regexp.MustCompile(`(?i)^\(\s*(?:continued(?:\s+on\s+next\s+page)?|cont(?:['’]?d|\.)?|续表|续)\s*\)`)
+
+// rePanelCaption matches a panel label that introduces the panel's title
+// ("Table 2 (a) Confusion matrix", "Table 1 (iv) Confusion matrix",
+// "Table 1 (12) Results"). Panels are single letters, Roman numerals, short
+// numbers, or a letter+number combination.
+var rePanelCaption = regexp.MustCompile(`^\(\s*(?:[A-Za-z]|[ivxlcdmIVXLCDM]{2,4}|\d{1,3}|[A-Za-z]\d{1,2})\s*\)`)
+
+// captionBoundary reports whether the text following an English "Table N"
+// marker starts a caption rather than continuing the sentence that cited it.
+// A caption continues with a separator (": Revenue", ". Data statistics"), a
+// title-like word ("Table 2 Performance comparison"), a parenthesised
+// continuation or panel label, or nothing; a citation continues with a comma,
+// a possessive/quote, or a lowercase word ("Table 1, we present", "Table 9
+// reports"). Rejecting the latter is a deliberate divergence from Python's
+// pattern-only is_caption: Python swallows the sentence, this path keeps it as
+// body text.
+func captionBoundary(rest string) bool {
+ rest = strings.TrimLeftFunc(rest, unicode.IsSpace)
+ if rest == "" {
+ return true
+ }
+ r, _ := utf8.DecodeRuneInString(rest)
+ switch r {
+ case ':', ':', '.', '.', '。', '|', '-', '–', '—', '“', '‘':
+ return true
+ case '(':
+ return reCaptionContinuation.MatchString(rest) || rePanelCaption.MatchString(rest)
+ }
+ if unicode.IsUpper(r) || unicode.IsDigit(r) {
+ return true
+ }
+ return unicode.Is(unicode.Han, r) || unicode.Is(unicode.Hiragana, r) ||
+ unicode.Is(unicode.Katakana, r) || unicode.Is(unicode.Hangul, r)
+}
+
+// captionMarkerPrefix reports whether text starts with a caption marker. It is
+// the pattern-only half of caption detection, shared by IsCaptionBox.
+func captionMarkerPrefix(t string) bool {
+ return reCJKTableCaptionText.MatchString(t) || reCJKFigureCaptionText.MatchString(t) ||
+ reEnglishTableCaptionPrefix.MatchString(t) || reEnglishFigureCaptionPrefix.MatchString(t)
+}
+
+// tableCaptionText classifies a standalone line as a table caption. The English
+// branch requires captionBoundary; the CJK branch is unchanged.
+func tableCaptionText(t string) bool {
+ if reCJKTableCaptionText.MatchString(t) {
+ return true
+ }
+ loc := reEnglishTableCaptionPrefix.FindStringIndex(t)
+ return loc != nil && captionBoundary(t[loc[1]:])
+}
+
+// figureCaptionText classifies a standalone line as a figure caption. Figure
+// captions keep the historical pattern-only rule: sentence-style captions
+// ("Figure 1 shows the architecture") are common and this issue does not cover
+// them.
+func figureCaptionText(t string) bool {
+ return reCJKFigureCaptionText.MatchString(t) || reEnglishFigureCaptionPrefix.MatchString(t)
+}
+
+// tableCaptionProse reports whether text starts with an English "Table N"
+// marker whose continuation makes it a sentence that CITES the table rather
+// than a caption ("Table 1, we present …").
+func tableCaptionProse(t string) bool {
+ loc := reEnglishTableCaptionPrefix.FindStringIndex(t)
+ return loc != nil && !captionBoundary(t[loc[1]:])
+}
+
+// CaptionKind returns "table" if the section is a table caption, "figure" if a
+// figure caption, or "" if not a caption. Layout-type hints keep precedence;
+// text patterns decide unlabeled boxes.
func CaptionKind(s pdf.Section) string {
lt := s.LayoutType
- if lt == pdf.DLALabelTableCaption || (strings.Contains(lt, "caption") && reTableCaptionText.MatchString(strings.TrimSpace(s.Text))) {
- return pdf.LayoutTypeTable
- }
- if lt == pdf.DLALabelFigureCaption || strings.Contains(lt, "caption") {
+ t := strings.TrimSpace(s.Text)
+ // A citing sentence stays body text even when the layout labelled it a
+ // caption: MergeCaptions would otherwise inject the prose into the nearest
+ // table's <caption> (#20672) or delete it when no table is near. The label
+ // still decides for real captions, including ones without a marker.
+ if tableCaptionProse(t) {
+ return ""
+ }
+ // A layout label decides the kind even when the text does not verify, so
+ // MergeCaptions still searches the labeled parent kind first. A
+ // non-canonical label such as "table_caption" keeps the kind its name
+ // declares instead of falling through to the figure default.
+ if strings.Contains(lt, "caption") {
+ if lt == pdf.DLALabelFigureCaption || strings.Contains(lt, "figure") {
+ return pdf.LayoutTypeFigure
+ }
+ if lt == pdf.DLALabelTableCaption || strings.Contains(lt, "table") || tableCaptionText(t) {
+ return pdf.LayoutTypeTable
+ }
return pdf.LayoutTypeFigure
}
// DLA may label captions as "text" or other types — check text patterns.
- t := strings.TrimSpace(s.Text)
- if reTableCaptionText.MatchString(t) {
+ if tableCaptionText(t) {
return pdf.LayoutTypeTable
}
- if reFigureCaptionText.MatchString(t) {
+ if figureCaptionText(t) {
return pdf.LayoutTypeFigure
}
return ""
diff --git a/internal/deepdoc/parser/pdf/table/testdata/parity/known_diffs.json b/internal/deepdoc/parser/pdf/table/testdata/parity/known_diffs.json
index bf2d3ba525..b3b706b046 100644
--- a/internal/deepdoc/parser/pdf/table/testdata/parity/known_diffs.json
+++ b/internal/deepdoc/parser/pdf/table/testdata/parity/known_diffs.json
@@ -1,6 +1,21 @@
{
"version": 1,
"rules": [
+ {
+ "id": "english-table-caption-cross-reference-kept",
+ "tag": "go_intentional",
+ "kind": "classification",
+ "applies_to": [
+ "*"
+ ],
+ "fields": [
+ "CaptionKind",
+ "IsCaptionBox",
+ "MergeCaptions"
+ ],
+ "permanent": true,
+ "reason": "Python's is_caption uses the start-anchored pattern (?i)^Table\\s+\\d+ alone, so a body sentence that BEGINS with a table cross-reference ('Table 1, we present ...', 'Table 9 reports ...') is classified as a caption and _extract_table_figure moves it into the nearest table's <caption>, deleting it as a standalone section. Go requires a caption boundary after the number (table_cells.go captionBoundary): punctuation separator, an uppercase/digit/CJK start, a parenthesised continuation or panel label, or end of line. A comma, possessive, or lowercase sentence continuation keeps the line as body text. Real captions without a colon ('Table 2 Performance comparison'), with a letter suffix ('Table 1a. Results'), or with a Roman/multi-digit panel label ('Table 1 (iv) ...', 'Table 1 (12) ...') still classify. Deliberate go_intentional divergence, locked by TestCaptionKind_EnglishTableCrossReferenceNotCaption, TestCaptionKind_EnglishTableCaptionBoundaries and TestMergeCaptions_EnglishTable1OpeningParagraphKept. Figure captions are unchanged (sentence-style figure captions remain captions). A layout-type hint still names the kind for captions without a marker ('Quarterly revenue breakdown'), but it cannot override the boundary rule for a line that starts with 'Table N' followed by prose; such a box stays body text (TestCaptionKind_LabeledCrossReferenceNotCaption, TestMergeCaptions_LabeledCrossReferenceStaysBodyText). IsCaptionBox keeps the pattern-only verdict because cell filling, no-merge marking, and caption-row stripping depend on it. KNOWN LIMITATION: the boundary keys on the first character, so a cross-reference that continues with a proper noun ('Table 3 Smith et al. report ...') classifies as a caption, indistinguishable from a caption title that starts with one ('Table 3 Smith-Kappa-Rosenberg parameters') without part-of-speech tagging; the sentence is then merged into the nearest table's <caption>, so its text survives in the output rather than being dropped. Pinned by TestCaptionKind_ProperNounContinuationIsCaption and TestMergeCaptions_ProperNounCrossReferenceLandsInCaption."
+ },
{
"id": "table-cell-fill-filled-threshold-0.85",
"tag": "go_intentional",
@@ -88,15 +103,15 @@
],
"fields": [
"CaptionKind",
- "reTableCaptionText",
- "reCaption",
+ "tableCaptionText",
+ "IsCaptionBox",
"MergeCaptions"
],
"owner_fix_side": "go",
"status": "resolved",
- "resolution": "reTableCaptionText and reCaption now start-anchor their English 'Table N' / 'Figure N' / 'Fig N' alternatives (^), mirroring Python's is_caption which uses re.match (start-anchored). A body paragraph that only MENTIONS 'Table 1' mid-sentence ('...Table 1 shows revenuebycategory.') is no longer misclassified as a caption, so MergeCaptions no longer drops it; 14 now emits title + paragraph + 2 tables (textSim 81.7% -> 90.0%, boxes ...->4). Regression: TestCaptionKind_MidSentenceTableMentionNotCaption, TestIsCaptionBox_MidSentenceTableMention, TestMergeCaptions_KeepsInterleavedParagraph, plus the end-to-end TestPipelineParity14InterleavedParagraphDropped (now GREEN).",
+ "resolution": "tableCaptionText/IsCaptionBox now start-anchor their English 'Table N' / 'Figure N' / 'Fig N' alternatives (^), aligning with Python's re.match for mid-sentence mentions. A body paragraph that only MENTIONS 'Table 1' mid-sentence ('...Table 1 shows revenuebycategory.') is no longer misclassified as a caption, so MergeCaptions no longer drops it; 14 now emits title + paragraph + 2 tables (textSim 81.7% -> 90.0%, boxes ...->4). A later change also rejects line-initial cross-references ('Table 1, we present …') via the caption boundary; see rule english-table-caption-cross-reference-kept. Regression: TestCaptionKind_MidSentenceTableMentionNotCaption, TestIsCaptionBox_MidSentenceTableMention, TestMergeCaptions_KeepsInterleavedParagraph, plus the end-to-end TestPipelineParity14InterleavedParagraphDropped (now GREEN).",
"tracking": "Original fields guessed the loss in processPageBoxes/NaiveVerticalMerge/BoxesToSections. Step-through of buildLayout proved the paragraph box survives every stage (Dedup, AssignColumn, TextMerge, FinalReadingOrderMerge, NaiveVerticalMerge, ExtractTableAndReplace, ConsolidateFigures, BoxesToSections) and is only dropped in MergeCaptions: CaptionKind mislabeled it a table caption (unanchored 'Table N' match), findNearestParent returned -1 (nearest section is the title, not a table), so the caption-without-target branch removed it. Root cause is caption DETECTION, not assembly.",
- "reason": "14_text_table_interleaved.pdf has a body paragraph ('The following analysis compares product performance across different market segments. Table 1 shows revenuebycategory.') interleaved BEFORE Table 1. Python extracts it as a separate text section; Go's pipeline dropped it entirely (Go output had only the title + the two tables). This was a real CONTENT loss, not an HTML-emission/format difference: the text was absent from Go's sections, and gridSim=100% still held because the paragraph is outside any table cell. RESOLVED: the drop was caused by unanchored 'Table N' caption detection (reTableCaptionText/reCaption matched 'Table 1' anywhere in the string), mislabeling the paragraph a caption that MergeCaptions then removed; anchoring to the start aligns Go with Python's re.match is_caption and retains the paragraph. 14's RESIDUAL gap is now caption emission only (rule table-html-emission-format). Originally exposed by TestPipelineParity14InterleavedParagraphDropped (was RED, now GREEN)."
+ "reason": "14_text_table_interleaved.pdf has a body paragraph ('The following analysis compares product performance across different market segments. Table 1 shows revenuebycategory.') interleaved BEFORE Table 1. Python extracts it as a separate text section; Go's pipeline dropped it entirely (Go output had only the title + the two tables). This was a real CONTENT loss, not an HTML-emission/format difference: the text was absent from Go's sections, and gridSim=100% still held because the paragraph is outside any table cell. RESOLVED: the drop was caused by unanchored 'Table N' caption detection (the table-caption detection matched 'Table 1' anywhere in the string), mislabeling the paragraph a caption that MergeCaptions then removed; anchoring to the start retains the paragraph. Line-initial cross-references ('Table 1, we present …') are now also kept as body text via the caption boundary. 14's RESIDUAL gap is now caption emission only (rule table-html-emission-format). Originally exposed by TestPipelineParity14InterleavedParagraphDropped (was RED, now GREEN)."
},
{
"id": "english-page-ocr-source-divergence",