Skip to content

RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.

refactor(parser): unify spreadsheet tables on a segmented HTML wire (#20358)

jay777219b67a4112a70
diff --git a/internal/engine/infinity/chunk.go b/internal/engine/infinity/chunk.go
index dae7635a7b..28b4a6f5b2 100644
--- a/internal/engine/infinity/chunk.go
+++ b/internal/engine/infinity/chunk.go
@@ -2983,7 +2983,7 @@ func numericRow(row interface{}, out *[]interface{}) bool {
// flat []interface{} the hex encoder takes. Two shapes reach the engine: values
// decoded from JSON arrive as []interface{} (numbers as float64), while a chunk
// built IN PROCESS carries Go's typed slices (the ingestion pipeline's
-// AddPositions emits []int / [][]int). Both must be encoded, because these
+// addPDFPositions emits []int / [][]int). Both must be encoded, because these
// columns are VARCHAR holding the hex form: handing Infinity the typed slice
// makes it try to store an int64 tensor and fail with
// "Not support to convert Tensor(int64,5) to Varchar" (InfinityException 3049).
@@ -3154,7 +3154,7 @@ func transformChunkFields(chunk map[string]interface{}, embeddingCols [][2]inter
case "position_int", "page_num_int", "top_int":
// Python flattens the position rows and hex-encodes every number
// (infinity_conn.py: `[num for row in v for num in row]`, "%08x").
- // The input may be Go-NATIVE: the ingestion pipeline's AddPositions
+ // The input may be Go-NATIVE: the ingestion pipeline's addPDFPositions
// emits []int / [][]int, and those used to miss the
// []interface{}-only branch and reach Infinity as a raw tensor
// ("Not support to convert Tensor(int64,5) to Varchar", 3049).
diff --git a/internal/engine/infinity/chunk_transform_test.go b/internal/engine/infinity/chunk_transform_test.go
index 69afb7f302..6543b34f49 100644
--- a/internal/engine/infinity/chunk_transform_test.go
+++ b/internal/engine/infinity/chunk_transform_test.go
@@ -93,7 +93,7 @@ func TestTransformChunkFields_IngestionShape(t *testing.T) {
}
// TestTransformChunkFieldsHexEncodesNativeIntSlices pins the Go-native slice
-// shapes the ingestion pipeline produces: AddPositions emits []int / [][]int
+// shapes the ingestion pipeline produces: addPDFPositions emits []int / [][]int
// (internal/ingestion/task/indexdoc/position.go), and the Infinity columns are
// VARCHAR holding the hex form. Left unconverted, Infinity rejects the insert
// with "Not support to convert Tensor(int64,5) to Varchar" (3049).
diff --git a/internal/ingestion/component/chunker/coverage_blind_spots_test.go b/internal/ingestion/component/chunker/coverage_blind_spots_test.go
index ed800d4f83..c1c9d4023f 100644
--- a/internal/ingestion/component/chunker/coverage_blind_spots_test.go
+++ b/internal/ingestion/component/chunker/coverage_blind_spots_test.go
@@ -160,14 +160,13 @@ func TestGeneralChunkerKeepsHeaderOnlySheetsSeparate(t *testing.T) {
if err != nil {
t.Fatalf("NewGeneralChunker: %v", err)
}
+ segOne := spreadsheetSegmentItem("Sheet1", []string{"Name"}, nil, 1, 1)
+ segTwo := spreadsheetSegmentItem("Sheet2", []string{"Amount"}, nil, 2, 1)
out, err := chunker.Invoke(t.Context(), nil, map[string]any{
"name": "empty-sheets.xlsx",
"file_type": "xlsx",
"output_format": "json",
- "json": []map[string]any{
- {"text": "Name", "doc_type_kwd": "table", "ck_type": "table_header", "sheet_index": 1, "table_id": "sheet-1"},
- {"text": "Amount", "doc_type_kwd": "table", "ck_type": "table_header", "sheet_index": 2, "table_id": "sheet-2"},
- },
+ "json": []map[string]any{segOne, segTwo},
})
if err != nil {
t.Fatalf("GeneralChunker.Invoke: %v", err)
@@ -176,7 +175,7 @@ func TestGeneralChunkerKeepsHeaderOnlySheetsSeparate(t *testing.T) {
if len(chunks) != 2 {
t.Fatalf("chunks = %#v, want one header-only chunk per sheet", chunks)
}
- if chunks[0]["text"] != "Name" || chunks[1]["text"] != "Amount" {
+ if chunks[0]["text"] != segOne["text"] || chunks[1]["text"] != segTwo["text"] {
t.Fatalf("header-only chunks = %#v", chunks)
}
}
diff --git a/internal/ingestion/component/chunker/general.go b/internal/ingestion/component/chunker/general.go
index ba9c6747ea..9700c159b2 100644
--- a/internal/ingestion/component/chunker/general.go
+++ b/internal/ingestion/component/chunker/general.go
@@ -463,11 +463,11 @@ func collectGeneralMediaContext(units []schema.ChunkDoc, index, budget int, abov
if generalMediaContextCrossesSpreadsheetBoundary(units[index], units[cursor]) {
break
}
- if itemDocType(units[cursor]) != "text" {
+ text, ok := generalContextSourceText(units[cursor])
+ if !ok {
continue
}
- text := units[cursor].Text
- tokens := generalUnitTokens(units[cursor])
+ tokens := tokenizeStr(text)
if tokens >= remaining {
piece := takeContextSentences(text, remaining, above)
if above {
@@ -487,6 +487,41 @@ func collectGeneralMediaContext(units []schema.ChunkDoc, index, budget int, abov
return strings.Join(parts, "\n")
}
+// generalContextSourceText returns the readable text a unit contributes as
+// media context. Text units contribute their body; spreadsheet table
+// segments contribute their data rows' cell text — the row IR's loose text
+// rows were the image context before the HTML wire unification, and the
+// repeating header stays out of the context exactly as it did then. Only
+// spreadsheet segments qualify: attachGeneralMediaContext is shared with the
+// PDF and DOCX paths, where a table item carries no identity and must keep
+// its previous behavior (no context contribution).
+func generalContextSourceText(doc schema.ChunkDoc) (string, bool) {
+ switch itemDocType(doc) {
+ case "text":
+ return doc.Text, true
+ case "table":
+ if !hasSpreadsheetIdentity(doc) || !isTableStrictHTML(doc.Text) {
+ return "", false
+ }
+ rows, headerCount := tableRowsWithHeader(doc.Text)
+ lines := make([]string, 0, len(rows)-headerCount)
+ for _, row := range rows[headerCount:] {
+ cells := make([]string, 0, len(row))
+ for _, cell := range row {
+ if cell != "" {
+ cells = append(cells, cell)
+ }
+ }
+ if len(cells) > 0 {
+ lines = append(lines, strings.Join(cells, " "))
+ }
+ }
+ return strings.Join(lines, "\n"), true
+ default:
+ return "", false
+ }
+}
+
func generalMediaContextCrossesSpreadsheetBoundary(media, candidate schema.ChunkDoc) bool {
if !hasSpreadsheetIdentity(media) && !hasSpreadsheetIdentity(candidate) {
return false
@@ -495,7 +530,7 @@ func generalMediaContextCrossesSpreadsheetBoundary(media, candidate schema.Chunk
}
func hasSpreadsheetIdentity(doc schema.ChunkDoc) bool {
- return doc.TableID != "" || doc.Sheet != "" || doc.SheetIndex != nil
+ return doc.Sheet != "" || doc.SheetIndex != nil
}
// takeContextSentences returns the whole sentences at one end of text whose
@@ -815,44 +850,13 @@ func (c *GeneralChunkerComponent) chunkSpreadsheet(ctx context.Context, upstream
}
attachGeneralMediaContext(units, c.param.TableContextSize, c.param.ImageContextSize)
chunks := make([]schema.ChunkDoc, 0, len(units))
- pendingRows := make([]schema.ChunkDoc, 0)
- var pendingHeader *schema.ChunkDoc
- flushRows := func() {
- if len(pendingRows) == 0 {
- return
- }
- chunks = append(chunks, mergeSpreadsheetRows(pendingRows, c.param.ChunkTokenSize)...)
- pendingRows = pendingRows[:0]
- }
- flushHeader := func() {
- if pendingHeader == nil {
- return
- }
- chunks = append(chunks, cloneChunkDoc(*pendingHeader))
- pendingHeader = nil
- }
for _, unit := range units {
- if unit.CKType == "table_header" {
- flushRows()
- flushHeader()
- header := cloneChunkDoc(unit)
- pendingHeader = &header
- continue
- }
- if unit.CKType == "table_row" {
- // A header immediately followed by row units is parser metadata for
- // that table, not an additional data chunk. If no row follows, the
- // deferred header is emitted as a header-only table below.
- pendingHeader = nil
- pendingRows = append(pendingRows, unit)
+ if itemDocType(unit) == "table" && isTableStrictHTML(unit.Text) {
+ chunks = append(chunks, c.splitSpreadsheetTable(unit)...)
continue
}
- flushRows()
- flushHeader()
chunks = append(chunks, cloneChunkDoc(unit))
}
- flushRows()
- flushHeader()
childrenPattern := compileChildrenPattern(c.param.ChildrenDelimiters)
chunks = finalizeGeneralChunks(chunks, childrenPattern)
if len(chunks) == 0 {
@@ -861,6 +865,64 @@ func (c *GeneralChunkerComponent) chunkSpreadsheet(ctx context.Context, upstream
return chunkOutputs(chunks), nil
}
+// splitSpreadsheetTable enforces the ChunkTokenSize budget on a table
+// segment: sub-tables replicate the caption and header rows, and the
+// row-aligned positions matrix is sliced per sub-table (R11) so each chunk
+// carries exactly its own rows' tuples. A segment that fits is emitted
+// whole; a split whose matrix cannot be aligned to the markup rows drops
+// positions rather than pointing every sub-table row at the full matrix (R1).
+//
+// The legacy path merged adjacent rows by token budget into multi-row chunks
+// and deferred the header; a segment now passes through whole unless the
+// budget forces a cut, and the header repeats in every part.
+func (c *GeneralChunkerComponent) splitSpreadsheetTable(unit schema.ChunkDoc) []schema.ChunkDoc {
+ parts, ranges, headerRows := splitLargeHTMLTable(unit.Text, c.param.ChunkTokenSize, tokenizeStr)
+ if ranges == nil {
+ return []schema.ChunkDoc{cloneChunkDoc(unit)}
+ }
+ var matrix [][]float64
+ // Spreadsheet items only: PDF items write layout boxes into the same
+ // field, and those must never be sliced into sub-tables as row tuples.
+ if unit.SheetIndex != nil && len(unit.Positions) > 0 {
+ if err := json.Unmarshal(unit.Positions, &matrix); err != nil {
+ matrix = nil
+ }
+ }
+ // The matrix carries one five-field tuple per <tr>; the splitter cut those
+ // same rows, so its last range end is the total data-row count — no second
+ // full parse of the markup is needed to compare lengths.
+ aligned := false
+ if len(ranges) > 0 {
+ if expected := headerRows + ranges[len(ranges)-1][1]; expected > 0 && len(matrix) == expected {
+ aligned = true
+ for _, tuple := range matrix {
+ if len(tuple) != 5 {
+ aligned = false
+ break
+ }
+ }
+ }
+ }
+ out := make([]schema.ChunkDoc, 0, len(parts))
+ for i, part := range parts {
+ piece := cloneChunkDoc(unit)
+ piece.Text = part
+ piece.Positions = nil
+ lo := headerRows + ranges[i][0]
+ hi := headerRows + ranges[i][1]
+ if aligned && hi <= len(matrix) {
+ sub := make([][]float64, 0, headerRows+hi-lo)
+ sub = append(sub, matrix[:headerRows]...)
+ sub = append(sub, matrix[lo:hi]...)
+ if encoded, err := json.Marshal(sub); err == nil {
+ piece.Positions = encoded
+ }
+ }
+ out = append(out, piece)
+ }
+ return out
+}
+
func (c *GeneralChunkerComponent) chunkGeneral(ctx context.Context, upstream schema.ChunkerFromUpstream) (map[string]any, error) {
if err := ctx.Err(); err != nil {
return nil, fmt.Errorf("GeneralChunker: %w", err)
@@ -933,32 +995,7 @@ func mergeGeneralUnits(units []schema.ChunkDoc, target int, overlapPct float64,
return applyGeneralOverlap(merged, overlapPct, joinSep)
}
-func mergeSpreadsheetRows(rows []schema.ChunkDoc, target int) []schema.ChunkDoc {
- merged := make([]schema.ChunkDoc, 0, len(rows))
- current := -1
- for _, row := range rows {
- if row.CKType != "table_row" || itemDocType(row) != "text" {
- merged = append(merged, cloneChunkDoc(row))
- current = -1
- continue
- }
- count := generalUnitTokens(row)
- if current < 0 || count > target || !sameSpreadsheetTable(merged[current], row) || intValue(merged[current].TKNums)+count > target {
- row.TKNums = intPtr(count)
- merged = append(merged, cloneChunkDoc(row))
- current = len(merged) - 1
- continue
- }
- mergeGeneralChunk(&merged[current], row, "\n")
- mergeSpreadsheetRowRange(&merged[current], row)
- }
- return merged
-}
-
func sameSpreadsheetTable(first, second schema.ChunkDoc) bool {
- if first.TableID != "" || second.TableID != "" {
- return first.TableID == second.TableID
- }
if first.SheetIndex != nil || second.SheetIndex != nil {
return first.SheetIndex != nil && second.SheetIndex != nil && *first.SheetIndex == *second.SheetIndex
}
@@ -980,32 +1017,6 @@ func spreadsheetPositionSheet(doc schema.ChunkDoc) (float64, bool) {
return row[0], true
}
-func mergeSpreadsheetRowRange(dst *schema.ChunkDoc, src schema.ChunkDoc) {
- dst.RowStart = minSpreadsheetInt(dst.RowStart, src.RowStart, false)
- dst.RowEnd = minSpreadsheetInt(dst.RowEnd, src.RowEnd, true)
- dst.ColStart = minSpreadsheetInt(dst.ColStart, src.ColStart, false)
- dst.ColEnd = minSpreadsheetInt(dst.ColEnd, src.ColEnd, true)
-}
-
-func minSpreadsheetInt(first, second *int, maximum bool) *int {
- if first == nil {
- if second == nil {
- return nil
- }
- value := *second
- return &value
- }
- if second == nil {
- value := *first
- return &value
- }
- value := *first
- if (maximum && *second > value) || (!maximum && *second < value) {
- value = *second
- }
- return &value
-}
-
func generalUnitTokens(unit schema.ChunkDoc) int {
if unit.TKNums != nil && *unit.TKNums > 0 {
return *unit.TKNums
diff --git a/internal/ingestion/component/chunker/general_merge_test.go b/internal/ingestion/component/chunker/general_merge_test.go
index b8bf8d5702..53d8e21d1d 100644
--- a/internal/ingestion/component/chunker/general_merge_test.go
+++ b/internal/ingestion/component/chunker/general_merge_test.go
@@ -42,65 +42,17 @@ func TestMergeGeneralUnitsAllowsOneBoundaryOverflow(t *testing.T) {
}
}
-func TestMergeSpreadsheetRowsUsesProjectedCapAndSheetBoundary(t *testing.T) {
- sheetOne := 1
- sheetTwo := 2
- rows := []schema.ChunkDoc{
- {Text: "row-1", DocType: "text", CKType: "table_row", TKNums: intPtr(2), SheetIndex: &sheetOne, RowStart: intPtr(2), RowEnd: intPtr(2), ColStart: intPtr(1), ColEnd: intPtr(2)},
- {Text: "row-2", DocType: "text", CKType: "table_row", TKNums: intPtr(1), SheetIndex: &sheetOne, RowStart: intPtr(3), RowEnd: intPtr(3), ColStart: intPtr(1), ColEnd: intPtr(2)},
- {Text: "row-3", DocType: "text", CKType: "table_row", TKNums: intPtr(1), SheetIndex: &sheetTwo, RowStart: intPtr(2), RowEnd: intPtr(2), ColStart: intPtr(1), ColEnd: intPtr(2)},
- }
-
- got := mergeSpreadsheetRows(rows, 3)
- if texts := generalChunkTexts(got); !reflect.DeepEqual(texts, []string{"row-1\nrow-2", "row-3"}) {
- t.Fatalf("texts = %q", texts)
- }
- if len(got) != 2 || got[0].TKNums == nil || *got[0].TKNums != 3 {
- t.Fatalf("merged row token count = %#v, want 3", got)
- }
- if got[0].RowStart == nil || *got[0].RowStart != 2 || got[0].RowEnd == nil || *got[0].RowEnd != 3 {
- t.Fatalf("merged row range = start:%v end:%v", got[0].RowStart, got[0].RowEnd)
- }
- if got[0].SheetIndex == nil || *got[0].SheetIndex != 1 || got[1].SheetIndex == nil || *got[1].SheetIndex != 2 {
- t.Fatalf("sheet boundaries not preserved: %#v", got)
- }
-}
-
-func TestMergeSpreadsheetRowsKeepsOversizedRowAndCapZeroAtomic(t *testing.T) {
- sheet := 1
- rows := []schema.ChunkDoc{
- {Text: "small", DocType: "text", CKType: "table_row", TKNums: intPtr(1), SheetIndex: &sheet, RowStart: intPtr(2), RowEnd: intPtr(2)},
- {Text: "oversized", DocType: "text", CKType: "table_row", TKNums: intPtr(8), SheetIndex: &sheet, RowStart: intPtr(3), RowEnd: intPtr(3)},
- {Text: "after", DocType: "text", CKType: "table_row", TKNums: intPtr(1), SheetIndex: &sheet, RowStart: intPtr(4), RowEnd: intPtr(4)},
- }
- if got := mergeSpreadsheetRows(rows, 3); !reflect.DeepEqual(generalChunkTexts(got), []string{"small", "oversized", "after"}) {
- t.Fatalf("oversized texts = %q", generalChunkTexts(got))
- }
- if got := mergeSpreadsheetRows(rows, 0); !reflect.DeepEqual(generalChunkTexts(got), []string{"small", "oversized", "after"}) {
- t.Fatalf("zero-cap texts = %q", generalChunkTexts(got))
- }
-}
-
-func TestGeneralChunkerSpreadsheetConsumesRowIR(t *testing.T) {
- component, err := NewGeneralChunker(map[string]any{"chunk_token_size": 3})
- if err != nil {
- t.Fatalf("NewGeneralChunker: %v", err)
- }
- out, err := component.Invoke(t.Context(), nil, map[string]any{
- "name": "orders.xlsx",
- "file_type": "xlsx",
- "output_format": "json",
- "json": []map[string]any{
- {"text": "ID; Status", "doc_type_kwd": "table", "ck_type": "table_header", "sheet_index": 1, "table_id": "sheet-1"},
- {"text": "ID:A-100; Status:paid", "doc_type_kwd": "text", "ck_type": "table_row", "tk_nums": 2, "sheet_index": 1, "table_id": "sheet-1", "row_start": 2, "row_end": 2},
- {"text": "ID:A-101; Status:pending", "doc_type_kwd": "text", "ck_type": "table_row", "tk_nums": 1, "sheet_index": 1, "table_id": "sheet-1", "row_start": 3, "row_end": 3},
- },
- })
- if err != nil {
- t.Fatalf("Invoke: %v", err)
- }
- if texts := outputTexts(t, out); !reflect.DeepEqual(texts, []string{"ID:A-100; Status:paid\nID:A-101; Status:pending"}) {
- t.Fatalf("spreadsheet chunks = %q", texts)
+// TestSameSpreadsheetTableUsesPositionSheet covers the media-context sheet
+// guard once carried by merged row runs: identity falls through to the
+// positions matrix when the sheet fields are absent.
+func TestSameSpreadsheetTableUsesPositionSheet(t *testing.T) {
+ one := schema.ChunkDoc{Text: "sheet-1", Positions: json.RawMessage(`[[1,2,2,1,2]]`)}
+ two := schema.ChunkDoc{Text: "sheet-2", Positions: json.RawMessage(`[[2,2,2,1,2]]`)}
+ if !sameSpreadsheetTable(one, one) {
+ t.Errorf("same position sheet rejected")
+ }
+ if sameSpreadsheetTable(one, two) {
+ t.Errorf("different position sheets accepted")
}
}
@@ -171,18 +123,6 @@ func TestApplyGeneralOverlapDoesNotAccumulatePositions(t *testing.T) {
}
}
-func TestMergeSpreadsheetRowsUsesPositionSheetWhenIdentityFieldsAreMissing(t *testing.T) {
- rows := []schema.ChunkDoc{
- {Text: "sheet-1-row", DocType: "text", CKType: "table_row", TKNums: intPtr(1), Positions: json.RawMessage(`[[1,2,2,1,2]]`)},
- {Text: "sheet-2-row", DocType: "text", CKType: "table_row", TKNums: intPtr(1), Positions: json.RawMessage(`[[2,2,2,1,2]]`)},
- }
-
- got := mergeSpreadsheetRows(rows, 10)
- if texts := generalChunkTexts(got); !reflect.DeepEqual(texts, []string{"sheet-1-row", "sheet-2-row"}) {
- t.Fatalf("position-only sheet boundary was lost: texts = %q", texts)
- }
-}
-
func TestMergeMarkdownUnitsCarriesCharacterOverlapIntoNextBudget(t *testing.T) {
units := []schema.ChunkDoc{
{Text: "alpha beta", DocType: "text", CKType: "text"},
diff --git a/internal/ingestion/component/chunker/general_test.go b/internal/ingestion/component/chunker/general_test.go
index e77939bb7a..455fa6f1cf 100644
--- a/internal/ingestion/component/chunker/general_test.go
+++ b/internal/ingestion/component/chunker/general_test.go
@@ -38,6 +38,43 @@ import (
"ragflow/internal/ingestion/task/indexdoc"
)
+// spreadsheetSegmentItem builds one wire-format spreadsheet table segment:
+// the markup the parser renders (captioned <table>, <th> header, <td> rows)
+// plus the row-aligned positions matrix (one tuple per <tr>, header first).
+// firstRow is the sheet row of the first DATA row; the header tuple covers
+// firstRow-1.
+func spreadsheetSegmentItem(sheet string, header []string, rows [][]string, sheetIndex, firstRow int) map[string]any {
+ var markup strings.Builder
+ markup.WriteString("<table><caption>" + sheet + "</caption>\n<tr>")
+ for _, cell := range header {
+ markup.WriteString("<th>" + cell + "</th>")
+ }
+ markup.WriteString("</tr>\n")
+ for _, row := range rows {
+ markup.WriteString("<tr>")
+ for _, cell := range row {
+ markup.WriteString("<td>" + cell + "</td>")
+ }
+ markup.WriteString("</tr>\n")
+ }
+ markup.WriteString("</table>\n")
+
+ tuple := func(row int) []float64 {
+ return []float64{float64(sheetIndex), float64(row), float64(row), 1, float64(len(header))}
+ }
+ matrix := [][]float64{tuple(firstRow - 1)}
+ for i := range rows {
+ matrix = append(matrix, tuple(firstRow+i))
+ }
+ return map[string]any{
+ "text": markup.String(),
+ "doc_type_kwd": "table",
+ "ck_type": "table",
+ "sheet_index": sheetIndex,
+ "positions": matrix,
+ }
+}
+
func TestGeneralChunkerRegistered(t *testing.T) {
factory, category, metadata, ok := runtime.DefaultRegistry.Lookup(ComponentNameGeneralChunker)
if !ok {
@@ -753,18 +790,12 @@ func TestGeneralChunkerSpreadsheetHeaderOnlyPreservesHeaderChunk(t *testing.T) {
if err != nil {
t.Fatalf("NewGeneralChunker: %v", err)
}
+ item := spreadsheetSegmentItem("Sheet1", []string{"Name", "Amount"}, nil, 1, 1)
out, err := component.Invoke(t.Context(), nil, map[string]any{
"name": "headers.xlsx",
"file_type": "xlsx",
"output_format": "json",
- "json": []map[string]any{{
- "text": "Name; Amount",
- "doc_type_kwd": "table",
- "ck_type": "table_header",
- "sheet_index": 1,
- "table_id": "sheet-1",
- "positions": []any{[]any{1.0, 1.0, 1.0, 1.0, 2.0}},
- }},
+ "json": []map[string]any{item},
})
if err != nil {
t.Fatalf("Invoke: %v", err)
@@ -773,14 +804,14 @@ func TestGeneralChunkerSpreadsheetHeaderOnlyPreservesHeaderChunk(t *testing.T) {
if len(chunks) != 1 {
t.Fatalf("chunks = %#v, want one header-only chunk", chunks)
}
- if chunks[0]["text"] != "Name; Amount" || chunks[0]["ck_type"] != "table_header" {
+ if chunks[0]["text"] != item["text"] || chunks[0]["ck_type"] != "table" {
t.Fatalf("header-only chunk = %#v", chunks[0])
}
}
func TestGeneralChunkerSpreadsheetAttachesImageContext(t *testing.T) {
component, err := NewGeneralChunker(map[string]any{
- "chunk_token_size": 10,
+ "chunk_token_size": 100,
"image_context_size": 10,
})
if err != nil {
@@ -791,9 +822,9 @@ func TestGeneralChunkerSpreadsheetAttachesImageContext(t *testing.T) {
"file_type": "xlsx",
"output_format": "json",
"json": []map[string]any{
- {"text": "Revenue", "doc_type_kwd": "text", "ck_type": "table_row", "table_id": "sheet-1", "sheet_index": 1, "tk_nums": 1},
- {"text": "B2", "doc_type_kwd": "image", "ck_type": "image", "image": "figure", "table_id": "sheet-1", "sheet_index": 1},
- {"text": "Growth", "doc_type_kwd": "text", "ck_type": "table_row", "table_id": "sheet-1", "sheet_index": 1, "tk_nums": 1},
+ spreadsheetSegmentItem("Sheet1", []string{"Name"}, [][]string{{"Revenue"}}, 1, 2),
+ {"text": "B2", "doc_type_kwd": "image", "ck_type": "image", "image": "figure", "sheet_index": 1},
+ spreadsheetSegmentItem("Sheet1", []string{"Name"}, [][]string{{"Growth"}}, 1, 3),
},
})
if err != nil {
@@ -801,7 +832,7 @@ func TestGeneralChunkerSpreadsheetAttachesImageContext(t *testing.T) {
}
chunks := outputChunks(t, out)
if len(chunks) != 3 {
- t.Fatalf("chunks = %#v, want row, image, row", chunks)
+ t.Fatalf("chunks = %#v, want segment, image, segment", chunks)
}
if chunks[1]["ck_type"] != "image" {
t.Fatalf("image chunk = %#v", chunks[1])
@@ -810,7 +841,7 @@ func TestGeneralChunkerSpreadsheetAttachesImageContext(t *testing.T) {
}
func TestGeneralChunkerSpreadsheetImageContextStopsAtSheetBoundary(t *testing.T) {
- component, err := NewGeneralChunker(map[string]any{"image_context_size": 10})
+ component, err := NewGeneralChunker(map[string]any{"chunk_token_size": 100, "image_context_size": 10})
if err != nil {
t.Fatalf("NewGeneralChunker: %v", err)
}
@@ -819,9 +850,9 @@ func TestGeneralChunkerSpreadsheetImageContextStopsAtSheetBoundary(t *testing.T)
"file_type": "xlsx",
"output_format": "json",
"json": []map[string]any{
- {"text": "Sheet one", "doc_type_kwd": "text", "ck_type": "table_row", "table_id": "sheet-1", "sheet_index": 1, "tk_nums": 1},
- {"text": "B2", "doc_type_kwd": "image", "ck_type": "image", "image": "figure", "table_id": "sheet-1", "sheet_index": 1},
- {"text": "Sheet two", "doc_type_kwd": "text", "ck_type": "table_row", "table_id": "sheet-2", "sheet_index": 2, "tk_nums": 1},
+ spreadsheetSegmentItem("Sheet1", []string{"Name"}, [][]string{{"Sheet one"}}, 1, 2),
+ {"text": "B2", "doc_type_kwd": "image", "ck_type": "image", "image": "figure", "sheet_index": 1},
+ spreadsheetSegmentItem("Sheet2", []string{"Name"}, [][]string{{"Sheet two"}}, 2, 2),
},
})
if err != nil {
@@ -829,11 +860,104 @@ func TestGeneralChunkerSpreadsheetImageContextStopsAtSheetBoundary(t *testing.T)
}
chunks := outputChunks(t, out)
if len(chunks) != 3 {
- t.Fatalf("chunks = %#v, want row, image, row", chunks)
+ t.Fatalf("chunks = %#v, want segment, image, segment", chunks)
}
assertMaterializedMediaContext(t, chunks[1], "Sheet oneB2")
}
+// TestGeneralChunkerSpreadsheetSplitsLargeSegmentIntoBoundedChunks pins the
+// chunker-side size budget: a segment over ChunkTokenSize is cut into
+// sub-tables that each replicate the header row and carry only their own
+// rows' position tuples (R11). Without the cut a wide/long row runs past the
+// embedding truncation point and its tail rows silently disappear.
+func TestGeneralChunkerSpreadsheetSplitsLargeSegmentIntoBoundedChunks(t *testing.T) {
+ component, err := NewGeneralChunker(map[string]any{"chunk_token_size": 20})
+ if err != nil {
+ t.Fatalf("NewGeneralChunker: %v", err)
+ }
+ var rows [][]string
+ for i := 0; i < 6; i++ {
+ rows = append(rows, []string{fmt.Sprintf("value-%d-with-some-padding", i)})
+ }
+ item := spreadsheetSegmentItem("Sheet1", []string{"Name"}, rows, 1, 2)
+ out, err := component.Invoke(t.Context(), nil, map[string]any{
+ "name": "long.xlsx",
+ "file_type": "xlsx",
+ "output_format": "json",
+ "json": []map[string]any{item},
+ })
+ if err != nil {
+ t.Fatalf("Invoke: %v", err)
+ }
+ chunks := outputChunks(t, out)
+ if len(chunks) < 2 {
+ t.Fatalf("chunks = %d, want the segment split under the token budget", len(chunks))
+ }
+ covered := 0
+ for i, chunk := range chunks {
+ text, _ := chunk["text"].(string)
+ if !strings.Contains(text, "<th>Name</th>") {
+ t.Errorf("chunk %d lost the replicated header", i)
+ }
+ var matrix [][]float64
+ raw, _ := json.Marshal(chunk["positions"])
+ if err := json.Unmarshal(raw, &matrix); err != nil {
+ t.Fatalf("chunk %d positions = %v, want a row-aligned matrix", i, chunk["positions"])
+ }
+ // One header tuple + this sub-table's own data tuples, never the
+ // full segment matrix.
+ if len(matrix) < 2 {
+ t.Fatalf("chunk %d matrix = %v, want header plus data rows", i, matrix)
+ }
+ if matrix[0][1] != 1 {
+ t.Errorf("chunk %d header tuple = %v, want sheet 1 row 1", i, matrix[0])
+ }
+ trCount := strings.Count(text, "<tr>")
+ if trCount != len(matrix) {
+ t.Errorf("chunk %d has %d rows but %d position tuples", i, trCount, len(matrix))
+ }
+ covered += len(matrix) - 1
+ }
+ if covered != len(rows) {
+ t.Errorf("sub-tables cover %d data rows, want %d", covered, len(rows))
+ }
+}
+
+// TestGeneralChunkerSpreadsheetDropsMisalignedPositionsOnSplit: when the
+// matrix cannot be aligned one tuple per row, a split drops positions
+// entirely rather than pointing every sub-table row at the full matrix (R1).
+func TestGeneralChunkerSpreadsheetDropsMisalignedPositionsOnSplit(t *testing.T) {
+ component, err := NewGeneralChunker(map[string]any{"chunk_token_size": 20})
+ if err != nil {
+ t.Fatalf("NewGeneralChunker: %v", err)
+ }
+ var rows [][]string
+ for i := 0; i < 6; i++ {
+ rows = append(rows, []string{fmt.Sprintf("value-%d-with-some-padding", i)})
+ }
+ item := spreadsheetSegmentItem("Sheet1", []string{"Name"}, rows, 1, 2)
+ // One whole-segment tuple instead of the per-row matrix.
+ item["positions"] = [][]float64{{1, 1, 7, 1, 1}}
+ out, err := component.Invoke(t.Context(), nil, map[string]any{
+ "name": "long.xlsx",
+ "file_type": "xlsx",
+ "output_format": "json",
+ "json": []map[string]any{item},
+ })
+ if err != nil {
+ t.Fatalf("Invoke: %v", err)
+ }
+ chunks := outputChunks(t, out)
+ if len(chunks) < 2 {
+ t.Fatalf("chunks = %d, want the segment split under the token budget", len(chunks))
+ }
+ for i, chunk := range chunks {
+ if _, exists := chunk["positions"]; exists {
+ t.Errorf("chunk %d keeps a misaligned positions payload: %v", i, chunk["positions"])
+ }
+ }
+}
+
// TestGeneralChunkerMediaContextReachesChunkIDAndIndexContent pins the
// downstream half of the fold: once the context is part of the chunk body, it
// is also part of the persisted content_with_weight and of the chunk id
@@ -955,3 +1079,96 @@ func TestGeneralChunkerPDFAttachesOutlineOnce(t *testing.T) {
t.Errorf("outline entry = %#v", entry)
}
}
+
+// TestGeneralChunkerNonSpreadsheetTableNotMediaContext: a table item without
+// spreadsheet identity must not become a media-context source. The DOCX and
+// PDF strategies share the collector, so reading their tables as context would
+// change those documents' image chunks (and their chunk ids) as a side effect
+// of the spreadsheet wire switch.
+func TestGeneralChunkerNonSpreadsheetTableNotMediaContext(t *testing.T) {
+ component, err := NewGeneralChunker(map[string]any{
+ "chunk_token_size": 100,
+ "image_context_size": 10,
+ })
+ if err != nil {
+ t.Fatalf("NewGeneralChunker: %v", err)
+ }
+ table := map[string]any{
+ "text": "<table><caption>Doc</caption>\n<tr><th>Name</th></tr>\n<tr><td>Revenue</td></tr>\n</table>\n",
+ "doc_type_kwd": "table",
+ "ck_type": "table",
+ }
+ out, err := component.Invoke(t.Context(), nil, map[string]any{
+ "name": "document.docx",
+ "file_type": "docx",
+ "output_format": "json",
+ "json": []map[string]any{
+ table,
+ {"text": "B2", "doc_type_kwd": "image", "ck_type": "image", "image": "figure"},
+ table,
+ },
+ })
+ if err != nil {
+ t.Fatalf("Invoke: %v", err)
+ }
+ chunks := outputChunks(t, out)
+ if len(chunks) != 3 {
+ t.Fatalf("chunks = %#v, want table, image, table", chunks)
+ }
+ if chunks[1]["ck_type"] != "image" {
+ t.Fatalf("image chunk = %#v", chunks[1])
+ }
+ if got, _ := chunks[1]["text"].(string); got != "B2" {
+ t.Errorf("image chunk text = %q, want B2 with no table text folded in", got)
+ }
+ for _, key := range []string{"context_above", "context_below"} {
+ if _, exists := chunks[1][key]; exists {
+ t.Errorf("image chunk must not carry %s: %v", key, chunks[1][key])
+ }
+ }
+}
+
+// TestGeneralChunkerSpreadsheetSplitDropsNonSpreadsheetPositions: sub-tables
+// only receive sliced positions when the item carries spreadsheet identity and
+// the matrix is five-field tuples aligned to the splitter's rows; a PDF-style
+// matrix or a short tuple is dropped rather than misattributed.
+func TestGeneralChunkerSpreadsheetSplitDropsNonSpreadsheetPositions(t *testing.T) {
+ rows := [][]string{{"v0", "p0"}, {"v1", "p1"}, {"v2", "p2"}, {"v3", "p3"}, {"v4", "p4"}, {"v5", "p5"}}
+ cases := []struct {
+ name string
+ mutate func(item map[string]any)
+ }{
+ {"no spreadsheet identity", func(item map[string]any) { delete(item, "sheet_index") }},
+ {"short tuples", func(item map[string]any) {
+ item["positions"] = [][]float64{{1}, {1}, {1}, {1}, {1}, {1}, {1}}
+ }},
+ }
+ for _, tc := range cases {
+ t.Run(tc.name, func(t *testing.T) {
+ component, err := NewGeneralChunker(map[string]any{"chunk_token_size": 20})
+ if err != nil {
+ t.Fatalf("NewGeneralChunker: %v", err)
+ }
+ item := spreadsheetSegmentItem("Sheet1", []string{"name", "price"}, rows, 1, 2)
+ tc.mutate(item)
+ out, err := component.Invoke(t.Context(), nil, map[string]any{
+ "name": "long.xlsx",
+ "file_type": "xlsx",
+ "output_format": "json",
+ "json": []map[string]any{item},
+ })
+ if err != nil {
+ t.Fatalf("Invoke: %v", err)
+ }
+ chunks := outputChunks(t, out)
+ if len(chunks) < 2 {
+ t.Fatalf("expected the segment to be split, got %d chunks", len(chunks))
+ }
+ for i, chunk := range chunks {
+ if p, ok := chunk["positions"]; ok && p != nil && fmt.Sprintf("%v", p) != "[]" && fmt.Sprintf("%v", p) != "<nil>" {
+ t.Errorf("chunk %d kept positions without a usable spreadsheet matrix: %v", i, p)
+ }
+ }
+ })
+ }
+}
diff --git a/internal/ingestion/component/chunker/html_rows.go b/internal/ingestion/component/chunker/html_rows.go
new file mode 100644
index 0000000000..a1874deac0
--- /dev/null
+++ b/internal/ingestion/component/chunker/html_rows.go
@@ -0,0 +1,168 @@
+//
+// Copyright 2026 The InfiniFlow Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+
+package chunker
+
+import (
+ "strings"
+
+ "golang.org/x/net/html"
+)
+
+// ---------------------------------------------------------------------------
+// HTML table row extraction (shared by the Table, QA and General chunkers)
+// ---------------------------------------------------------------------------
+
+// tableRows returns every row's cell text, header rows included.
+func tableRows(htmlStr string) [][]string {
+ rows, _ := tableRowsWithHeader(htmlStr)
+ return rows
+}
+
+// tableRowsWithHeader returns every row's cell text in document order, plus
+// the number of leading header rows. A row counts as a header row when any of
+// its cells is a <th>. When no row uses <th> at all the first row is treated
+// as the header — the convention spreadsheet and markup tables are read with.
+//
+// The markup is parsed into a tree rather than matched with a regex because
+// this input is not guaranteed to be well formed: seven parsers render table
+// items (xlsx, csv, docx, html, pdf, …) and some of that markup originates
+// from user-supplied documents. A tree also settles the cases a tag-level
+// scan gets wrong: a nested <table> no longer terminates its enclosing row
+// early — that row keeps its own cells, with the nested table's text folded
+// into the cell holding it — and a row or cell missing its closing tag is
+// recovered rather than dropped.
+func tableRowsWithHeader(htmlStr string) (rows [][]string, headerCount int) {
+ // A <tr> outside a <table> is discarded by the HTML5 "in body" insertion
+ // mode, so a bare row fragment would yield nothing. Give the parser the
+ // table context it needs instead of dropping the rows silently.
+ lower := strings.ToLower(htmlStr)
+ if strings.Contains(lower, "<tr") && !strings.Contains(lower, "<table") {
+ htmlStr = "<table>" + htmlStr + "</table>"
+ }
+ doc, err := html.Parse(strings.NewReader(htmlStr))
+ if err != nil {
+ return nil, 0
+ }
+ var headerFlags []bool
+ var walk func(*html.Node)
+ walk = func(n *html.Node) {
+ // An inert subtree is parsed but never rendered. <template> puts its
+ // content straight into the ordinary child list (the parser has no
+ // separate template-contents field), so without this its rows would
+ // be read as rows of the enclosing table.
+ if n.Type == html.ElementNode && isInertElement(n.Data) {
+ return
+ }
+ if isHTMLElement(n, "tr") {
+ var cells []string
+ anyTh := false
+ for c := n.FirstChild; c != nil; c = c.NextSibling {
+ if isHTMLElement(c, "td") || isHTMLElement(c, "th") {
+ if isHTMLElement(c, "th") {
+ anyTh = true
+ }
+ cells = append(cells, cellText(c))
+ }
+ }
+ // Return without descending: the cells above already collected
+ // the nested table's text, so its rows must not be reported a
+ // second time as rows of the enclosing table.
+ rows = append(rows, cells)
+ headerFlags = append(headerFlags, anyTh)
+ return
+ }
+ for c := n.FirstChild; c != nil; c = c.NextSibling {
+ walk(c)
+ }
+ }
+ walk(doc)
+ for len(headerFlags) > headerCount && headerFlags[headerCount] {
+ headerCount++
+ }
+ if len(rows) > 0 && headerCount == 0 {
+ headerCount = 1
+ }
+ return rows, headerCount
+}
+
+// isTableStrictHTML reports whether block text is an outer <table> element
+// (the inlined GFM/HTML table). Only such blocks are emitted as structured
+// table items; other raw HTML (e.g. <div>, <style>) is plain text. The
+// spreadsheet wire is the markup this reports on (see the parser's
+// renderSpreadsheetTable), so callers that slice a positions matrix use
+// this strict form rather than the candidate filter.
+func isTableStrictHTML(s string) bool {
+ return strings.HasPrefix(strings.TrimSpace(strings.ToLower(s)), "<table")
+}
+
+// isTableHTML is the cheap candidate filter for table-vs-prose routing: text
+// that opens an outer <table> element, or a bare-row fragment holding <tr>.
+// It only narrows the candidates — the caller decides on tableRows' result —
+// so a block that merely opens with "<table" text and holds no row stays on
+// the prose path instead of silently pairing nothing.
+func isTableHTML(s string) bool {
+ lower := strings.ToLower(s)
+ return strings.HasPrefix(strings.TrimSpace(lower), "<table") || strings.Contains(lower, "<tr")
+}
+
+// cellText returns the visible text of a table cell. The parser hands text
+// nodes over already unescaped, nested markup contributes its text without
+// its tags (a nested table's cells are concatenated, not separated), and a
+// <br> becomes a newline instead of silently gluing the two halves of the
+// cell together.
+func cellText(cell *html.Node) string {
+ var sb strings.Builder
+ var walk func(*html.Node)
+ walk = func(n *html.Node) {
+ switch {
+ case n.Type == html.TextNode:
+ sb.WriteString(n.Data)
+ case isHTMLElement(n, "br"):
+ sb.WriteByte('\n')
+ case n.Type == html.ElementNode && isInertElement(n.Data):
+ // Stop here rather than descending: the content is parsed but
+ // never rendered, so it is not text a reader of the cell sees.
+ return
+ }
+ for c := n.FirstChild; c != nil; c = c.NextSibling {
+ walk(c)
+ }
+ }
+ walk(cell)
+ return strings.TrimSpace(sb.String())
+}
+
+// isHTMLElement reports whether n is an element with the given tag name in
+// the HTML namespace. Foreign content reuses HTML tag names for unrelated
+// elements — an <svg><tr> is not a table row — so matching on the tag name
+// alone would read markup that carries no table semantics.
+func isHTMLElement(n *html.Node, tag string) bool {
+ return n.Type == html.ElementNode && n.Namespace == "" && n.Data == tag
+}
+
+// isInertElement reports whether an element's content is inert — parsed, but
+// never rendered as visible text. An inline <script> or <style> inside a
+// table cell of a user-supplied document would otherwise be embedded into the
+// Q&A pair as if it were part of the sentence, and a <template>'s placeholder
+// rows would be read as real ones.
+func isInertElement(tag string) bool {
+ switch tag {
+ case "script", "style", "noscript", "template":
+ return true
+ }
+ return false
+}
diff --git a/internal/ingestion/component/chunker/html_rows_test.go b/internal/ingestion/component/chunker/html_rows_test.go
new file mode 100644
index 0000000000..76690052bb
--- /dev/null
+++ b/internal/ingestion/component/chunker/html_rows_test.go
@@ -0,0 +1,70 @@
+//
+// Copyright 2026 The InfiniFlow Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+
+package chunker
+
+import "testing"
+
+// TestIsTableStrictHTML pins the predicate's prefix semantics: it reports
+// whether the block opens with the "<table" text, exactly as the markdown
+// parser has always classified its HTML blocks. It is deliberately loose (a
+// "<tableau>" token passes), which is why routing never treats it as the final
+// word — the row walker decides.
+func TestIsTableStrictHTML(t *testing.T) {
+ cases := []struct {
+ text string
+ want bool
+ }{
+ {"<table>", true},
+ {" <TABLE class=\"x\">", true},
+ {"<table-x>", true}, // historical prefix semantics: "<table" prefix only
+ {"<tableau>", true},
+ {"<div><table>", false},
+ {"plain text", false},
+ {"", false},
+ }
+ for _, tc := range cases {
+ if got := isTableStrictHTML(tc.text); got != tc.want {
+ t.Errorf("isTableStrictHTML(%q) = %v, want %v", tc.text, got, tc.want)
+ }
+ }
+}
+
+// TestIsTableHTMLCoversWhatTheWalkerReads guards the routing invariant: the
+// candidate filter must never deny a payload the row walker can read, or a
+// readable table would silently fall through to the prose extractor.
+func TestIsTableHTMLCoversWhatTheWalkerReads(t *testing.T) {
+ texts := []string{
+ "<table><tr><th>q</th><th>a</th></tr></table>",
+ "<table><caption>c</caption>\n<tr><td>q</td><td>a</td></tr>\n</table>\n",
+ "<tr><td>q</td><td>a</td></tr>", // bare fragment
+ "<tr><td>q</td><td>a</td></tr><tr><td>q2</td><td>a2</td></tr>", // bare rows
+ "<tbody><tr><td>q</td><td>a</td></tr></tbody>", // rows wrapped in tbody
+ "<table><tr><td>q<table><tr><td>inner</td></tr></table></td></tr></table>", // nested
+ "<table><template><tr><td>x</td></tr></template></table>", // template rows are inert
+ "<table></table>", // nothing to read
+ "plain text",
+ "",
+ }
+ for _, text := range texts {
+ if len(tableRows(text)) == 0 {
+ continue
+ }
+ if !isTableHTML(text) {
+ t.Errorf("tableRows can read %q but isTableHTML denies it", text)
+ }
+ }
+}
diff --git a/internal/ingestion/component/chunker/pdfcrop_cgo.go b/internal/ingestion/component/chunker/pdfcrop_cgo.go
index 315a047b47..0bd5105cfd 100644
--- a/internal/ingestion/component/chunker/pdfcrop_cgo.go
+++ b/internal/ingestion/component/chunker/pdfcrop_cgo.go
@@ -240,7 +240,7 @@ func cropImageChunks(ctx context.Context, engine deepdoctype.PDFEngine, chunks [
// region (Python restore_pdf_text_previews). A pre-existing Image is never
// re-cropped — cropImageChunks honors that separately.
//
-// CKType is the chunker-layer refinement (heading/table_header/table_row/text/
+// CKType is the chunker-layer refinement (heading/text/image/table/
// image/table). General and token chunkers always set it before cropping, so
// they are handled by the CKType branch above. Group and hierarchy chunkers
// forward the parser's output verbatim, which carries only the coarser
diff --git a/internal/ingestion/component/chunker/pdfcrop_cgo_test.go b/internal/ingestion/component/chunker/pdfcrop_cgo_test.go
index 784ebbe4e4..15447b1512 100644
--- a/internal/ingestion/component/chunker/pdfcrop_cgo_test.go
+++ b/internal/ingestion/component/chunker/pdfcrop_cgo_test.go
@@ -86,7 +86,7 @@ func TestNeedsCrop(t *testing.T) {
{"doc image fallback", schema.ChunkDoc{DocType: "image", PDFPositions: jsonPositions(t, []float64{1, 10, 100, 10, 100})}, true},
{"doc table fallback", schema.ChunkDoc{DocType: "table", Positions: jsonPositions(t, []float64{1, 10, 100, 10, 100})}, true},
{"doc text fallback", schema.ChunkDoc{DocType: "text", PDFPositions: jsonPositions(t, []float64{1, 10, 100, 10, 100})}, true},
- // Heading / table_header / table_row all carry doc_type_kwd "text" but
+ // Heading-tagged blocks carry doc_type_kwd "text" but
// must NOT be previewed when CKType is empty — only a plain text body
// chunk legitimately has CKType empty AND should be previewed. Since
// group/hierarchy cannot distinguish heading from body at this layer,
diff --git a/internal/ingestion/component/chunker/qa.go b/internal/ingestion/component/chunker/qa.go
index aa819d5127..9e9d36d785 100644
--- a/internal/ingestion/component/chunker/qa.go
+++ b/internal/ingestion/component/chunker/qa.go
@@ -38,7 +38,6 @@ import (
"github.com/gomarkdown/markdown"
"github.com/gomarkdown/markdown/parser"
- "golang.org/x/net/html"
"gorm.io/gorm"
"ragflow/internal/agent/runtime"
@@ -208,145 +207,6 @@ func isCSV(name string) bool {
return strings.HasSuffix(strings.ToLower(name), ".csv")
}
-// ---------------------------------------------------------------------------
-// HTML / spreadsheet QA extraction
-// ---------------------------------------------------------------------------
-
-// tableRows walks the parsed HTML and returns the <td>/<th> text of every
-// <tr>, in document order.
-//
-// The markup is parsed into a tree rather than matched with a regex because
-// this input is not guaranteed to be well formed: seven parsers render table
-// items (xlsx, csv, docx, html, pdf, …) and some of that markup originates
-// from user-supplied documents. A tree also settles the cases a tag-level
-// scan gets wrong: a nested <table> no longer terminates its enclosing row
-// early — that row keeps its own cells, with the nested table's text folded
-// into the cell holding it — and a row or cell missing its closing tag is
-// recovered rather than dropped.
-func tableRows(htmlStr string) [][]string {
- // A <tr> outside a <table> is discarded by the HTML5 "in body" insertion
- // mode, so a bare row fragment would yield nothing. Give the parser the
- // table context it needs instead of dropping the rows silently.
- lower := strings.ToLower(htmlStr)
- if strings.Contains(lower, "<tr") && !strings.Contains(lower, "<table") {
- htmlStr = "<table>" + htmlStr + "</table>"
- }
- doc, err := html.Parse(strings.NewReader(htmlStr))
- if err != nil {
- return nil
- }
- var rows [][]string
- var walk func(*html.Node)
- walk = func(n *html.Node) {
- // An inert subtree is parsed but never rendered. <template> puts its
- // content straight into the ordinary child list (the parser has no
- // separate template-contents field), so without this its rows would
- // be read as rows of the enclosing table.
- if n.Type == html.ElementNode && isInertElement(n.Data) {
- return
- }
- if isHTMLElement(n, "tr") {
- var cells []string
- for c := n.FirstChild; c != nil; c = c.NextSibling {
- if isHTMLElement(c, "td") || isHTMLElement(c, "th") {
- cells = append(cells, cellText(c))
- }
- }
- // Return without descending: the cells above already collected
- // the nested table's text, so its rows must not be reported a
- // second time as rows of the enclosing table.
- rows = append(rows, cells)
- return
- }
- for c := n.FirstChild; c != nil; c = c.NextSibling {
- walk(c)
- }
- }
- walk(doc)
- return rows
-}
-
-// cellText returns the visible text of a table cell. The parser hands text
-// nodes over already unescaped, nested markup contributes its text without
-// its tags (a nested table's cells are concatenated, not separated), and a
-// <br> becomes a newline instead of silently gluing the two halves of the
-// cell together.
-func cellText(cell *html.Node) string {
- var sb strings.Builder
- var walk func(*html.Node)
- walk = func(n *html.Node) {
- switch {
- case n.Type == html.TextNode:
- sb.WriteString(n.Data)
- case isHTMLElement(n, "br"):
- sb.WriteByte('\n')
- case n.Type == html.ElementNode && isInertElement(n.Data):
- // Stop here rather than descending: the content is parsed but
- // never rendered, so it is not text a reader of the cell sees.
- return
- }
- for c := n.FirstChild; c != nil; c = c.NextSibling {
- walk(c)
- }
- }
- walk(cell)
- return strings.TrimSpace(sb.String())
-}
-
-// isHTMLElement reports whether n is an element with the given tag name in
-// the HTML namespace. Foreign content reuses HTML tag names for unrelated
-// elements — an <svg><tr> is not a table row — so matching on the tag name
-// alone would read markup that carries no table semantics.
-func isHTMLElement(n *html.Node, tag string) bool {
- return n.Type == html.ElementNode && n.Namespace == "" && n.Data == tag
-}
-
-// isInertElement reports whether an element's content is inert — parsed, but
-// never rendered as visible text. An inline <script> or <style> inside a
-// table cell of a user-supplied document would otherwise be embedded into the
-// Q&A pair as if it were part of the sentence, and a <template>'s placeholder
-// rows would be read as real ones.
-func isInertElement(tag string) bool {
- switch tag {
- case "script", "style", "noscript", "template":
- return true
- }
- return false
-}
-
-// extractQATable turns table markup into Q&A pairs: the first two non-empty
-// cells of a row become the question and the answer. strictPairs is the CSV
-// contract (Python qa.py:365) and requires a row to have exactly two cells
-// instead of taking the first two.
-func extractQATable(htmlStr string, strictPairs bool) []qaPair {
- if htmlStr == "" {
- return nil
- }
- rows := tableRows(htmlStr)
- pairs := make([]qaPair, 0, len(rows))
- for _, cells := range rows {
- // Python qa.py:365 requires exactly two fields for CSV pairs.
- if strictPairs && len(cells) != 2 {
- if len(pairs) > 0 {
- pairs[len(pairs)-1].Answer += "\n" + strings.Join(cells, ",")
- }
- continue
- }
- var texts []string
- for _, cell := range cells {
- if cell != "" {
- texts = append(texts, cell)
- }
- }
- if len(texts) >= 2 {
- // RowNum mirrors Python qa.py's enumerate over the extracted
- // pairs (beAdoc(..., row_num=ii)) → top_int.
- pairs = append(pairs, qaPair{Question: texts[0], Answer: texts[1], RowNum: len(pairs)})
- }
- }
- return pairs
-}
-
// ---------------------------------------------------------------------------
// Markdown QA extraction
// ---------------------------------------------------------------------------
@@ -543,59 +403,118 @@ func extractQAJSON(items []schema.ChunkDoc, fileType string) []qaPair {
strictCSV := strings.EqualFold(fileType, "csv")
for _, item := range items {
var tmp []qaPair
- if item.CKType == "table_header" || item.CKType == "table_row" {
- // QA spreadsheets have no header row: the parser's structural
- // table_header is still the first question/answer record.
- if strictCSV && len(item.Cells) != 2 {
- if len(pairs) > 0 {
- pairs[len(pairs)-1].Answer += "\n" + strings.Join(item.Cells, ",")
- }
- continue
- }
- tmp = extractQARowCells(item.Cells, strictCSV, item.RowStart)
+ txt, _ := itemText(item)
+ if txt == "" {
+ continue
+ }
+ // Route on what the row walker can read, not on the type label.
+ // doc_type:"table" is not truthful for every producer: the
+ // opendataloader, tcadp and somark PDF parsers tag text that holds
+ // no <table> markup, so such an item used to enter the table
+ // extractor, find no rows, and silently lose every pair; conversely
+ // a text-labelled block holding real table markup is read as a
+ // table. isTableHTML is only the cheap candidate filter: the
+ // walker's result decides, so nothing it can read is denied, and a
+ // block that merely opens with "<table" text (no row) stays on the
+ // prose path instead of silently pairing nothing. (Python's qa.py
+ // drops plain-text table payloads such as PDF tables entirely; the
+ // walker keeps their pairs.) Pre-upgrade row-IR payloads
+ // (ck_type: table_row/table_header with cells) hold no markup and
+ // fall through to the text extractor, so documents from before this
+ // wire must be re-parsed rather than re-chunked.
+ var rows [][]string
+ if isTableHTML(txt) {
+ rows = tableRows(txt)
+ }
+ if len(rows) > 0 {
+ tmp = qaPairsFromRows(rows, strictCSV, item.Positions, item.SheetIndex != nil)
} else {
- txt, _ := itemText(item)
- if txt == "" {
- continue
- }
- // Non-spreadsheet table items may still carry HTML markup. Keep
- // the HTML fallback for parsers that do not expose typed cells.
- if itemDocType(item) == "table" {
- tmp = extractQATable(txt, strictCSV)
- } else {
- tmp = extractQAText(txt)
- }
+ tmp = extractQAText(txt)
}
// Preserve the source item's image id and coordinates on each
- // extracted pair
+ // extracted pair. A row-aligned positions matrix already gave each
+ // pair its own tuple (R1: a row chunk must not carry the whole
+ // table's matrix in positions[0]).
for _, p := range tmp {
p.Image = item.Image
p.PDFPositions = item.PDFPositions
- p.Positions = item.Positions
+ if len(p.Positions) == 0 {
+ p.Positions = item.Positions
+ }
pairs = append(pairs, p)
}
}
return pairs
}
-func extractQARowCells(cells []string, strict bool, rowStart *int) []qaPair {
- if strict && len(cells) != 2 {
+// extractQATable turns table markup from an HTML payload into Q&A pairs: the
+// JSON dispatch routes on the same walker, so both paths share
+// qaPairsFromRows. These items carry no positions payload of their own.
+func extractQATable(htmlStr string, strictPairs bool) []qaPair {
+ if htmlStr == "" {
return nil
}
- texts := make([]string, 0, len(cells))
- for _, cell := range cells {
- if cell = strings.TrimSpace(cell); cell != "" {
- texts = append(texts, cell)
+ return qaPairsFromRows(tableRows(htmlStr), strictPairs, nil, false)
+}
+
+// qaPairsFromRows builds the pairs of one table: the first two non-empty
+// cells of a row become the question and the answer. strictPairs is the CSV
+// contract (Python qa.py:365) and requires a row to have exactly two cells
+// instead of taking the first two.
+//
+// positions is the item's own positions payload, and spreadsheetPositions
+// says whether it may be read as the spreadsheet wire's matrix. When it is
+// (identity present, one five-field tuple per <tr>), each pair carries only
+// its row's tuple and takes its 0-based record index from that tuple's
+// rowStart (top_int = positions[i][1] - 1), which reproduces the legacy
+// row-IR numbering. Otherwise pairs are numbered by extraction order,
+// mirroring Python qa.py's enumerate over the extracted pairs — PDF items
+// write layout boxes into the same field, so they must not be reinterpreted
+// as spreadsheet rows.
+func qaPairsFromRows(rows [][]string, strictPairs bool, positions json.RawMessage, spreadsheetPositions bool) []qaPair {
+ var matrix [][]float64
+ if len(positions) > 0 {
+ if err := json.Unmarshal(positions, &matrix); err != nil {
+ matrix = nil
+ }
+ }
+ rowAligned := spreadsheetPositions && len(matrix) == len(rows) && len(rows) > 0
+ if rowAligned {
+ for _, tuple := range matrix {
+ if len(tuple) != 5 {
+ rowAligned = false
+ break
+ }
}
}
- if len(texts) < 2 {
- return nil
- }
- rowNum := -1
- if rowStart != nil && *rowStart > 0 {
- rowNum = *rowStart - 1
+ pairs := make([]qaPair, 0, len(rows))
+ for i, cells := range rows {
+ // Python qa.py:365 requires exactly two fields for CSV pairs.
+ if strictPairs && len(cells) != 2 {
+ if len(pairs) > 0 {
+ pairs[len(pairs)-1].Answer += "\n" + strings.Join(cells, ",")
+ }
+ continue
+ }
+ var texts []string
+ for _, cell := range cells {
+ if cell != "" {
+ texts = append(texts, cell)
+ }
+ }
+ if len(texts) < 2 {
+ continue
+ }
+ pair := qaPair{Question: texts[0], Answer: texts[1], RowNum: len(pairs)}
+ if rowAligned {
+ pair.RowNum = int(matrix[i][1]) - 1
+ if encoded, err := json.Marshal([][]float64{matrix[i]}); err == nil {
+ pair.Positions = encoded
+ }
+ }
+ pairs = append(pairs, pair)
}
- return []qaPair{{Question: texts[0], Answer: texts[1], RowNum: rowNum}}
+ return pairs
}
func init() {
diff --git a/internal/ingestion/component/chunker/qa_spreadsheet_contract_test.go b/internal/ingestion/component/chunker/qa_spreadsheet_contract_test.go
index 4c5161d91f..becbdc7934 100644
--- a/internal/ingestion/component/chunker/qa_spreadsheet_contract_test.go
+++ b/internal/ingestion/component/chunker/qa_spreadsheet_contract_test.go
@@ -46,10 +46,9 @@ func TestCSVTypedRowsFoldMalformedRowIntoPreviousAnswer(t *testing.T) {
"name": "qa.csv",
"file_type": "csv",
"output_format": "json",
- "json": []map[string]any{
- {"doc_type_kwd": "table", "ck_type": "table_header", "cells": []string{"q1", "a1"}},
- {"doc_type_kwd": "text", "ck_type": "table_row", "cells": []string{"question", "", "extra"}},
- },
+ "json": []map[string]any{spreadsheetSegmentItem("Sheet1",
+ []string{"q1", "a1"},
+ [][]string{{"question", "", "extra"}}, 1, 2)},
})
if err != nil {
t.Fatal(err)
diff --git a/internal/ingestion/component/chunker/qa_test.go b/internal/ingestion/component/chunker/qa_test.go
index 472771afd8..468edcf9c3 100644
--- a/internal/ingestion/component/chunker/qa_test.go
+++ b/internal/ingestion/component/chunker/qa_test.go
@@ -17,10 +17,12 @@
package chunker
import (
+ "encoding/json"
"strings"
"testing"
"ragflow/internal/agent/runtime"
+ "ragflow/internal/ingestion/component/schema"
)
func TestQAChunker_Registered(t *testing.T) {
@@ -180,9 +182,9 @@ func TestQAChunker_JSONCSVNameUsesStrictRowShapeWithoutFileType(t *testing.T) {
"name": "questions.csv",
"output_format": "json",
"json": []map[string]any{{
- "doc_type_kwd": "text",
- "ck_type": "table_row",
- "cells": []string{"question", "answer", "unexpected"},
+ "text": "<table><tr><td>question</td><td></td><td>unexpected</td></tr></table>",
+ "doc_type_kwd": "table",
+ "ck_type": "table",
}},
}
out, err := comp.Invoke(t.Context(), nil, inputs)
@@ -419,7 +421,7 @@ func TestQAChunker_XLSXJSONRegression(t *testing.T) {
}
}
-func TestQAChunkerSpreadsheetRowIRTreatsFirstRowAsQAData(t *testing.T) {
+func TestQAChunkerSpreadsheetWireTreatsFirstRowAsQAData(t *testing.T) {
comp, err := NewQAChunker(map[string]any{"lang": "english"})
if err != nil {
t.Fatal(err)
@@ -429,8 +431,7 @@ func TestQAChunkerSpreadsheetRowIRTreatsFirstRowAsQAData(t *testing.T) {
"file_type": "xlsx",
"output_format": "json",
"json": []map[string]any{
- {"text": "ID; Status", "doc_type_kwd": "table", "ck_type": "table_header", "cells": []string{"ID", "Status"}},
- {"text": "ID:A-100; Status:paid", "doc_type_kwd": "text", "ck_type": "table_row", "cells": []string{"A-100", "paid"}},
+ spreadsheetSegmentItem("Sheet1", []string{"ID", "Status"}, [][]string{{"A-100", "paid"}}, 1, 2),
},
})
if err != nil {
@@ -446,4 +447,116 @@ func TestQAChunkerSpreadsheetRowIRTreatsFirstRowAsQAData(t *testing.T) {
if got, _ := chunks[1]["text"].(string); got != "Question: A-100\tAnswer: paid" {
t.Fatalf("second-row QA = %q", got)
}
+ // top_int keeps the legacy 0-based record index: rowStart - 1.
+ for i, want := range []any{float64(0), float64(1)} {
+ top, _ := chunks[i]["top_int"].([]any)
+ if len(top) != 1 || top[0] != want {
+ t.Errorf("chunk[%d] top_int = %v, want [%v]", i, chunks[i]["top_int"], want)
+ }
+ }
+ // R1: each pair carries only its own row's tuple, not the segment matrix.
+ for i, wantRow := range []float64{1, 2} {
+ raw, _ := json.Marshal(chunks[i]["positions"])
+ var matrix [][]float64
+ if err := json.Unmarshal(raw, &matrix); err != nil {
+ t.Fatalf("chunk[%d] positions = %v: %v", i, chunks[i]["positions"], err)
+ }
+ if len(matrix) != 1 {
+ t.Fatalf("chunk[%d] positions = %v, want one tuple", i, chunks[i]["positions"])
+ }
+ if tuple := matrix[0]; len(tuple) != 5 || tuple[1] != wantRow {
+ t.Errorf("chunk[%d] tuple = %v, want rowStart %v", i, tuple, wantRow)
+ }
+ }
+}
+
+// TestQAChunker_PlainTablePayloadNotSwallowed: a doc_type:"table" item whose
+// payload is plain delimited text (the shape of a PDF table item) used to
+// enter the HTML table extractor, find no <tr>, and lose every pair silently.
+// Routing on what the row walker can read sends it to the text extractor
+// instead.
+func TestQAChunker_PlainTablePayloadNotSwallowed(t *testing.T) {
+ items := []schema.ChunkDoc{
+ {Text: "what is it\tit is a thing\nwho comes\tbob", DocType: "table"},
+ }
+ pairs := extractQAJSON(items, "")
+ if len(pairs) != 2 {
+ t.Fatalf("got %d pairs, want 2 (plain payload under a table label must still yield pairs)", len(pairs))
+ }
+ if pairs[0].Question != "what is it" || pairs[0].Answer != "it is a thing" {
+ t.Errorf("pair0 = %q/%q", pairs[0].Question, pairs[0].Answer)
+ }
+}
+
+// TestQAChunker_TextLabelledHTMLTableReadAsTable documents the accepted
+// non-monotonic side of payload routing: a text-labelled block that
+// genuinely holds table markup is now read as a table.
+func TestQAChunker_TextLabelledHTMLTableReadAsTable(t *testing.T) {
+ items := []schema.ChunkDoc{
+ {Text: "<table><tr><td>q1</td><td>a1</td></tr><tr><td>q2</td><td>a2</td></tr></table>", DocType: "text"},
+ }
+ pairs := extractQAJSON(items, "")
+ if len(pairs) != 2 {
+ t.Fatalf("got %d pairs, want 2", len(pairs))
+ }
+ if pairs[1].Question != "q2" || pairs[1].Answer != "a2" {
+ t.Errorf("pair1 = %q/%q", pairs[1].Question, pairs[1].Answer)
+ }
+}
+
+// TestQAChunker_RowlessTableTextStaysProse: text that merely opens with
+// "<table" — a table element without rows, or a "<tableau>" token — must
+// stay on the prose path. The table extractor would find nothing to pair
+// there, so routing it by prefix alone would silently turn readable text
+// into zero pairs.
+func TestQAChunker_RowlessTableTextStaysProse(t *testing.T) {
+ items := []schema.ChunkDoc{
+ {Text: "<table><caption>schema</caption></table>\nwhat is it\tit is a thing", DocType: "table"},
+ {Text: "<tableau>\nwhat is it\tit is a thing", DocType: "table"},
+ }
+ pairs := extractQAJSON(items, "")
+ if len(pairs) != 2 {
+ t.Fatalf("got %d pairs, want one from each item via the prose path", len(pairs))
+ }
+ for i, pair := range pairs {
+ if pair.Question != "what is it" || pair.Answer != "it is a thing" {
+ t.Errorf("pair%d = %q/%q", i, pair.Question, pair.Answer)
+ }
+ }
+}
+
+// TestQAChunkerPositionsRequireSpreadsheetIdentityAndFiveFields: a positions
+// matrix is read as per-row spreadsheet tuples only when the item carries
+// spreadsheet identity and every tuple has the five wire fields. A short tuple
+// must not panic the pair builder, and a matrix without identity must not move
+// the row numbers (PDF items write layout boxes into the same field).
+func TestQAChunkerPositionsRequireSpreadsheetIdentityAndFiveFields(t *testing.T) {
+ table := "<table><tr><th>q</th><th>a</th></tr></table>"
+ cases := []struct {
+ name string
+ positions string
+ sheetIndex *int
+ wantRowNum int
+ }{
+ {"aligned sheet matrix", `[[1,5,5,1,2]]`, intPtr(1), 4},
+ {"short tuple", `[[1]]`, intPtr(1), 0},
+ {"sheet matrix without identity", `[[1,5,5,1,2]]`, nil, 0},
+ }
+ for _, tc := range cases {
+ t.Run(tc.name, func(t *testing.T) {
+ item := schema.ChunkDoc{
+ Text: table,
+ DocType: "table",
+ Positions: json.RawMessage(tc.positions),
+ SheetIndex: tc.sheetIndex,
+ }
+ pairs := extractQAJSON([]schema.ChunkDoc{item}, "")
+ if len(pairs) != 1 {
+ t.Fatalf("got %d pairs, want 1", len(pairs))
+ }
+ if pairs[0].RowNum != tc.wantRowNum {
+ t.Errorf("RowNum = %d, want %d", pairs[0].RowNum, tc.wantRowNum)
+ }
+ })
+ }
}
diff --git a/internal/ingestion/component/chunker/table.go b/internal/ingestion/component/chunker/table.go
index 0dc53979fb..dc405675bb 100644
--- a/internal/ingestion/component/chunker/table.go
+++ b/internal/ingestion/component/chunker/table.go
@@ -20,16 +20,19 @@
//
// Unlike TokenChunker (which token-shreds a row's text into multiple
// pieces) or OneChunker (which merges many rows into a single chunk),
-// TableChunker keeps the row as the unit of chunking. The table parser is
-// responsible for producing one structured record (ChunkDoc) per row with
-// the formatted "- field: value" content and any column-role / field_map
-// metadata; this chunker passes each record through unchanged, one chunk
-// per row.
+// TableChunker keeps the row as the unit of chunking. Every table arrives as
+// rendered HTML — the spreadsheet parsers emit captioned table segments — and
+// each data row becomes one chunk whose text repeats the column names in the
+// Python "- field: value" line format, carrying its own position tuple when
+// the item is a spreadsheet item with a row-aligned matrix. A table whose only
+// row is the header stays one whole chunk.
package chunker
import (
"context"
+ "encoding/json"
"fmt"
+ "strings"
"ragflow/internal/agent/runtime"
"ragflow/internal/ingestion/component/schema"
@@ -111,7 +114,11 @@ func (c *TableChunkerComponent) invoke(_ context.Context, inputs map[string]any)
}
// tableItems returns the per-row records, preferring JSONResult and
-// falling back to Chunks. Each record becomes exactly one chunk.
+// falling back to Chunks. Each HTML table row becomes exactly one chunk;
+// every other payload record passes through as one chunk — including
+// pre-upgrade row-IR records (ck_type: table_row/table_header with cells),
+// which hold no markup and therefore keep no per-row positions; documents
+// from before this wire must be re-parsed rather than re-chunked.
func tableItems(items, chunks []schema.ChunkDoc) []schema.ChunkDoc {
source := items
if len(source) == 0 {
@@ -120,42 +127,89 @@ func tableItems(items, chunks []schema.ChunkDoc) []schema.ChunkDoc {
if len(source) == 0 {
return nil
}
- dataTables := make(map[string]struct{})
- for _, item := range source {
- if item.CKType == "table_row" {
- dataTables[spreadsheetTableKey(item)] = struct{}{}
- }
- }
filtered := make([]schema.ChunkDoc, 0, len(source))
for _, item := range source {
- // Spreadsheet parsers expose the header as schema metadata. It is
- // already represented in each table_row and must not become a data
- // chunk of its own. A table with no data rows still needs its header
- // as the only searchable representation.
- if item.CKType == "table_header" {
- if _, hasRows := dataTables[spreadsheetTableKey(item)]; hasRows {
- continue
- }
- }
- filtered = append(filtered, item)
+ filtered = append(filtered, expandHTMLTableRows(item)...)
}
return filtered
}
-func spreadsheetTableKey(item schema.ChunkDoc) string {
- if item.TableID != "" {
- return "table:" + item.TableID
+// expandHTMLTableRows turns one HTML <table> payload into one chunk per data
+// row. Non-table payloads pass through unchanged. A table whose only row is
+// the header keeps the whole markup as its single chunk: the header line is
+// then the only searchable representation.
+func expandHTMLTableRows(item schema.ChunkDoc) []schema.ChunkDoc {
+ if !isTableHTML(item.Text) {
+ return []schema.ChunkDoc{item}
}
- if item.SheetIndex != nil {
- return fmt.Sprintf("sheet-index:%d", *item.SheetIndex)
+ rows, headerCount := tableRowsWithHeader(item.Text)
+ if len(rows) <= headerCount {
+ return []schema.ChunkDoc{item}
}
- if item.Sheet != "" {
- return "sheet:" + item.Sheet
+ names := rows[0]
+ // R1: a row chunk must carry only its own tuple. That is only sound when
+ // the item carries spreadsheet identity and its matrix was built
+ // row-aligned (one five-field tuple per <tr>, header included); a
+ // whole-table tuple must not be copied onto every row, and PDF items write
+ // layout boxes into the same field, so misaligned or non-spreadsheet
+ // payloads get no positions at all.
+ var matrix [][]float64
+ aligned := false
+ if item.SheetIndex != nil && len(item.Positions) > 0 {
+ if err := json.Unmarshal(item.Positions, &matrix); err == nil && len(matrix) == len(rows) {
+ aligned = true
+ for _, tuple := range matrix {
+ if len(tuple) != 5 {
+ aligned = false
+ break
+ }
+ }
+ }
}
- if sheet, ok := spreadsheetPositionSheet(item); ok {
- return fmt.Sprintf("position-sheet:%g", sheet)
+ out := make([]schema.ChunkDoc, 0, len(rows)-headerCount)
+ for i, row := range rows[headerCount:] {
+ text := tableRowRecordText(names, row)
+ if text == "" {
+ continue
+ }
+ doc := item
+ doc.Text = text
+ doc.TKNums = nil
+ doc.Positions = nil
+ if aligned {
+ if tuple, err := json.Marshal([][]float64{matrix[headerCount+i]}); err == nil {
+ doc.Positions = tuple
+ }
+ }
+ out = append(out, doc)
+ }
+ if len(out) == 0 {
+ return []schema.ChunkDoc{item}
+ }
+ return out
+}
+
+// tableRowRecordText renders one row in rag/app/table.py's line format:
+// "- {field}: {value}" per non-empty cell, newline joined. A cell whose
+// column has no header name keeps its value alone rather than being dropped.
+func tableRowRecordText(names, cells []string) string {
+ lines := make([]string, 0, len(cells))
+ for j, cell := range cells {
+ value := strings.TrimSpace(cell)
+ if value == "" {
+ continue
+ }
+ name := ""
+ if j < len(names) {
+ name = strings.TrimSpace(names[j])
+ }
+ if name != "" {
+ lines = append(lines, "- "+name+": "+value)
+ continue
+ }
+ lines = append(lines, "- "+value)
}
- return "unknown"
+ return strings.Join(lines, "\n")
}
func init() {
diff --git a/internal/ingestion/component/chunker/table_split.go b/internal/ingestion/component/chunker/table_split.go
new file mode 100644
index 0000000000..0957114cef
--- /dev/null
+++ b/internal/ingestion/component/chunker/table_split.go
@@ -0,0 +1,189 @@
+//
+// Copyright 2026 The InfiniFlow Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+
+package chunker
+
+import (
+ "regexp"
+ "strings"
+)
+
+var trRegex = regexp.MustCompile(`(?is)<tr\b[^>]*>.*?</tr>`)
+
+// tableOpenTagEnd returns the index of the ">" that closes the open tag whose
+// name s begins after, skipping ">" inside quoted attribute values. It returns
+// -1 when the tag never closes before the next "<": a malformed open tag must
+// not be cut at the next ">", which may belong to a closing tag — slicing
+// "<table </table>" there would invert the slice bounds.
+func tableOpenTagEnd(s string) int {
+ quote := byte(0)
+ for i := 0; i < len(s); i++ {
+ switch c := s[i]; {
+ case quote != 0:
+ if c == quote {
+ quote = 0
+ }
+ case c == '"' || c == '\'':
+ quote = c
+ case c == '>':
+ return i
+ case c == '<':
+ return -1
+ }
+ }
+ return -1
+}
+
+// splitLargeHTMLTable splits an HTML table into sub-tables that fit within
+// maxTokens, replicating the caption and header row(s) in each sub-table.
+// Token counting is injected (countTokens) so the caller's tokenizer decides
+// what "size" means. This is the chunker-side size budget: Python's flow path
+// never splits a spreadsheet sheet, but honouring chunk_token_size keeps a
+// wide or long table from losing its tail rows to embedding truncation.
+//
+// The text is expected to hold one outer <table> with flat rows — the shape
+// the spreadsheet parsers render. Markup this function cannot cut safely (an
+// open tag that never closes, a ">" inside an attribute value, a nested or
+// second table) is returned unchanged, so callers fall back to one whole
+// chunk instead of cutting at a guessed boundary.
+//
+// Alongside the parts it returns dataRanges and headerRows: for part i, the
+// half-open range [start, end) of DATA rows (rows below the header in the
+// original table) the part contains, and the number of leading header rows
+// every part replicates. A caller carrying a per-row position matrix slices
+// matrix[:headerRows] + matrix[headerRows+start : headerRows+end] to stay
+// aligned with the sub-table markup. headerRows and ranges are 0/nil when
+// the text is returned unchanged.
+func splitLargeHTMLTable(text string, maxTokens int, countTokens func(string) int) (parts []string, dataRanges [][2]int, headerRows int) {
+ lower := strings.ToLower(text)
+ startIdx := strings.Index(lower, "<table")
+ endIdx := strings.LastIndex(lower, "</table>")
+ if startIdx < 0 || endIdx < 0 || endIdx <= startIdx {
+ return []string{text}, nil, 0
+ }
+
+ prefix := text[:startIdx]
+ afterOpen := text[startIdx:]
+ const openTag = "<table"
+ // The match must be a tag name, not the prefix of a longer one such as
+ // "<tableau>".
+ if len(afterOpen) <= len(openTag) || !strings.ContainsRune(" \t\n\f\r/>", rune(afterOpen[len(openTag)])) {
+ return []string{text}, nil, 0
+ }
+ tagRel := tableOpenTagEnd(afterOpen[len(openTag):])
+ if tagRel < 0 {
+ return []string{text}, nil, 0
+ }
+ tagEnd := len(openTag) + tagRel
+ innerStart := startIdx + tagEnd + 1
+ if innerStart > endIdx {
+ return []string{text}, nil, 0
+ }
+ tableOpenTag := afterOpen[:tagEnd+1]
+ inner := text[innerStart:endIdx]
+ // A second table (or a nested one) has no modelled boundary here; leaving
+ // the text alone keeps the caller on the single-chunk path.
+ if innerLower := strings.ToLower(inner); strings.Contains(innerLower, "<table") || strings.Contains(innerLower, "</table>") {
+ return []string{text}, nil, 0
+ }
+ suffix := text[endIdx+len("</table>"):]
+
+ // Extract caption if present.
+ lowerInner := strings.ToLower(inner)
+ capStart := strings.Index(lowerInner, "<caption")
+ var captionHTML string
+ if capStart >= 0 {
+ capEndRel := strings.Index(lowerInner[capStart:], "</caption>")
+ if capEndRel >= 0 {
+ capEnd := capStart + capEndRel + len("</caption>")
+ captionHTML = inner[capStart:capEnd]
+ }
+ }
+
+ // Extract all rows.
+ rows := trRegex.FindAllString(inner, -1)
+ if len(rows) <= 1 {
+ return []string{text}, nil, 0
+ }
+
+ // Determine header rows vs data rows.
+ headerEnd := 0
+ for i, r := range rows {
+ if strings.Contains(strings.ToLower(r), "<th") {
+ headerEnd = i + 1
+ } else {
+ break
+ }
+ }
+ if headerEnd == 0 || headerEnd >= len(rows) {
+ headerEnd = 1
+ }
+ headerRowHTML := rows[:headerEnd]
+ dataRows := rows[headerEnd:]
+
+ baseHeader := tableOpenTag + captionHTML + strings.Join(headerRowHTML, "")
+ prefixTokens := countTokens(prefix)
+ baseTokens := countTokens(baseHeader + "</table>")
+
+ // prefix (text before the table, e.g. a short Markdown heading) belongs to
+ // the table's position in the document: repeat only the table wrapper,
+ // caption and header rows in every chunk, and prepend prefix to the first.
+ var chunks []string
+ var ranges [][2]int
+ var currentDataRows []string
+ currentTokens := baseTokens + prefixTokens
+ currentStart := 0
+ firstChunk := true
+
+ flush := func(end int) {
+ if len(currentDataRows) == 0 {
+ return
+ }
+ var b strings.Builder
+ if firstChunk {
+ b.WriteString(prefix)
+ firstChunk = false
+ }
+ b.WriteString(baseHeader)
+ for _, dr := range currentDataRows {
+ b.WriteString(dr)
+ }
+ b.WriteString("</table>")
+ chunks = append(chunks, b.String())
+ ranges = append(ranges, [2]int{currentStart, end})
+ currentDataRows = nil
+ currentTokens = baseTokens
+ currentStart = end
+ }
+
+ for i, row := range dataRows {
+ rowTokens := countTokens(row)
+ if len(currentDataRows) > 0 && currentTokens+rowTokens > maxTokens {
+ flush(i)
+ }
+ currentDataRows = append(currentDataRows, row)
+ currentTokens += rowTokens
+ }
+ flush(len(dataRows))
+
+ if len(chunks) <= 1 {
+ return []string{text}, nil, 0
+ }
+ if suffix != "" {
+ chunks[len(chunks)-1] += suffix
+ }
+ return chunks, ranges, headerEnd
+}
diff --git a/internal/ingestion/component/chunker/table_split_test.go b/internal/ingestion/component/chunker/table_split_test.go
new file mode 100644
index 0000000000..652bdba8d4
--- /dev/null
+++ b/internal/ingestion/component/chunker/table_split_test.go
@@ -0,0 +1,165 @@
+//
+// Copyright 2026 The InfiniFlow Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+
+package chunker
+
+import (
+ "strings"
+ "testing"
+)
+
+// charTokens approximates a tokenizer for tests: one token per byte.
+func charTokens(s string) int { return len(s) }
+
+func buildTable(header string, data []string) string {
+ var b strings.Builder
+ b.WriteString("<table><caption>cap</caption>\n")
+ b.WriteString("<tr>" + header + "</tr>\n")
+ for _, row := range data {
+ b.WriteString("<tr>" + row + "</tr>\n")
+ }
+ b.WriteString("</table>\n")
+ return b.String()
+}
+
+func TestSplitLargeHTMLTableNoSplit(t *testing.T) {
+ small := buildTable("<th>a</th><th>b</th>", []string{"<td>1</td><td>2</td>"})
+ parts, ranges, headerRows := splitLargeHTMLTable(small, len(small)+100, charTokens)
+ if len(parts) != 1 || parts[0] != small || ranges != nil || headerRows != 0 {
+ t.Fatalf("small table must return unchanged with nil ranges, got %d parts ranges=%v headerRows=%d", len(parts), ranges, headerRows)
+ }
+ parts, ranges, headerRows = splitLargeHTMLTable("no table here", 1, charTokens)
+ if len(parts) != 1 || parts[0] != "no table here" || ranges != nil || headerRows != 0 {
+ t.Fatalf("non-table text must pass through unchanged")
+ }
+}
+
+func TestSplitLargeHTMLTableReplicatesHeaderAndSlicesRanges(t *testing.T) {
+ data := make([]string, 6)
+ for i := range data {
+ data[i] = "<td>d" + string(rune('0'+i)) + "</td><td>x</td>"
+ }
+ text := buildTable("<th>h1</th><th>h2</th>", data)
+ // Budget that holds header + 2 data rows at most.
+ rowLen := len("<td>d0</td><td>x</td>")
+ budget := len("<table><caption>cap</caption><tr><th>h1</th><th>h2</th></tr></table>") + 2*rowLen
+ parts, ranges, headerRows := splitLargeHTMLTable(text, budget, charTokens)
+ if headerRows != 1 {
+ t.Fatalf("headerRows = %d, want 1 (the single <th> row)", headerRows)
+ }
+ if len(parts) < 3 {
+ t.Fatalf("expected several sub-tables, got %d", len(parts))
+ }
+ if len(ranges) != len(parts) {
+ t.Fatalf("ranges count %d != parts count %d", len(ranges), len(parts))
+ }
+ seen := 0
+ for i, p := range parts {
+ if !strings.HasPrefix(p, "<table><caption>cap</caption>") {
+ t.Errorf("part %d lost its caption: %q", i, p)
+ }
+ if !strings.Contains(p, "<th>h1</th>") {
+ t.Errorf("part %d lost the header row", i)
+ }
+ lo, hi := ranges[i][0], ranges[i][1]
+ if hi <= lo || hi > len(data) {
+ t.Fatalf("part %d range [%d,%d) out of bounds", i, lo, hi)
+ }
+ for r := lo; r < hi; r++ {
+ if !strings.Contains(p, data[r]) {
+ t.Errorf("part %d missing row %d", i, r)
+ }
+ seen++
+ }
+ // Rows outside the range must not leak into this part.
+ for r := 0; r < len(data); r++ {
+ if (r < lo || r >= hi) && strings.Contains(p, data[r]) {
+ t.Errorf("part %d leaked row %d outside [%d,%d)", i, r, lo, hi)
+ }
+ }
+ }
+ if seen != len(data) {
+ t.Errorf("rows covered %d, want %d", seen, len(data))
+ }
+}
+
+func TestSplitLargeHTMLTableHeaderlessReplicatesFirstRow(t *testing.T) {
+ // Original behaviour: with no <th> row the first row is treated as the
+ // repeating header.
+ var b strings.Builder
+ b.WriteString("<table>")
+ b.WriteString("<tr><td>r0</td></tr>")
+ for i := 1; i < 5; i++ {
+ b.WriteString("<tr><td>row" + string(rune('0'+i)) + "</td></tr>")
+ }
+ b.WriteString("</table>")
+ parts, ranges, headerRows := splitLargeHTMLTable(b.String(), len("<table><tr><td>r0</td></tr><tr><td>row1</td></tr></table>")+10, charTokens)
+ if headerRows != 1 {
+ t.Fatalf("headerRows = %d, want 1 (first row treated as header)", headerRows)
+ }
+ if len(parts) < 2 {
+ t.Fatalf("expected split, got %d parts", len(parts))
+ }
+ for i, p := range parts {
+ if !strings.Contains(p, "<td>r0</td>") {
+ t.Errorf("part %d lost replicated first row", i)
+ }
+ }
+ if ranges[0][0] != 0 {
+ t.Errorf("ranges index DATA rows below the treated header, want first range to start at 0, got %v", ranges[0])
+ }
+}
+
+// TestSplitLargeHTMLTableRefusesUncuttableMarkup: an open tag that never
+// closes, a longer tag name, or a nested table is markup this splitter must
+// not cut at a guessed boundary — the text comes back unchanged. The
+// "<table </table>" case used to invert the slice bounds.
+func TestSplitLargeHTMLTableRefusesUncuttableMarkup(t *testing.T) {
+ unchanged := []string{
+ "<table </table>",
+ "<tableau><tr><td>a</td></tr><tr><td>b</td></tr></table>",
+ "<table><tr><td>a</td></tr><table><tr><td>b</td></tr></table></table>",
+ }
+ for _, text := range unchanged {
+ parts, ranges, headerRows := splitLargeHTMLTable(text, 1, charTokens)
+ if len(parts) != 1 || parts[0] != text || ranges != nil || headerRows != 0 {
+ t.Errorf("text %q must pass through unchanged, got %d parts ranges=%v headerRows=%d", text, len(parts), ranges, headerRows)
+ }
+ }
+}
+
+// TestSplitLargeHTMLTableQuotedAngleBracketStaysInOpenTag: a ">" inside a
+// quoted attribute value must not end the open tag — every part repeats the
+// complete tag instead of being cut at the attribute.
+func TestSplitLargeHTMLTableQuotedAngleBracketStaysInOpenTag(t *testing.T) {
+ text := "<table data-meta=\"a>b\">" +
+ "<tr><td>r0</td></tr><tr><td>r1</td></tr><tr><td>r2</td></tr></table>"
+ parts, ranges, headerRows := splitLargeHTMLTable(text, 60, charTokens)
+ if headerRows != 1 {
+ t.Fatalf("headerRows = %d, want 1 (first row treated as header)", headerRows)
+ }
+ if len(parts) < 2 {
+ t.Fatalf("expected a split, got %d parts", len(parts))
+ }
+ if len(ranges) != len(parts) {
+ t.Fatalf("ranges count %d != parts count %d", len(ranges), len(parts))
+ }
+ for i, p := range parts {
+ if !strings.HasPrefix(p, "<table data-meta=\"a>b\">") {
+ t.Errorf("part %d lost the complete open tag: %q", i, p)
+ }
+ }
+}
diff --git a/internal/ingestion/component/chunker/table_test.go b/internal/ingestion/component/chunker/table_test.go
index c5cde7d98f..99e653c4da 100644
--- a/internal/ingestion/component/chunker/table_test.go
+++ b/internal/ingestion/component/chunker/table_test.go
@@ -17,6 +17,8 @@
package chunker
import (
+ "fmt"
+ "strings"
"testing"
)
@@ -80,53 +82,173 @@ func TestTableChunker_EmptyRows(t *testing.T) {
}
}
-func TestTableChunkerSkipsSpreadsheetHeaderRecord(t *testing.T) {
+// TestTableChunkerPreservesHeaderOnlyTable: the header row only becomes a
+// chunk itself when the segment has no data rows — then the whole markup is
+// the table's only searchable representation.
+func TestTableChunkerPreservesHeaderOnlyTable(t *testing.T) {
+ segment := spreadsheetSegmentItem("Sheet1", []string{"ID", "Name"}, nil, 1, 1)
+ chunks := tableChunksOf(t, map[string]any{
+ "name": "empty.xlsx",
+ "output_format": "json",
+ "json": []map[string]any{segment},
+ })
+ if len(chunks) != 1 {
+ t.Fatalf("got %d chunks, want the header-only table preserved", len(chunks))
+ }
+ if chunks[0]["ck_type"] != "table" || chunks[0]["text"] != segment["text"] {
+ t.Fatalf("chunk = %#v, want the header-only segment whole", chunks[0])
+ }
+}
+
+func TestTableChunkerPreservesHeaderOnlySheetAlongsideDataSheet(t *testing.T) {
+ dataSegment := spreadsheetSegmentItem("Sheet1", []string{"ID", "Name"}, [][]string{{"A-1", "paid"}}, 1, 2)
+ headerSegment := spreadsheetSegmentItem("Sheet2", []string{"ID", "Status"}, nil, 2, 1)
+ chunks := tableChunksOf(t, map[string]any{
+ "name": "mixed.xlsx",
+ "output_format": "json",
+ "json": []map[string]any{dataSegment, headerSegment},
+ })
+ if len(chunks) != 2 {
+ t.Fatalf("got %d chunks, want data row plus header-only segment", len(chunks))
+ }
+ // The header row of the data segment never becomes a chunk of its own;
+ // its column names live in the row record text instead.
+ if chunks[0]["text"] != "- ID: A-1\n- Name: paid" {
+ t.Fatalf("chunk0 = %#v", chunks[0])
+ }
+ if chunks[1]["text"] != headerSegment["text"] {
+ t.Fatalf("chunk1 = %#v, want the header-only segment whole", chunks[1])
+ }
+}
+
+// TestTableChunker_ExpandsHTMLRowsWithAlignedPositions covers the P2
+// expander: one HTML <table> item with a row-aligned position matrix (one
+// tuple per <tr>, header included) becomes one chunk per data row, each
+// carrying its own tuple and the Python "- field: value" line format.
+func TestTableChunker_ExpandsHTMLRowsWithAlignedPositions(t *testing.T) {
+ table := "<table><caption>orders</caption>\n" +
+ "<tr><th>ID</th><th>Status</th></tr>\n" +
+ "<tr><td>A-1</td><td>paid</td></tr>\n" +
+ "<tr><td>A-2</td><td></td></tr>\n" +
+ "</table>\n"
chunks := tableChunksOf(t, map[string]any{
"name": "orders.xlsx",
"output_format": "json",
"json": []map[string]any{
- {"text": "ID; Status", "doc_type_kwd": "table", "ck_type": "table_header", "cells": []string{"ID", "Status"}},
- {"text": "A-100; paid", "doc_type_kwd": "table", "ck_type": "table_row", "cells": []string{"A-100", "paid"}},
+ {
+ "text": table,
+ "doc_type_kwd": "table",
+ "ck_type": "table",
+ "sheet_index": 1,
+ "positions": [][]float64{{1, 1, 1, 1, 2}, {1, 2, 2, 1, 2}, {1, 3, 3, 1, 2}},
+ },
},
})
- if len(chunks) != 1 {
- t.Fatalf("got %d chunks, want only the data row", len(chunks))
+ if len(chunks) != 2 {
+ t.Fatalf("got %d chunks, want one per data row", len(chunks))
+ }
+ if chunks[0]["text"] != "- ID: A-1\n- Status: paid" {
+ t.Errorf("chunk0 text = %v", chunks[0]["text"])
}
- if chunks[0]["ck_type"] != "table_row" || chunks[0]["text"] != "A-100; paid" {
- t.Fatalf("chunks = %#v, want the table row only", chunks)
+ // The empty cell is skipped, exactly like rag/app/table.py:656.
+ if chunks[1]["text"] != "- ID: A-2" {
+ t.Errorf("chunk1 text = %v", chunks[1]["text"])
+ }
+ if got := fmt.Sprintf("%v", chunks[0]["positions"]); !strings.Contains(got, "2") || strings.Contains(got, "3") {
+ t.Errorf("chunk0 positions should hold only its own tuple, got %v", chunks[0]["positions"])
+ }
+ if got := fmt.Sprintf("%v", chunks[1]["positions"]); !strings.Contains(got, "3") {
+ t.Errorf("chunk1 positions should hold its own tuple, got %v", chunks[1]["positions"])
}
}
-func TestTableChunkerPreservesHeaderOnlyTable(t *testing.T) {
+// TestTableChunker_WholeTableTupleNotCopiedToRows guards R1: when the
+// position matrix is not row-aligned (a single whole-table tuple), row
+// chunks must not inherit it — every row would otherwise locate to the same
+// first coordinate. Row chunks get no positions instead.
+func TestTableChunker_WholeTableTupleNotCopiedToRows(t *testing.T) {
+ table := "<table><tr><th>ID</th></tr><tr><td>a</td></tr><tr><td>b</td></tr></table>"
chunks := tableChunksOf(t, map[string]any{
- "name": "empty.xlsx",
+ "name": "sheet.csv",
"output_format": "json",
"json": []map[string]any{
- {"text": "ID; Name", "doc_type_kwd": "table", "ck_type": "table_header", "table_id": "sheet-1", "sheet_index": 1},
+ {
+ "text": table,
+ "doc_type_kwd": "table",
+ "sheet_index": 1,
+ "positions": [][]float64{{1, 1, 2, 1, 1}},
+ },
},
})
- if len(chunks) != 1 {
- t.Fatalf("got %d chunks, want the header-only table preserved", len(chunks))
+ if len(chunks) != 2 {
+ t.Fatalf("got %d chunks, want 2", len(chunks))
}
- if chunks[0]["ck_type"] != "table_header" || chunks[0]["text"] != "ID; Name" {
- t.Fatalf("chunk = %#v, want the header record", chunks[0])
+ for i, c := range chunks {
+ if p, ok := c["positions"]; ok && p != nil && fmt.Sprintf("%v", p) != "[]" && fmt.Sprintf("%v", p) != "<nil>" {
+ t.Errorf("chunk %d leaked whole-table positions: %v", i, p)
+ }
}
}
-func TestTableChunkerPreservesHeaderOnlySheetAlongsideDataSheet(t *testing.T) {
+// TestTableChunker_NonSpreadsheetPositionsNotAttached: a markup item without
+// spreadsheet identity must not have its positions read as per-row tuples —
+// PDF items write layout boxes into the same field, so a five-field matrix
+// that happens to align must still be ignored.
+func TestTableChunker_NonSpreadsheetPositionsNotAttached(t *testing.T) {
+ table := "<table><tr><th>ID</th></tr><tr><td>a</td></tr><tr><td>b</td></tr></table>"
chunks := tableChunksOf(t, map[string]any{
- "name": "mixed.xlsx",
+ "name": "document.docx",
"output_format": "json",
"json": []map[string]any{
- {"text": "ID; Name", "doc_type_kwd": "table", "ck_type": "table_header", "table_id": "sheet-1", "sheet_index": 1},
- {"text": "ID; Status", "doc_type_kwd": "table", "ck_type": "table_header", "table_id": "sheet-2", "sheet_index": 2},
- {"text": "A-1; paid", "doc_type_kwd": "text", "ck_type": "table_row", "table_id": "sheet-2", "sheet_index": 2},
+ {
+ "text": table,
+ "doc_type_kwd": "table",
+ "ck_type": "table",
+ "positions": [][]float64{{1, 1, 1, 1, 1}, {1, 2, 2, 1, 1}, {1, 3, 3, 1, 1}},
+ },
},
})
if len(chunks) != 2 {
- t.Fatalf("got %d chunks, want header-only sheet plus data row", len(chunks))
+ t.Fatalf("got %d chunks, want one per data row", len(chunks))
+ }
+ for i, c := range chunks {
+ if p, ok := c["positions"]; ok && p != nil && fmt.Sprintf("%v", p) != "[]" && fmt.Sprintf("%v", p) != "<nil>" {
+ t.Errorf("chunk %d kept positions without spreadsheet identity: %v", i, p)
+ }
+ }
+}
+
+// TestTableChunker_HeaderOnlyHTMLTableKeepsWholePayload: a table whose only
+// row is the header has nothing to expand; the whole markup stays one chunk.
+func TestTableChunker_HeaderOnlyHTMLTableKeepsWholePayload(t *testing.T) {
+ table := "<table><caption>t</caption>\n<tr><th>A</th><th>B</th></tr>\n</table>\n"
+ chunks := tableChunksOf(t, map[string]any{
+ "name": "template.xlsx",
+ "output_format": "json",
+ "json": []map[string]any{
+ {"text": table, "doc_type_kwd": "table", "ck_type": "table"},
+ },
+ })
+ if len(chunks) != 1 || chunks[0]["text"] != table {
+ t.Fatalf("header-only table must stay a single whole-payload chunk, got %#v", chunks)
+ }
+}
+
+// TestTableChunker_HeaderlessHTMLFirstRowIsHeader: with no <th> row the
+// first row names the columns, matching SplitLargeHTMLTable's convention.
+func TestTableChunker_HeaderlessHTMLFirstRowIsHeader(t *testing.T) {
+ table := "<table><tr><td>k</td><td>v</td></tr><tr><td>a</td><td>b</td></tr></table>"
+ chunks := tableChunksOf(t, map[string]any{
+ "name": "kv.csv",
+ "output_format": "json",
+ "json": []map[string]any{
+ {"text": table, "doc_type_kwd": "table"},
+ },
+ })
+ if len(chunks) != 1 {
+ t.Fatalf("got %d chunks, want the single data row", len(chunks))
}
- if chunks[0]["ck_type"] != "table_header" || chunks[1]["ck_type"] != "table_row" {
- t.Fatalf("chunks = %#v, want header then row", chunks)
+ if chunks[0]["text"] != "- k: a\n- v: b" {
+ t.Errorf("text = %v", chunks[0]["text"])
}
}
diff --git a/internal/ingestion/component/chunker/table_wire_invariant_test.go b/internal/ingestion/component/chunker/table_wire_invariant_test.go
new file mode 100644
index 0000000000..7502bf0557
--- /dev/null
+++ b/internal/ingestion/component/chunker/table_wire_invariant_test.go
@@ -0,0 +1,68 @@
+package chunker
+
+import (
+ "testing"
+
+ "ragflow/internal/agent/runtime"
+)
+
+// TestTableWireHasNoRowIRChunkTypes is the cross-chunker guard of the wire
+// contract: the row IR (ck_type table_header/table_row) was deleted, so no
+// chunker may ever emit those values again. Every registered ingestion
+// chunker is enumerated rather than a hand-picked list, so a new chunker that
+// reintroduces the row IR fails here.
+func TestTableWireHasNoRowIRChunkTypes(t *testing.T) {
+ inputs := map[string]any{
+ "name": "orders.xlsx",
+ "file_type": "xlsx",
+ "output_format": "json",
+ "json": []map[string]any{
+ spreadsheetSegmentItem("Sheet1", []string{"ID", "Status"}, [][]string{{"A-1", "paid"}, {"A-2", "open"}}, 1, 2),
+ },
+ }
+ tested := 0
+ for _, name := range runtime.DefaultRegistry.NamesByCategory(runtime.CategoryIngestion) {
+ factory, _, metadata, ok := runtime.DefaultRegistry.Lookup(name)
+ if !ok || factory == nil {
+ continue
+ }
+ if _, isChunker := metadata.Outputs["chunks"]; !isChunker {
+ continue
+ }
+ component, err := factory(name, nil)
+ if err != nil {
+ // Some chunkers (e.g. GroupTitle) declare required parameters;
+ // without them there is no component to exercise.
+ t.Logf("skipping %s: %v", name, err)
+ continue
+ }
+ out, err := component.Invoke(t.Context(), nil, inputs)
+ if err != nil {
+ t.Logf("skipping %s: %v", name, err)
+ continue
+ }
+ if out["_ERROR"] != nil {
+ t.Logf("skipping %s: %v", name, out["_ERROR"])
+ continue
+ }
+ chunks, ok := out["chunks"].([]map[string]any)
+ if !ok {
+ t.Errorf("%s returned chunks of type %T, want []map[string]any", name, out["chunks"])
+ continue
+ }
+ if len(chunks) == 0 {
+ t.Logf("skipping %s: no chunks for the fixture", name)
+ continue
+ }
+ for i, chunk := range chunks {
+ kind, _ := chunk["ck_type"].(string)
+ if kind == "table_header" || kind == "table_row" {
+ t.Errorf("%s chunk[%d] ck_type = %q: the row IR must not come back", name, i, kind)
+ }
+ }
+ tested++
+ }
+ if tested < 4 {
+ t.Fatalf("exercised %d chunkers, want the registered chunker set", tested)
+ }
+}
diff --git a/internal/ingestion/component/chunker/token.go b/internal/ingestion/component/chunker/token.go
index e58daa9511..a15703104d 100644
--- a/internal/ingestion/component/chunker/token.go
+++ b/internal/ingestion/component/chunker/token.go
@@ -550,12 +550,6 @@ func (c *TokenChunkerComponent) invokeJSONPayload(ctx context.Context, items []s
}
lanes := partition(len(items), workers)
perItem := make([][]schema.ChunkDoc, len(items))
- dataTables := make(map[string]struct{})
- for _, item := range items {
- if item.CKType == "table_row" {
- dataTables[spreadsheetTableKey(item)] = struct{}{}
- }
- }
// Computed once before the goroutines so the split path can decide whether
// the main delimiter is a custom (backtick) one — those are dropped, bare
@@ -573,15 +567,6 @@ func (c *TokenChunkerComponent) invokeJSONPayload(ctx context.Context, items []s
perItem[i] = nil
continue
}
- if items[i].CKType == "table_header" {
- if _, hasRows := dataTables[spreadsheetTableKey(items[i])]; hasRows {
- // The typed header is metadata carried by every row. It
- // must not become an independent TokenChunker chunk when
- // row IR is consumed by a legacy pipeline.
- perItem[i] = nil
- continue
- }
- }
if isTextParserSentenceFallback(fileType, c.param.Delimiters, items[i]) {
perItem[i] = splitTextParserSentences(items[i])
} else {
@@ -727,14 +712,6 @@ func splitTextParserSentences(item schema.ChunkDoc) []schema.ChunkDoc {
// the chunk reproduces the source exactly, but a custom (backtick) delimiter
// is dropped (Python-compatible split instruction).
func chunkFromItem(it schema.ChunkDoc, delimPattern *regexp.Regexp, keepDelim bool) []schema.ChunkDoc {
- if it.CKType == "table_row" {
- row := cloneChunkDoc(it)
- row.DocType = "text"
- row.CKType = "table_row"
- row.Text = itemTextOrFallback(it)
- row.TKNums = intPtr(tokenizeStr(row.Text))
- return []schema.ChunkDoc{row}
- }
ckType := itemDocType(it)
txt := itemTextOrFallback(it)
if ckType != "text" {
diff --git a/internal/ingestion/component/chunker/token_pdfpos_test.go b/internal/ingestion/component/chunker/token_pdfpos_test.go
index c07a8134eb..9bde5d471b 100644
--- a/internal/ingestion/component/chunker/token_pdfpos_test.go
+++ b/internal/ingestion/component/chunker/token_pdfpos_test.go
@@ -93,7 +93,7 @@ func TestCloneChunkDoc_DeepCopiesPDFPositions(t *testing.T) {
// TestMergeByTokenSizeFromJSON_PositionsDecodeToMatrix verifies the
// chunker-side contract for diff 1.4: preserved `positions` must decode
// (via ChunkDoc.ToMap → decodeStructuredValue) to a [][]float64 matrix so
-// the downstream task-layer processChunkPositions → AddPositions can
+// the downstream task-layer processChunkPositions → addPDFPositions can
// convert it to page_num_int / top_int / position_int. The coordinate
// conversion itself lives in internal/ingestion/task (processChunkPositions),
// not in the chunker.
diff --git a/internal/ingestion/component/chunker/token_test.go b/internal/ingestion/component/chunker/token_test.go
index 12c6827464..b7cb1cce7a 100644
--- a/internal/ingestion/component/chunker/token_test.go
+++ b/internal/ingestion/component/chunker/token_test.go
@@ -70,21 +70,18 @@ func TestTokenChunker_InvokeEmptyInput(t *testing.T) {
}
}
-func TestTokenChunkerPreservesSpreadsheetRowBoundaries(t *testing.T) {
+func TestTokenChunkerPreservesSpreadsheetSegmentBoundaries(t *testing.T) {
c, err := NewTokenChunker(map[string]any{"chunk_token_size": 512})
if err != nil {
t.Fatalf("NewTokenChunker: %v", err)
}
+ segOne := spreadsheetSegmentItem("Sheet1", []string{"ID", "Status"}, [][]string{{"A-1", "paid"}}, 1, 2)
+ segTwo := spreadsheetSegmentItem("Sheet2", []string{"ID", "Status"}, [][]string{{"B-1", "open"}}, 2, 2)
out, err := c.Invoke(context.Background(), nil, map[string]any{
"name": "orders.xlsx",
"file_type": "xlsx",
"output_format": "json",
- "json": []map[string]any{
- {"text": "ID; Status", "doc_type_kwd": "table", "ck_type": "table_header", "sheet_index": 1},
- {"text": "ID: A-1; Status: paid", "doc_type_kwd": "text", "ck_type": "table_row", "sheet_index": 1},
- {"text": "ID; Status", "doc_type_kwd": "table", "ck_type": "table_header", "sheet_index": 2},
- {"text": "ID: B-1; Status: open", "doc_type_kwd": "text", "ck_type": "table_row", "sheet_index": 2},
- },
+ "json": []map[string]any{segOne, segTwo},
})
if err != nil {
t.Fatalf("Invoke: %v", err)
@@ -94,14 +91,14 @@ func TestTokenChunkerPreservesSpreadsheetRowBoundaries(t *testing.T) {
t.Fatalf("chunks = %T, want []map[string]any", out["chunks"])
}
if len(chunks) != 2 {
- t.Fatalf("chunks = %#v, want one row chunk per sheet", chunks)
+ t.Fatalf("chunks = %#v, want one chunk per segment", chunks)
}
- if chunks[0]["text"] != "ID: A-1; Status: paid" || chunks[1]["text"] != "ID: B-1; Status: open" {
- t.Fatalf("row chunks = %#v", chunks)
+ if chunks[0]["text"] != segOne["text"] || chunks[1]["text"] != segTwo["text"] {
+ t.Fatalf("segment chunks = %#v", chunks)
}
for i, chunk := range chunks {
- if chunk["ck_type"] != "table_row" {
- t.Errorf("chunk[%d] ck_type = %v, want table_row", i, chunk["ck_type"])
+ if chunk["ck_type"] != "table" {
+ t.Errorf("chunk[%d] ck_type = %v, want table", i, chunk["ck_type"])
}
}
}
@@ -111,25 +108,23 @@ func TestTokenChunkerPreservesHeaderOnlySheetAlongsideDataSheet(t *testing.T) {
if err != nil {
t.Fatalf("NewTokenChunker: %v", err)
}
+ segData := spreadsheetSegmentItem("Sheet1", []string{"ID", "Status"}, [][]string{{"A-1", "paid"}}, 1, 2)
+ segHeaderOnly := spreadsheetSegmentItem("Sheet2", []string{"Name", "Owner"}, nil, 2, 1)
out, err := c.Invoke(t.Context(), nil, map[string]any{
"name": "orders.xlsx",
"file_type": "xlsx",
"output_format": "json",
- "json": []map[string]any{
- {"text": "ID; Status", "doc_type_kwd": "table", "ck_type": "table_header", "table_id": "sheet-1", "sheet_index": 1},
- {"text": "ID: A-1; Status: paid", "doc_type_kwd": "text", "ck_type": "table_row", "table_id": "sheet-1", "sheet_index": 1},
- {"text": "Name; Owner", "doc_type_kwd": "table", "ck_type": "table_header", "table_id": "sheet-2", "sheet_index": 2},
- },
+ "json": []map[string]any{segData, segHeaderOnly},
})
if err != nil {
t.Fatalf("Invoke: %v", err)
}
chunks, _ := out["chunks"].([]map[string]any)
if len(chunks) != 2 {
- t.Fatalf("chunks = %#v, want data row and header-only sheet", chunks)
+ t.Fatalf("chunks = %#v, want data segment and header-only segment", chunks)
}
- if chunks[0]["text"] != "ID: A-1; Status: paid" || chunks[1]["text"] != "Name; Owner" {
- t.Fatalf("chunks = %#v, want row followed by header-only sheet", chunks)
+ if chunks[0]["text"] != segData["text"] || chunks[1]["text"] != segHeaderOnly["text"] {
+ t.Fatalf("chunks = %#v, want data segment followed by header-only segment", chunks)
}
}
diff --git a/internal/ingestion/component/chunker/xlsx_qa_regression_test.go b/internal/ingestion/component/chunker/xlsx_qa_regression_test.go
index 2434affcd9..719e6c7a02 100644
--- a/internal/ingestion/component/chunker/xlsx_qa_regression_test.go
+++ b/internal/ingestion/component/chunker/xlsx_qa_regression_test.go
@@ -66,8 +66,8 @@ func qaChunksFromXLSX(t *testing.T, data []byte) []map[string]any {
}
// TestXLSXQAFirstRowIsData protects the QA spreadsheet contract: workbooks
-// contain question/answer rows without a header, so the parser's structural
-// table_header item is still the first QA pair.
+// contain question/answer rows without a header, so the leading header row
+// of the segment (the first <tr>) is still the first QA pair.
func TestXLSXQAFirstRowIsData(t *testing.T) {
chunks := qaChunksFromXLSX(t, xlsxWorkbook(t, [][]string{
{"q1", "a1"},
diff --git a/internal/ingestion/component/schema/chunk_types.go b/internal/ingestion/component/schema/chunk_types.go
index 6648dd6b6d..08ec8e9821 100644
--- a/internal/ingestion/component/schema/chunk_types.go
+++ b/internal/ingestion/component/schema/chunk_types.go
@@ -124,15 +124,8 @@ type ChunkDoc struct {
TopInt []int `json:"top_int,omitempty"`
PDFPositions json.RawMessage `json:"_pdf_positions,omitempty"`
Positions json.RawMessage `json:"positions,omitempty"`
- TableID string `json:"table_id,omitempty"`
Sheet string `json:"sheet,omitempty"`
SheetIndex *int `json:"sheet_index,omitempty"`
- Headers []string `json:"headers,omitempty"`
- Cells []string `json:"cells,omitempty"`
- RowStart *int `json:"row_start,omitempty"`
- RowEnd *int `json:"row_end,omitempty"`
- ColStart *int `json:"col_start,omitempty"`
- ColEnd *int `json:"col_end,omitempty"`
Extra map[string]json.RawMessage `json:"-"`
}
@@ -156,7 +149,7 @@ func (d *ChunkDoc) UnmarshalJSON(data []byte) error {
"context_above", "context_below", "questions", "keywords", "summary",
"chunk_order_int", "title_tks", "title_sm_tks", "content_ltks",
"content_sm_ltks", "tag_kwd", "page_number", "top_int", "_pdf_positions", "positions",
- "table_id", "sheet", "sheet_index", "headers", "cells", "row_start", "row_end", "col_start", "col_end",
+ "sheet", "sheet_index",
} {
delete(raw, key)
}
diff --git a/internal/ingestion/component/schema/schema_test.go b/internal/ingestion/component/schema/schema_test.go
index db1f7ce946..14e5088708 100644
--- a/internal/ingestion/component/schema/schema_test.go
+++ b/internal/ingestion/component/schema/schema_test.go
@@ -237,20 +237,13 @@ func TestChunkerFromUpstreamJSONRoundTrip(t *testing.T) {
}
func TestChunkDocSpreadsheetFieldsRoundTrip(t *testing.T) {
- sheetIndex, rowStart, rowEnd, colStart, colEnd := 2, 42, 42, 1, 3
+ sheetIndex := 2
original := ChunkDoc{
- Text: "ID:A-100; Status:paid",
- DocType: "text",
- CKType: "table_row",
- TableID: "sheet-2",
+ Text: "<table><tr><th>ID</th></tr></table>",
+ DocType: "table",
+ CKType: "table",
Sheet: "Orders",
SheetIndex: &sheetIndex,
- Headers: []string{"ID", "Status", "Note"},
- Cells: []string{"A-100", "paid", ""},
- RowStart: &rowStart,
- RowEnd: &rowEnd,
- ColStart: &colStart,
- ColEnd: &colEnd,
}
data, err := json.Marshal(original)
@@ -261,20 +254,29 @@ func TestChunkDocSpreadsheetFieldsRoundTrip(t *testing.T) {
if err := json.Unmarshal(data, &decoded); err != nil {
t.Fatalf("unmarshal: %v", err)
}
- if decoded.TableID != original.TableID || decoded.Sheet != original.Sheet {
- t.Fatalf("spreadsheet identity mismatch: got table=%q sheet=%q", decoded.TableID, decoded.Sheet)
+ if decoded.Sheet != original.Sheet {
+ t.Fatalf("sheet mismatch: got %q", decoded.Sheet)
}
if decoded.SheetIndex == nil || *decoded.SheetIndex != sheetIndex {
t.Fatalf("sheet index mismatch: got %v", decoded.SheetIndex)
}
- if decoded.RowStart == nil || *decoded.RowStart != rowStart || decoded.RowEnd == nil || *decoded.RowEnd != rowEnd {
- t.Fatalf("row range mismatch: start=%v end=%v", decoded.RowStart, decoded.RowEnd)
- }
- if decoded.ColStart == nil || *decoded.ColStart != colStart || decoded.ColEnd == nil || *decoded.ColEnd != colEnd {
- t.Fatalf("column range mismatch: start=%v end=%v", decoded.ColStart, decoded.ColEnd)
+}
+
+// TestChunkDocLegacyRowIRKeysPassThrough: the deleted row-IR keys (table_id,
+// headers, cells and the per-row coordinate fields) are no longer typed
+// fields, but payloads that still carry them must survive a decode/encode
+// round trip through Extra; the index boundary strips them from the store.
+func TestChunkDocLegacyRowIRKeysPassThrough(t *testing.T) {
+ var decoded ChunkDoc
+ raw := `{"text":"row","doc_type_kwd":"text","ck_type":"table_row","table_id":"sheet-2","headers":["ID","Status"],"cells":["A-100","paid"],"row_start":42,"row_end":42,"col_start":1,"col_end":3}`
+ if err := json.Unmarshal([]byte(raw), &decoded); err != nil {
+ t.Fatalf("unmarshal: %v", err)
}
- if !reflect.DeepEqual(decoded.Headers, original.Headers) || !reflect.DeepEqual(decoded.Cells, original.Cells) {
- t.Fatalf("cells mismatch: headers=%v cells=%v", decoded.Headers, decoded.Cells)
+ out := decoded.ToMap()
+ for _, key := range []string{"table_id", "headers", "cells", "row_start", "row_end", "col_start", "col_end"} {
+ if _, ok := out[key]; !ok {
+ t.Errorf("legacy key %q lost in round trip: %#v", key, out)
+ }
}
}
@@ -683,5 +685,3 @@ func TestFlattenLegacyParserSetups(t *testing.T) {
})
}
}
-
-func ptrString(s string) *string { return &s }
diff --git a/internal/ingestion/component/tokenizer_unit_test.go b/internal/ingestion/component/tokenizer_unit_test.go
index 9118ddf10b..2fcbd4b473 100644
--- a/internal/ingestion/component/tokenizer_unit_test.go
+++ b/internal/ingestion/component/tokenizer_unit_test.go
@@ -430,7 +430,7 @@ func TestValidateTokenizerOutputs_SymbolOnlyContentLtksIsEmptyFails(t *testing.T
// Tokenizer-(T)1: the tokenizer emits chunks via schema.ChunkDocsToMaps
// (ChunkDoc.ToMap), which must carry the raw `positions` / `_pdf_positions`
// through untouched so the downstream executor stage
-// (internal/ingestion/task processChunkPositions → AddPositions) can convert
+// (internal/ingestion/task processChunkPositions → addPDFPositions) can convert
// them into position_int / page_num_int / top_int exactly once. This does NOT
// require the C++ analyzer pool, so it runs under plain `go test`.
func TestChunkDocsToMaps_PreservesPDFPositions(t *testing.T) {
diff --git a/internal/ingestion/task/indexdoc/position.go b/internal/ingestion/task/indexdoc/position.go
index 26ef0a65df..c68bb8d074 100644
--- a/internal/ingestion/task/indexdoc/position.go
+++ b/internal/ingestion/task/indexdoc/position.go
@@ -16,38 +16,63 @@
package indexdoc
-// AddPositions adds position fields to a chunk map.
-// Input positions is a flat []float64 grouped as [pn, left, right, top, bottom]
-// every 5 elements. pn is ALREADY 1-indexed — the 0→1 conversion happens
-// once, at the parser boundary (normalizePDFPageNumber for the DeepDoc PDF
-// path; TCADP writes 1-indexed directly). This function is a passthrough: it
-// must NOT add +1, otherwise the PDF path (which already normalized) would
-// double-increment page numbers.
-//
-// Mirrors Python: rag.nlp.add_positions() (Python adds +1 because its
-// callers feed 0-indexed values; the Go pipeline normalizes earlier).
-func AddPositions(chunk map[string]any, positions []float64) {
+// positionsToIntMatrix groups a flat position array by five and truncates each
+// value to int — the stored shape of position_int. It returns nil for an empty
+// array or one whose length is not a multiple of five: malformed input is
+// dropped, never reinterpreted.
+func positionsToIntMatrix(positions []float64) [][]int {
if len(positions) == 0 || len(positions)%5 != 0 {
- return
+ return nil
}
- n := len(positions) / 5
- pageNumInt := make([]int, 0, n)
- topInt := make([]int, 0, n)
- positionInt := make([][]int, 0, n)
-
+ matrix := make([][]int, 0, len(positions)/5)
for i := 0; i < len(positions); i += 5 {
- pn := int(positions[i]) // already 1-indexed
- left := int(positions[i+1])
- right := int(positions[i+2])
- top := int(positions[i+3])
- bottom := int(positions[i+4])
-
- pageNumInt = append(pageNumInt, pn)
- topInt = append(topInt, top)
- positionInt = append(positionInt, []int{pn, left, right, top, bottom})
+ matrix = append(matrix, []int{
+ int(positions[i]), int(positions[i+1]), int(positions[i+2]),
+ int(positions[i+3]), int(positions[i+4]),
+ })
}
+ return matrix
+}
+// addPDFPositions stores PDF layout boxes: positions is a flat []float64
+// grouped by five as [page, left, right, top, bottom] and is written to
+// page_num_int / top_int / position_int. Page numbers are ALREADY 1-indexed on
+// entry — the 0→1 conversion happens once, at the parser boundary
+// (normalizePDFPageNumber for the DeepDoc PDF path; TCADP writes 1-indexed
+// directly) — so this is a passthrough and must NOT add +1, or the PDF path
+// would double-increment.
+//
+// Mirrors Python: rag.nlp.add_positions() (Python adds +1 because its callers
+// feed 0-indexed values; the Go pipeline normalizes earlier).
+func addPDFPositions(chunk map[string]any, positions []float64) {
+ matrix := positionsToIntMatrix(positions)
+ if matrix == nil {
+ return
+ }
+ pageNumInt := make([]int, 0, len(matrix))
+ topInt := make([]int, 0, len(matrix))
+ for _, box := range matrix {
+ pageNumInt = append(pageNumInt, box[0])
+ topInt = append(topInt, box[3])
+ }
chunk["page_num_int"] = pageNumInt
chunk["top_int"] = topInt
- chunk["position_int"] = positionInt
+ chunk["position_int"] = matrix
+}
+
+// addSpreadsheetPositions stores spreadsheet cell ranges: positions is a flat
+// []float64 grouped by five as [sheet, rowStart, rowEnd, colStart, colEnd] and
+// is written to position_int only — that field is the document preview's
+// coordinate carrier (the chunk API serves "positions" from it, and the Excel
+// preview reads the five values positionally as sheet / rows / columns).
+//
+// page_num_int and top_int are PDF layout fields: deriving them from these
+// tuples would store a sheet number as a page and a column as a top, and would
+// clobber the row index the QA chunker wrote to top_int (the same field
+// Python's beAdoc/beAdocDocx set from row_num for its spreadsheet chunks,
+// which never carry positions at all).
+func addSpreadsheetPositions(chunk map[string]any, positions []float64) {
+ if matrix := positionsToIntMatrix(positions); matrix != nil {
+ chunk["position_int"] = matrix
+ }
}
diff --git a/internal/ingestion/task/indexdoc/position_test.go b/internal/ingestion/task/indexdoc/position_test.go
index 50464b0d55..551275b4b3 100644
--- a/internal/ingestion/task/indexdoc/position_test.go
+++ b/internal/ingestion/task/indexdoc/position_test.go
@@ -5,7 +5,7 @@ import (
)
// =============================================================================
-// AddPositions
+// addPDFPositions
// Canonical format: [pageNum, left, right, top, bottom] × N
// Contract: input page numbers are ALREADY 1-indexed (the 0→1 conversion
// happens once, at the parser boundary in normalizePDFPageNumber). This
@@ -14,11 +14,11 @@ import (
// double-incremented page number.
// =============================================================================
-func TestAddPositions_Basic(t *testing.T) {
+func TestAddPDFPositions_Basic(t *testing.T) {
chunk := map[string]any{}
// [pn=1 (first page, 1-indexed), left=100, right=50, top=200, bottom=150]
positions := []float64{1, 100, 50, 200, 150}
- AddPositions(chunk, positions)
+ addPDFPositions(chunk, positions)
pageNum, ok := chunk["page_num_int"].([]int)
if !ok || len(pageNum) != 1 || pageNum[0] != 1 {
@@ -37,13 +37,13 @@ func TestAddPositions_Basic(t *testing.T) {
}
}
-func TestAddPositions_MultiplePositions(t *testing.T) {
+func TestAddPDFPositions_MultiplePositions(t *testing.T) {
chunk := map[string]any{}
positions := []float64{
1, 100, 50, 200, 150, // pn=1, left=100, right=50, top=200, bottom=150
2, 200, 60, 300, 250, // pn=2, left=200, right=60, top=300, bottom=250
}
- AddPositions(chunk, positions)
+ addPDFPositions(chunk, positions)
pageNum := chunk["page_num_int"].([]int)
if len(pageNum) != 2 || pageNum[0] != 1 || pageNum[1] != 2 {
@@ -59,36 +59,36 @@ func TestAddPositions_MultiplePositions(t *testing.T) {
}
}
-func TestAddPositions_NilPositions(t *testing.T) {
+func TestAddPDFPositions_NilPositions(t *testing.T) {
chunk := map[string]any{}
- AddPositions(chunk, nil)
+ addPDFPositions(chunk, nil)
if _, exists := chunk["page_num_int"]; exists {
t.Error("page_num_int should not be set for nil positions")
}
}
-func TestAddPositions_EmptyPositions(t *testing.T) {
+func TestAddPDFPositions_EmptyPositions(t *testing.T) {
chunk := map[string]any{}
- AddPositions(chunk, []float64{})
+ addPDFPositions(chunk, []float64{})
if _, exists := chunk["page_num_int"]; exists {
t.Error("page_num_int should not be set for empty positions")
}
}
-func TestAddPositions_PartialPositions(t *testing.T) {
+func TestAddPDFPositions_PartialPositions(t *testing.T) {
chunk := map[string]any{}
positions := []float64{1, 100} // only 2 elements, not a complete position
- AddPositions(chunk, positions)
+ addPDFPositions(chunk, positions)
if _, exists := chunk["page_num_int"]; exists {
t.Error("page_num_int should not be set for partial positions")
}
}
-func TestAddPositions_PassthroughNoOffset(t *testing.T) {
- // page numbers are 1-indexed on entry; AddPositions must not add +1.
+func TestAddPDFPositions_PassthroughNoOffset(t *testing.T) {
+ // page numbers are 1-indexed on entry; addPDFPositions must not add +1.
chunk := map[string]any{}
positions := []float64{6, 100, 50, 200, 150} // pn=6, 1-indexed
- AddPositions(chunk, positions)
+ addPDFPositions(chunk, positions)
pageNum := chunk["page_num_int"].([]int)
if pageNum[0] != 6 {
@@ -99,3 +99,43 @@ func TestAddPositions_PassthroughNoOffset(t *testing.T) {
t.Errorf("position_int[0][0] = %d, want 6 (passthrough, no +1)", position[0][0])
}
}
+
+// =============================================================================
+// addSpreadsheetPositions
+// Vocabulary: [sheet, rowStart, rowEnd, colStart, colEnd] × N
+// Contract: position_int is the preview's coordinate carrier and is written;
+// page_num_int/top_int are PDF layout fields and must NOT be derived from these
+// tuples, so a chunk's own top_int (the QA row index) survives.
+// =============================================================================
+
+func TestAddSpreadsheetPositions_WritesPositionIntOnly(t *testing.T) {
+ chunk := map[string]any{"top_int": []int{41}}
+ addSpreadsheetPositions(chunk, []float64{2, 42, 42, 1, 3})
+
+ matrix, ok := chunk["position_int"].([][]int)
+ if !ok || len(matrix) != 1 || len(matrix[0]) != 5 {
+ t.Fatalf("position_int = %v, want one five-field tuple", chunk["position_int"])
+ }
+ want := []int{2, 42, 42, 1, 3}
+ for i, v := range want {
+ if matrix[0][i] != v {
+ t.Fatalf("position_int[0] = %v, want %v", matrix[0], want)
+ }
+ }
+ if _, exists := chunk["page_num_int"]; exists {
+ t.Errorf("page_num_int must not be derived from a spreadsheet tuple: %v", chunk["page_num_int"])
+ }
+ if top, ok := chunk["top_int"].([]int); !ok || len(top) != 1 || top[0] != 41 {
+ t.Errorf("top_int = %v, want the chunk's own row index [41]", chunk["top_int"])
+ }
+}
+
+func TestAddSpreadsheetPositions_MalformedNoop(t *testing.T) {
+ for _, positions := range [][]float64{nil, {}, {2, 42, 42, 1}} {
+ chunk := map[string]any{}
+ addSpreadsheetPositions(chunk, positions)
+ if _, exists := chunk["position_int"]; exists {
+ t.Errorf("positions %v must be dropped, got %v", positions, chunk["position_int"])
+ }
+ }
+}
diff --git a/internal/ingestion/task/indexdoc/process.go b/internal/ingestion/task/indexdoc/process.go
index 453172dfbd..5e80a9c937 100644
--- a/internal/ingestion/task/indexdoc/process.go
+++ b/internal/ingestion/task/indexdoc/process.go
@@ -94,10 +94,13 @@ func ProcessChunksForPipeline(
}
cleanupConsumedChunkFields(ck)
+ // The spreadsheet identity is keyed off fields the strip is about to
+ // remove, so it must be read before it.
+ spreadsheet := isSpreadsheetChunk(ck)
stripPipelineOnlyFields(ck)
metadata = mergeChunkMetadata(metadata, ck)
RenameTextToContentWithWeight(ck)
- processChunkPositions(ck)
+ processChunkPositions(ck, spreadsheet)
}
return metadata, nil
}
@@ -188,23 +191,31 @@ func mergeChunkMetadata(metadata map[string]any, ck map[string]any) map[string]a
}
// processChunkPositions converts the raw "positions" field into indexable
-// position fields (page_num_int, top_int, position_int) via AddPositions,
-// then removes the raw "positions" and the sibling "_pdf_positions" internal
-// field. _pdf_positions is the parser-emitted position matrix consumed by
-// the chunker's image-crop pass (pdfcrop_cgo.go); after the chunker it is
-// dead weight with no index column, so it is pruned unconditionally —
-// independent of "positions" and not gated on the early-return below.
+// position fields, then removes the raw "positions" and the sibling
+// "_pdf_positions" internal field. _pdf_positions is the parser-emitted
+// position matrix consumed by the chunker's image-crop pass (pdfcrop_cgo.go);
+// after the chunker it is dead weight with no index column, so it is pruned
+// unconditionally — independent of "positions" and not gated on the
+// early-return below.
+//
+// The field carries two vocabularies, and the branch on them is explicit
+// rather than inferred from the shape:
+// - PDF items: [page, left, right, top, bottom] → page_num_int, top_int and
+// position_int (addPDFPositions);
+// - spreadsheet items (identity present): [sheet, rowStart, rowEnd,
+// colStart, colEnd] → position_int only (addSpreadsheetPositions), which
+// leaves the chunk's own top_int — the QA chunker's row index — intact.
//
// Two source types reach this point:
-// - []float64 — flat array of 5-tuples [page,left,right,top,bottom,…] from
-// parsers that emit positions directly as a flat float64 slice.
+// - []float64 — flat array of 5-tuples from parsers that emit positions
+// directly as a flat float64 slice.
// - [][]float64 — the production path: positions flow through ChunkDoc
// (json.RawMessage → decodeStructuredValue) which produces a slice of
// 5-element groups.
//
-// Both are flattened into a single []float64 for AddPositions, which groups
-// by 5 internally. Unexpected types are logged and discarded.
-func processChunkPositions(ck map[string]any) {
+// Both are flattened into a single []float64, which the store helpers group by
+// five. Unexpected types are logged and discarded.
+func processChunkPositions(ck map[string]any, spreadsheet bool) {
delete(ck, "_pdf_positions")
poss, exists := ck["positions"]
if !exists {
@@ -212,19 +223,38 @@ func processChunkPositions(ck map[string]any) {
}
switch v := poss.(type) {
case []float64:
- AddPositions(ck, v)
+ storePositions(ck, v, spreadsheet)
case [][]float64:
flat := make([]float64, 0, len(v)*5)
for _, group := range v {
flat = append(flat, group...)
}
- AddPositions(ck, flat)
+ storePositions(ck, flat, spreadsheet)
default:
common.Warn(fmt.Sprintf("chunk positions unexpected type %T; discarding", poss))
}
delete(ck, "positions")
}
+func storePositions(ck map[string]any, flat []float64, spreadsheet bool) {
+ if spreadsheet {
+ addSpreadsheetPositions(ck, flat)
+ return
+ }
+ addPDFPositions(ck, flat)
+}
+
+// isSpreadsheetChunk reports whether a chunk's "positions" carry the
+// spreadsheet vocabulary, judged by the identity fields the strip removes
+// later in the chunk loop.
+func isSpreadsheetChunk(ck map[string]any) bool {
+ if v, ok := ck["sheet_index"]; ok && v != nil {
+ return true
+ }
+ sheet, ok := ck["sheet"].(string)
+ return ok && sheet != ""
+}
+
// AggregateTableDocMetadata collects unique per-column values across all chunks
// for columns with role "metadata" or "both", merges them into document metadata.
// Mirrors Python: rag/utils/table_es_metadata.py:aggregate_table_doc_metadata
diff --git a/internal/ingestion/task/indexdoc/process_test.go b/internal/ingestion/task/indexdoc/process_test.go
index 4fe492a740..ca453519aa 100644
--- a/internal/ingestion/task/indexdoc/process_test.go
+++ b/internal/ingestion/task/indexdoc/process_test.go
@@ -351,7 +351,7 @@ func TestProcessChunkPositions_FlatFloat64(t *testing.T) {
// positions is 1-indexed (parser normalized before we see it)
"positions": []float64{1, 100, 50, 200, 150},
}
- processChunkPositions(chunk)
+ processChunkPositions(chunk, false)
if _, exists := chunk["positions"]; exists {
t.Fatal("positions key must be removed")
@@ -369,7 +369,7 @@ func TestProcessChunkPositions_2DFloat64(t *testing.T) {
{2, 200, 60, 300, 250},
},
}
- processChunkPositions(chunk)
+ processChunkPositions(chunk, false)
if _, exists := chunk["positions"]; exists {
t.Fatal("positions key must be removed")
@@ -391,7 +391,7 @@ func TestProcessChunkPositions_NoPositions(t *testing.T) {
"text": "hello",
"_pdf_positions": []any{[]any{0, 1, 2, 3, 4}},
}
- processChunkPositions(chunk)
+ processChunkPositions(chunk, false)
if _, exists := chunk["page_num_int"]; exists {
t.Error("page_num_int must not be set when positions is missing")
}
@@ -497,3 +497,63 @@ func TestProcessChunksForPipeline_StripsPipelineOnlyFields(t *testing.T) {
t.Errorf("doc_id = %v, want doc-1 (the strip must not touch persist fields)", ck["doc_id"])
}
}
+
+// =============================================================================
+// processChunkPositions — the two position vocabularies
+// =============================================================================
+
+// TestProcessChunkPositions_SpreadsheetKeepsRowIndex: a spreadsheet chunk's
+// positions become position_int (the preview carrier) instead of being decoded
+// as PDF boxes — no page_num_int, and the chunk's own top_int (the QA row
+// index, aligned with Python's beAdoc) survives.
+func TestProcessChunkPositions_SpreadsheetKeepsRowIndex(t *testing.T) {
+ chunk := map[string]any{
+ "positions": [][]float64{{2, 42, 42, 1, 3}},
+ "top_int": []int{41},
+ }
+ processChunkPositions(chunk, true)
+
+ matrix, ok := chunk["position_int"].([][]int)
+ if !ok || len(matrix) != 1 || matrix[0][0] != 2 || matrix[0][4] != 3 {
+ t.Fatalf("position_int = %v, want the sheet tuple", chunk["position_int"])
+ }
+ if _, exists := chunk["page_num_int"]; exists {
+ t.Errorf("page_num_int must not be derived from spreadsheet tuples: %v", chunk["page_num_int"])
+ }
+ if top, ok := chunk["top_int"].([]int); !ok || len(top) != 1 || top[0] != 41 {
+ t.Errorf("top_int = %v, want the chunk's own row index [41]", chunk["top_int"])
+ }
+ if _, exists := chunk["positions"]; exists {
+ t.Error("raw positions must be removed")
+ }
+}
+
+// TestProcessChunksForPipeline_SpreadsheetPositionsKeepRowIndex pins the whole
+// chunk loop: the spreadsheet identity has to be read before
+// stripPipelineOnlyFields removes sheet_index, or the positions of a
+// spreadsheet chunk would be decoded as PDF layout boxes.
+func TestProcessChunksForPipeline_SpreadsheetPositionsKeepRowIndex(t *testing.T) {
+ chunks := []map[string]any{{
+ "text": "<table><tr><th>q</th><th>a</th></tr></table>",
+ "sheet_index": 1,
+ "top_int": []int{41},
+ "positions": [][]float64{{1, 42, 42, 1, 3}},
+ }}
+ if _, err := ProcessChunksForPipeline(chunks, "doc-1", "doc.xlsx", time.Now()); err != nil {
+ t.Fatalf("ProcessChunksForPipeline: %v", err)
+ }
+ ck := chunks[0]
+ if _, exists := ck["sheet_index"]; exists {
+ t.Error("sheet_index must be stripped at the index boundary")
+ }
+ matrix, ok := ck["position_int"].([][]int)
+ if !ok || len(matrix) != 1 || matrix[0][0] != 1 || matrix[0][1] != 42 {
+ t.Fatalf("position_int = %v, want the sheet tuple", ck["position_int"])
+ }
+ if _, exists := ck["page_num_int"]; exists {
+ t.Errorf("page_num_int must not be derived from spreadsheet tuples: %v", ck["page_num_int"])
+ }
+ if top, ok := ck["top_int"].([]int); !ok || len(top) != 1 || top[0] != 41 {
+ t.Errorf("top_int = %v, want the chunk's own row index [41]", ck["top_int"])
+ }
+}
diff --git a/internal/parser/parser/csv_parser.go b/internal/parser/parser/csv_parser.go
index 156e0ae9d5..28b8bbf02e 100644
--- a/internal/parser/parser/csv_parser.go
+++ b/internal/parser/parser/csv_parser.go
@@ -50,7 +50,6 @@ const (
type CSVParser struct {
ParseMethod string
OutputFormat string
- HTML4Excel bool
TCADPAPIServer string
TCADPAPIKey string
TCADPTableResultType string
@@ -78,9 +77,7 @@ func (p *CSVParser) ConfigureFromSetup(setup map[string]any) {
if v, ok := setup["output_format"].(string); ok && v != "" {
p.OutputFormat = v
}
- if v, ok := setup["html4excel"].(bool); ok {
- p.HTML4Excel = v
- }
+ deprecatedHTML4Excel(setup, p.String())
deprecatedChunkRows(setup, p.String())
if v, ok := setup["tcadp_apiserver"].(string); ok && v != "" {
p.TCADPAPIServer = v
@@ -122,10 +119,6 @@ func (p *CSVParser) ParseWithResult(ctx context.Context, filename string, data [
decoded, encName := DecodeToUTF8(data, "text/csv")
text := string(decoded)
if strings.TrimSpace(text) == "" {
- var emptyJSON []map[string]any
- if p.HTML4Excel {
- emptyJSON = []map[string]any{NewTableJSONItem("<table><caption>Data</caption></table>", csvSheetName, [][]float64{{1, 1, 1, 1, 1}})}
- }
return ParseResult{
OutputFormat: spreadsheetOutputFormat,
File: map[string]any{
@@ -135,7 +128,6 @@ func (p *CSVParser) ParseWithResult(ctx context.Context, filename string, data [
"format": "csv",
"sheets": 1,
},
- JSON: emptyJSON,
}
}
@@ -151,14 +143,9 @@ func (p *CSVParser) ParseWithResult(ctx context.Context, filename string, data [
for i := range dataRows {
dataRows[i] = i + 2
}
- var items []map[string]any
- if p.HTML4Excel {
- if table := recordsToHTMLTableItem(records, csvSheetName, 1, 1, dataRows); table != nil {
- items = []map[string]any{table}
- }
- } else {
- items = recordsToSpreadsheetItems(records, csvSheetName, 1, 1, dataRows)
- }
+ // One wire shape for csv as for xlsx: a single segmented HTML table with
+ // row-aligned positions (csv has no in-cell image anchors).
+ items := buildSheetItems(records, csvSheetName, 1, 1, dataRows, nil)
return ParseResult{
OutputFormat: spreadsheetOutputFormat,
File: map[string]any{
diff --git a/internal/parser/parser/csv_parser_test.go b/internal/parser/parser/csv_parser_test.go
index 8d1a0fc311..4e45e23515 100644
--- a/internal/parser/parser/csv_parser_test.go
+++ b/internal/parser/parser/csv_parser_test.go
@@ -3,6 +3,7 @@ package parser
import (
"context"
"fmt"
+ "reflect"
"strings"
"testing"
)
@@ -21,42 +22,65 @@ func TestCSVParser_EmitsSpreadsheetRows(t *testing.T) {
t.Fatalf("res.OutputFormat = %q, want %q", res.OutputFormat, "json")
}
- if len(res.JSON) != 3 {
- t.Fatalf("len(res.JSON) = %d, want header plus two rows", len(res.JSON))
- }
- if res.JSON[0]["ck_type"] != "table_header" {
- t.Fatalf("header ck_type = %v, want table_header", res.JSON[0]["ck_type"])
- }
- if res.JSON[1]["ck_type"] != "table_row" {
- t.Fatalf("row ck_type = %v, want table_row", res.JSON[1]["ck_type"])
- }
- if res.JSON[1]["text"] != "Name:Alice; Age:30; City:New York ——Data" {
- t.Errorf("row text = %v", res.JSON[1]["text"])
- }
- if res.JSON[1]["sheet"] != "Data" || res.JSON[1]["sheet_index"] != 1 {
- t.Errorf("row sheet metadata = %#v", res.JSON[1])
- }
- positions, ok := res.JSON[1]["positions"].([][]float64)
- if !ok || len(positions) == 0 {
- t.Fatalf("item positions invalid: %v", res.JSON[1]["positions"])
- }
- pos := positions[0]
- // [sheet, rowStart, rowEnd, colStart, colEnd]
- if len(pos) != 5 || pos[0] != 1 || pos[1] != 2 || pos[2] != 2 || pos[3] != 1 || pos[4] != 3 {
- t.Errorf("unexpected positions: %v", pos)
+ // One wire shape: the whole sheet is one segmented HTML table item.
+ if len(res.JSON) != 1 {
+ t.Fatalf("len(res.JSON) = %d, want one HTML table item", len(res.JSON))
+ }
+ item := res.JSON[0]
+ if item["ck_type"] != "table" || item["doc_type_kwd"] != "table" {
+ t.Fatalf("item types = %v/%v, want table/table", item["ck_type"], item["doc_type_kwd"])
+ }
+ wantText := "<table><caption>Data</caption>\n" +
+ "<tr><th>Name</th><th>Age</th><th>City</th></tr>\n" +
+ "<tr><td>Alice</td><td>30</td><td>New York</td></tr>\n" +
+ "<tr><td>Bob</td><td>25</td><td>San Francisco</td></tr>\n" +
+ "</table>\n"
+ if item["text"] != wantText {
+ t.Errorf("text = %q, want %q", item["text"], wantText)
+ }
+ if item["sheet"] != "Data" || item["sheet_index"] != 1 {
+ t.Errorf("sheet metadata = %#v", item)
+ }
+ positions, ok := item["positions"].([][]float64)
+ if !ok {
+ t.Fatalf("item positions invalid: %T", item["positions"])
+ }
+ // Row-aligned matrix: one tuple per <tr> in markup order, header first.
+ if len(positions) != 3 {
+ t.Fatalf("len(positions) = %d, want 3 (one per <tr>)", len(positions))
+ }
+ for i, want := range [][]float64{{1, 1, 1, 1, 3}, {1, 2, 2, 1, 3}, {1, 3, 3, 1, 3}} {
+ for j := range want {
+ if positions[i][j] != want[j] {
+ t.Errorf("positions[%d] = %v, want %v", i, positions[i], want)
+ break
+ }
+ }
}
}
-func TestCSVParserHTML4ExcelRemainsAtomic(t *testing.T) {
+// TestCSVParserHTML4ExcelIsIgnored pins the retirement: html4excel is still
+// accepted at the entry (with a deprecation warning) but selects nothing —
+// the wire is identical to the default path.
+func TestCSVParserHTML4ExcelIsIgnored(t *testing.T) {
+ data := []byte("Question,Answer\nQ1,A1\n")
+ plain := NewCSVParser()
+ plainRes := plain.ParseWithResult(context.Background(), "qa.csv", data)
+ if plainRes.Err != nil {
+ t.Fatalf("ParseWithResult failed: %v", plainRes.Err)
+ }
p := NewCSVParser()
p.ConfigureFromSetup(map[string]any{"html4excel": true})
- res := p.ParseWithResult(context.Background(), "qa.csv", []byte("Question,Answer\nQ1,A1\n"))
+ res := p.ParseWithResult(context.Background(), "qa.csv", data)
if res.Err != nil {
t.Fatalf("ParseWithResult failed: %v", res.Err)
}
if len(res.JSON) != 1 || res.JSON[0]["ck_type"] != "table" {
t.Fatalf("items = %#v, want one table item", res.JSON)
}
+ if !reflect.DeepEqual(res.JSON, plainRes.JSON) {
+ t.Errorf("html4excel changed the wire: %v vs %v", res.JSON, plainRes.JSON)
+ }
}
func TestCSVParser_ConfigureOutputFormatJSON(t *testing.T) {
@@ -74,11 +98,14 @@ func TestCSVParser_ConfigureOutputFormatJSON(t *testing.T) {
if res.OutputFormat != "json" {
t.Fatalf("res.OutputFormat = %q, want 'json'", res.OutputFormat)
}
- if len(res.JSON) != 2 {
- t.Fatalf("len(res.JSON) = %d, want header plus row", len(res.JSON))
+ if len(res.JSON) != 1 {
+ t.Fatalf("len(res.JSON) = %d, want one HTML table item", len(res.JSON))
+ }
+ if res.JSON[0]["ck_type"] != "table" {
+ t.Fatalf("ck_type = %v, want table", res.JSON[0]["ck_type"])
}
- if res.JSON[1]["ck_type"] != "table_row" {
- t.Fatalf("row ck_type = %v, want table_row", res.JSON[1]["ck_type"])
+ if positions := res.JSON[0]["positions"].([][]float64); len(positions) != 2 {
+ t.Fatalf("len(positions) = %d, want one tuple per <tr>", len(positions))
}
}
@@ -110,11 +137,14 @@ func TestCSVParser_PreservesEveryDataRow(t *testing.T) {
t.Fatalf("ParseWithResult failed: %v", res.Err)
}
- if len(res.JSON) != 5 {
- t.Fatalf("len(res.JSON) = %d, want header plus four rows", len(res.JSON))
+ if len(res.JSON) != 1 {
+ t.Fatalf("len(res.JSON) = %d, want one HTML table item", len(res.JSON))
}
- for i, item := range res.JSON[1:] {
- pos := item["positions"].([][]float64)[0]
+ positions := res.JSON[0]["positions"].([][]float64)
+ if len(positions) != 5 {
+ t.Fatalf("len(positions) = %d, want one tuple per <tr> (header plus four rows)", len(positions))
+ }
+ for i, pos := range positions[1:] {
wantRow := float64(i + 2)
if pos[1] != wantRow || pos[2] != wantRow {
t.Errorf("row %d position = [%v, %v], want [%v, %v]", i, pos[1], pos[2], wantRow, wantRow)
@@ -128,14 +158,18 @@ func TestCSVParserKeepsVariableRowWidths(t *testing.T) {
if res.Err != nil {
t.Fatalf("ParseWithResult failed: %v", res.Err)
}
- if len(res.JSON) != 3 {
- t.Fatalf("items = %d, want header plus two rows", len(res.JSON))
+ if len(res.JSON) != 1 {
+ t.Fatalf("items = %d, want one HTML table item", len(res.JSON))
}
- if got := len(res.JSON[1]["cells"].([]string)); got != 2 {
- t.Fatalf("first data row width = %d, want 2", got)
+ text, _ := res.JSON[0]["text"].(string)
+ if !strings.Contains(text, "<tr><td>Q1</td><td>A1</td></tr>") ||
+ !strings.Contains(text, "<tr><td>Q2</td><td>A2</td><td>extra</td></tr>") {
+ t.Fatalf("row widths not preserved in markup: %q", text)
}
- if got := len(res.JSON[2]["cells"].([]string)); got != 3 {
- t.Fatalf("second data row width = %d, want 3", got)
+ positions := res.JSON[0]["positions"].([][]float64)
+ // Per-row column ranges: Q1 row spans cols 1-2, Q2 row spans 1-3.
+ if len(positions) != 3 || positions[1][4] != 2 || positions[2][4] != 3 {
+ t.Fatalf("position matrix lost per-row widths: %v", positions)
}
}
@@ -148,8 +182,13 @@ func TestCSVParser_ReadsTheSeparatorTheFileWasWrittenWith(t *testing.T) {
if res.Err != nil {
t.Fatalf("separator %q: ParseWithResult failed: %v", sep, res.Err)
}
- if got := res.JSON[len(res.JSON)-1]["text"]; got != "Name:Widget; Region:EU; Units:12 ——Data" {
- t.Errorf("separator %q: row text = %v", sep, got)
+ if len(res.JSON) != 1 {
+ t.Fatalf("separator %q: items = %d, want one HTML table item", sep, len(res.JSON))
+ }
+ text, _ := res.JSON[0]["text"].(string)
+ if !strings.Contains(text, "<tr><th>Name</th><th>Region</th><th>Units</th></tr>") ||
+ !strings.Contains(text, "<tr><td>Widget</td><td>EU</td><td>12</td></tr>") {
+ t.Errorf("separator %q: row not split into three columns: %q", sep, text)
}
}
}
@@ -214,7 +253,11 @@ func TestCSVParser_KeepsAnEmptyTabSeparatedField(t *testing.T) {
if res.Err != nil {
t.Fatalf("ParseWithResult failed: %v", res.Err)
}
- if got := res.JSON[len(res.JSON)-1]["text"]; got != "Name:Widget; Units:12 ——Data" {
- t.Errorf("row text = %v", got)
+ if len(res.JSON) != 1 {
+ t.Fatalf("items = %d, want one HTML table item", len(res.JSON))
+ }
+ text, _ := res.JSON[0]["text"].(string)
+ if !strings.Contains(text, "<tr><td>Widget</td><td></td><td>12</td></tr>") {
+ t.Errorf("empty tab-separated field not kept: %q", text)
}
}
diff --git a/internal/parser/parser/office_table_render.go b/internal/parser/parser/office_table_render.go
index 98a86d677c..f8c3db1df5 100644
--- a/internal/parser/parser/office_table_render.go
+++ b/internal/parser/parser/office_table_render.go
@@ -42,100 +42,33 @@ var tableIllegalCharsRe = regexp.MustCompile(`[\x00-\x08]|\x0B|\x0C|[\x0E-\x1F]`
// value parsing.
var numericCellRe = regexp.MustCompile(`^[\$\+\-]?[\d,]+(\.\d+)?%?$`)
-func recordsToSpreadsheetItems(records [][]string, sheet string, sheetIndex, headerRow int, dataRows []int) []map[string]any {
- if len(records) == 0 {
- return nil
- }
- if headerRow <= 0 {
- headerRow = 1
- }
- if len(dataRows) != len(records)-1 {
- dataRows = make([]int, len(records)-1)
- for i := range dataRows {
- dataRows[i] = headerRow + i + 1
- }
- }
- maxCols := 0
- for _, row := range records {
- if len(row) > maxCols {
- maxCols = len(row)
- }
- }
- if maxCols == 0 {
- return nil
- }
- header := append([]string(nil), records[0]...)
- if len(header) == 0 {
- header = padSpreadsheetRow(header, maxCols)
- }
- headerColEnd := len(header)
- tableID := fmt.Sprintf("sheet-%d", sheetIndex)
- items := make([]map[string]any, 0, len(records))
- headerText := spreadsheetRowText(header, nil, sheet)
- items = append(items, map[string]any{
- "text": headerText,
- "doc_type_kwd": DocTypeTable,
- "ck_type": "table_header",
- "table_id": tableID,
- "sheet": sheet,
- "sheet_index": sheetIndex,
- "cells": header,
- "row_start": headerRow,
- "row_end": headerRow,
- "col_start": 1,
- "col_end": headerColEnd,
- "positions": [][]float64{{float64(sheetIndex), float64(headerRow), float64(headerRow), 1, float64(headerColEnd)}},
- })
- for i, sourceRow := range records[1:] {
- cells := append([]string(nil), sourceRow...)
- colStart, colEnd := nonEmptyColumnRange(cells)
- if colStart == 0 {
- continue
- }
- rowNumber := dataRows[i]
- items = append(items, map[string]any{
- "text": spreadsheetRowText(header, cells, sheet),
- "doc_type_kwd": DocTypeText,
- "ck_type": "table_row",
- "table_id": tableID,
- "sheet": sheet,
- "sheet_index": sheetIndex,
- "headers": header,
- "cells": cells,
- "row_start": rowNumber,
- "row_end": rowNumber,
- "col_start": colStart,
- "col_end": colEnd,
- "positions": [][]float64{{float64(sheetIndex), float64(rowNumber), float64(rowNumber), float64(colStart), float64(colEnd)}},
- })
- }
- return items
+// spreadsheetSegmentRow is one emitted data row: its cells for the markup
+// and its row-aligned position tuple.
+type spreadsheetSegmentRow struct {
+ cells []string
+ tuple []float64
+ rowNum int
+ colStart int
}
-func recordsToHTMLTableItem(records [][]string, sheet string, sheetIndex, headerRow int, dataRows []int) map[string]any {
- if len(records) == 0 {
- return nil
- }
- if headerRow <= 0 {
- headerRow = 1
- }
- colEnd := 1
- for _, row := range records {
- if len(row) > colEnd {
- colEnd = len(row)
- }
- }
+// renderTableHTML renders one sheet segment in the spreadsheet wire format: a
+// captioned <table> whose header row is <th> cells and whose data rows are
+// <td> cells, one row per line. It is the byte contract every consumer reads —
+// the chunkers parse this shape back row by row (see
+// internal/ingestion/component/chunker/html_rows.go), and the positions matrix
+// emitted next to it is aligned one tuple per <tr>, header included.
+func renderSpreadsheetTable(sheet string, header []string, rows [][]string) string {
var builder strings.Builder
builder.WriteString("<table><caption>")
builder.WriteString(html.EscapeString(sheet))
builder.WriteString("</caption>\n<tr>")
- for _, cell := range records[0] {
+ for _, cell := range header {
builder.WriteString("<th>")
builder.WriteString(html.EscapeString(strings.TrimSpace(cell)))
builder.WriteString("</th>")
}
builder.WriteString("</tr>\n")
- for _, row := range records[1:] {
+ for _, row := range rows {
builder.WriteString("<tr>")
for _, cell := range row {
builder.WriteString("<td>")
@@ -145,98 +78,172 @@ func recordsToHTMLTableItem(records [][]string, sheet string, sheetIndex, header
builder.WriteString("</tr>\n")
}
builder.WriteString("</table>\n")
- rowStart, rowEnd := headerRow, headerRow
- if len(dataRows) > 0 {
- rowStart, rowEnd = dataRows[0], dataRows[len(dataRows)-1]
- }
- return NewTableJSONItem(builder.String(), sheet, [][]float64{{
- float64(sheetIndex), float64(rowStart), float64(rowEnd), 1, float64(colEnd),
- }})
+ return builder.String()
}
-func padSpreadsheetRow(row []string, width int) []string {
- padded := make([]string, width)
- copy(padded, row)
- return padded
-}
+// buildSheetItems renders one sheet into its wire items: HTML <table>
+// segments split at image anchor boundaries, interleaved with the anchored
+// image items in document order. Every segment repeats the header row (so
+// segment 2+ never loses column names) and carries a row-aligned position
+// matrix — one tuple per <tr> in strict markup order, header included — so
+// row-level consumers index positions by <tr> number with no side channel.
+//
+// An image lands after the last row whose (row, colStart) sorts at or before
+// its anchor; rows before the header's anchor keep the header in front of
+// the image (a header-only lead segment); fully empty rows are skipped from
+// both markup and matrix.
+//
+// Segmentation is semantic only — sheet boundaries and image anchors, never a
+// size budget. The legacy html4excel path pre-cut a fixed 12 rows here, which
+// ignored the user's chunk size; that budget belongs to the chunker (see the
+// chunker's splitSpreadsheetTable).
+//
+// Every tuple is a spreadsheet position, [sheet, rowStart, rowEnd, colStart,
+// colEnd]. The same `positions` field carries PDF layout boxes
+// ([page, left, right, top, bottom]) on PDF items, so a consumer must key the
+// vocabulary off sheet_index / ck_type and never reinterpret one as the other.
+func buildSheetItems(records [][]string, sheet string, sheetIndex, headerRow int, dataRows []int, images []map[string]any) []map[string]any {
+ sortImagesByAnchor(images)
+ if len(records) == 0 {
+ return images
+ }
+ if headerRow <= 0 {
+ headerRow = 1
+ }
+ header := append([]string(nil), records[0]...)
+ headerColEnd := len(header)
+ if headerColEnd == 0 {
+ for _, row := range records {
+ if len(row) > headerColEnd {
+ headerColEnd = len(row)
+ }
+ }
+ }
+ if headerColEnd == 0 {
+ headerColEnd = 1
+ }
+ headerTuple := []float64{float64(sheetIndex), float64(headerRow), float64(headerRow), 1, float64(headerColEnd)} // [sheet, rowStart, rowEnd, colStart, colEnd]
-func nonEmptyColumnRange(row []string) (int, int) {
- start, end := 0, 0
- for i, cell := range row {
- if strings.TrimSpace(cell) == "" {
+ rows := make([]spreadsheetSegmentRow, 0, len(records)-1)
+ for i, source := range records[1:] {
+ cells := append([]string(nil), source...)
+ colStart, colEnd := nonEmptyColumnRange(cells)
+ if colStart == 0 {
continue
}
- col := i + 1
- if start == 0 {
- start = col
+ rowNum := headerRow + i + 1
+ if len(dataRows) == len(records)-1 {
+ rowNum = dataRows[i]
}
- end = col
+ rows = append(rows, spreadsheetSegmentRow{
+ cells: cells,
+ tuple: []float64{float64(sheetIndex), float64(rowNum), float64(rowNum), float64(colStart), float64(colEnd)},
+ rowNum: rowNum,
+ colStart: colStart,
+ })
}
- return start, end
-}
-func spreadsheetRowText(headers, cells []string, sheet string) string {
- values := headers
- if cells != nil {
- values = cells
- }
- parts := make([]string, 0, len(values))
- for i, value := range values {
- value = strings.TrimSpace(value)
- if value == "" {
- continue
- }
- if cells == nil {
- parts = append(parts, value)
- continue
+ anchored := make([]struct {
+ cut int
+ img map[string]any
+ }, 0, len(images))
+ for _, img := range images {
+ row, _ := numericItemInt(img["row_start"])
+ col, _ := numericItemInt(img["col_start"])
+ cut := 0
+ for cut < len(rows) {
+ r := rows[cut]
+ if r.rowNum > row || (r.rowNum == row && r.colStart > col) {
+ break
+ }
+ cut++
}
- header := ""
- if i < len(headers) {
- header = strings.TrimSpace(headers[i])
+ anchored = append(anchored, struct {
+ cut int
+ img map[string]any
+ }{cut, img})
+ }
+ sort.SliceStable(anchored, func(i, j int) bool {
+ if anchored[i].cut != anchored[j].cut {
+ return anchored[i].cut < anchored[j].cut
}
- if header != "" {
- parts = append(parts, header+":"+value)
- } else {
- parts = append(parts, value)
+ ri, _ := numericItemInt(anchored[i].img["row_start"])
+ rj, _ := numericItemInt(anchored[j].img["row_start"])
+ if ri != rj {
+ return ri < rj
}
+ ci, _ := numericItemInt(anchored[i].img["col_start"])
+ cj, _ := numericItemInt(anchored[j].img["col_start"])
+ return ci < cj
+ })
+
+ items := make([]map[string]any, 0, len(rows)+len(images))
+ newSegment := func(body []spreadsheetSegmentRow) map[string]any {
+ cells := make([][]string, 0, len(body))
+ matrix := make([][]float64, 0, len(body)+1)
+ matrix = append(matrix, headerTuple)
+ for _, r := range body {
+ cells = append(cells, r.cells)
+ matrix = append(matrix, r.tuple)
+ }
+ item := NewTableJSONItem(renderSpreadsheetTable(sheet, header, cells), sheet, matrix)
+ item["sheet_index"] = sheetIndex
+ return item
+ }
+
+ prev := 0
+ headerLeadUsed := false
+ for i := 0; i < len(anchored); {
+ cut := anchored[i].cut
+ if cut == 0 && !headerLeadUsed {
+ // A cut at 0 keeps the header in front of the image, matching the
+ // old IR's always-first header record: emit a header-only lead
+ // segment, the data rows follow in the next segment.
+ items = append(items, newSegment(nil))
+ headerLeadUsed = true
+ } else if cut > prev {
+ items = append(items, newSegment(rows[prev:cut]))
+ prev = cut
+ }
+ for i < len(anchored) && anchored[i].cut == cut {
+ items = append(items, anchored[i].img)
+ i++
+ }
+ }
+ if prev < len(rows) || !headerLeadUsed && len(rows) == 0 && len(anchored) == 0 {
+ // A header-only sheet (no data rows, no images) still emits its
+ // header as the only searchable representation of the column schema.
+ items = append(items, newSegment(rows[prev:]))
}
- text := strings.Join(parts, "; ")
- if cells != nil && sheet != "" && !strings.Contains(strings.ToLower(sheet), "sheet") {
- text += " ——" + sheet
- }
- return text
+ return items
}
-func sortSpreadsheetItems(items []map[string]any) {
- sort.SliceStable(items, func(i, j int) bool {
- headerI := items[i]["ck_type"] == "table_header"
- headerJ := items[j]["ck_type"] == "table_header"
- if headerI != headerJ {
- return headerI
- }
- ri, ci := spreadsheetItemCoordinate(items[i])
- rj, cj := spreadsheetItemCoordinate(items[j])
+func sortImagesByAnchor(images []map[string]any) {
+ sort.SliceStable(images, func(i, j int) bool {
+ ri, _ := numericItemInt(images[i]["row_start"])
+ rj, _ := numericItemInt(images[j]["row_start"])
if ri != rj {
return ri < rj
}
- if ci != cj {
- return ci < cj
- }
- return spreadsheetItemKindRank(items[i]) < spreadsheetItemKindRank(items[j])
+ ci, _ := numericItemInt(images[i]["col_start"])
+ cj, _ := numericItemInt(images[j]["col_start"])
+ return ci < cj
})
}
-func spreadsheetItemCoordinate(item map[string]any) (int, int) {
- row, _ := numericItemInt(item["row_start"])
- col, _ := numericItemInt(item["col_start"])
- return row, col
-}
-
-func spreadsheetItemKindRank(item map[string]any) int {
- if item["ck_type"] == "image" {
- return 1
+func nonEmptyColumnRange(row []string) (int, int) {
+ start, end := 0, 0
+ for i, cell := range row {
+ if strings.TrimSpace(cell) == "" {
+ continue
+ }
+ col := i + 1
+ if start == 0 {
+ start = col
+ }
+ end = col
}
- return 0
+ return start, end
}
func numericItemInt(value any) (int, bool) {
@@ -256,9 +263,10 @@ func numericItemInt(value any) (int, bool) {
}
// deprecatedChunkRows reports the removed parser-side chunking option. Row
-// boundaries are now emitted as structured IR and the selected chunker owns
-// token-budget merging; silently accepting chunk_rows would make an existing
-// configuration look effective when it is not.
+// segmentation is now owned by the parser's semantic segments and the
+// selected chunker owns token-budget splitting; silently accepting
+// chunk_rows would make an existing configuration look effective when it is
+// not.
func deprecatedChunkRows(setup map[string]any, parserName string) {
raw, exists := setup["chunk_rows"]
if !exists {
@@ -274,6 +282,17 @@ func deprecatedChunkRows(setup map[string]any, parserName string) {
zap.String("parser", parserName), zap.Int("chunk_rows", rows))
}
+// deprecatedHTML4Excel reports the retired html4excel option. Both
+// spreadsheet builders collapsed into the segmented-HTML wire, so the flag
+// selects nothing anymore. Only an enabled flag warns: false is the default
+// every parser config carries, and warning there would be noise.
+func deprecatedHTML4Excel(setup map[string]any, parserName string) {
+ if enabled, ok := setup["html4excel"].(bool); ok && enabled {
+ common.Warn("spreadsheet parser ignored deprecated html4excel; spreadsheet tables are always emitted as segmented HTML",
+ zap.String("parser", parserName))
+ }
+}
+
// extractXLSXImages returns the floating and in-cell images anchored to a
// worksheet as structured parser items. Excelize exposes both kinds through
// GetPictureCells/GetPictures; walking the reported anchor cells avoids
diff --git a/internal/parser/parser/office_table_render_test.go b/internal/parser/parser/office_table_render_test.go
index b66a67ec46..ddbfd3c1f1 100644
--- a/internal/parser/parser/office_table_render_test.go
+++ b/internal/parser/parser/office_table_render_test.go
@@ -1,14 +1,16 @@
package parser
import (
+ "html"
"reflect"
+ "regexp"
"strings"
"testing"
"github.com/xuri/excelize/v2"
)
-func TestXLSXParserEmitsSpreadsheetRows(t *testing.T) {
+func TestXLSXParserEmitsSegmentedHTMLTable(t *testing.T) {
data := newTestXLSX(t, func(f *excelize.File) {
mustSetCell(t, f, "Sheet1", "A1", "ID")
mustSetCell(t, f, "Sheet1", "B1", "Status")
@@ -22,51 +24,52 @@ func TestXLSXParserEmitsSpreadsheetRows(t *testing.T) {
if res.Err != nil {
t.Fatalf("ParseWithResult: %v", res.Err)
}
- if len(res.JSON) != 3 {
- t.Fatalf("items = %d, want header plus two rows", len(res.JSON))
+ if len(res.JSON) != 1 {
+ t.Fatalf("items = %d, want one segmented HTML table item", len(res.JSON))
}
- header := res.JSON[0]
- if header["ck_type"] != "table_header" {
- t.Fatalf("header ck_type = %v, want table_header", header["ck_type"])
+ item := res.JSON[0]
+ if item["ck_type"] != "table" || item["doc_type_kwd"] != "table" {
+ t.Fatalf("item types = ck_type:%v doc_type:%v", item["ck_type"], item["doc_type_kwd"])
}
- if got, want := header["cells"], []string{"ID", "Status"}; !reflect.DeepEqual(got, want) {
- t.Fatalf("header cells = %#v, want %#v", got, want)
+ text, _ := item["text"].(string)
+ wantText := "<table><caption>Sheet1</caption>\n" +
+ "<tr><th>ID</th><th>Status</th></tr>\n" +
+ "<tr><td>A-100</td><td>paid</td></tr>\n" +
+ "<tr><td>A-101</td><td>pending</td></tr>\n" +
+ "</table>\n"
+ if text != wantText {
+ t.Fatalf("text = %q, want %q", text, wantText)
}
- row := res.JSON[1]
- if row["ck_type"] != "table_row" || row["doc_type_kwd"] != "text" {
- t.Fatalf("row types = ck_type:%v doc_type:%v", row["ck_type"], row["doc_type_kwd"])
+ if item["sheet"] != "Sheet1" || item["sheet_index"] != 1 {
+ t.Fatalf("sheet metadata = %#v", item)
}
- if got, want := row["cells"], []string{"A-100", "paid"}; !reflect.DeepEqual(got, want) {
- t.Fatalf("row cells = %#v, want %#v", got, want)
+ positions := item["positions"].([][]float64)
+ if len(positions) != 3 {
+ t.Fatalf("len(positions) = %d, want one tuple per <tr>", len(positions))
}
- if row["sheet"] != "Sheet1" || row["sheet_index"] != 1 {
- t.Fatalf("row sheet metadata = %#v", row)
+ if got := positions[1]; !reflect.DeepEqual(got, []float64{1, 2, 2, 1, 2}) {
+ t.Fatalf("row 2 tuple = %v, want [1 2 2 1 2]", got)
}
- if row["row_start"] != 2 || row["row_end"] != 2 || row["col_start"] != 1 || row["col_end"] != 2 {
- t.Fatalf("row coordinates = %#v", row)
+ if got := positions[2]; !reflect.DeepEqual(got, []float64{1, 3, 3, 1, 2}) {
+ t.Fatalf("row 3 tuple = %v, want [1 3 3 1 2]", got)
}
}
-func TestSortSpreadsheetItemsKeepsHeaderBeforeRowsWhenHeaderIsLater(t *testing.T) {
- items := []map[string]any{
- {"ck_type": "table_row", "row_start": 1, "col_start": 1},
- {"ck_type": "table_row", "row_start": 2, "col_start": 1},
- {"ck_type": "table_header", "row_start": 5, "col_start": 1},
- }
-
- sortSpreadsheetItems(items)
- if items[0]["ck_type"] != "table_header" {
- t.Fatalf("sorted items = %#v, want table_header first", items)
- }
-}
-
-func TestXLSXParserHTML4ExcelRemainsAtomic(t *testing.T) {
+// TestXLSXParserHTML4ExcelIsIgnored pins the retirement: html4excel is still
+// accepted at the entry (with a deprecation warning) but selects nothing —
+// the wire is identical to the default path.
+func TestXLSXParserHTML4ExcelIsIgnored(t *testing.T) {
data := newTestXLSX(t, func(f *excelize.File) {
mustSetCell(t, f, "Sheet1", "A1", "Question")
mustSetCell(t, f, "Sheet1", "B1", "Answer")
mustSetCell(t, f, "Sheet1", "A2", "Q1")
mustSetCell(t, f, "Sheet1", "B2", "A1")
})
+ plain, _ := NewXLSXParser("")
+ plainRes := plain.ParseWithResult(t.Context(), "qa.xlsx", data)
+ if plainRes.Err != nil {
+ t.Fatalf("ParseWithResult: %v", plainRes.Err)
+ }
p, _ := NewXLSXParser("")
p.ConfigureFromSetup(map[string]any{"html4excel": true})
res := p.ParseWithResult(t.Context(), "qa.xlsx", data)
@@ -79,6 +82,9 @@ func TestXLSXParserHTML4ExcelRemainsAtomic(t *testing.T) {
if res.JSON[0]["ck_type"] != "table" || !strings.Contains(res.JSON[0]["text"].(string), "<table>") {
t.Fatalf("html4excel item = %#v", res.JSON[0])
}
+ if !reflect.DeepEqual(res.JSON, plainRes.JSON) {
+ t.Errorf("html4excel changed the wire: %v vs %v", res.JSON, plainRes.JSON)
+ }
}
func TestXLSXParserDoesNotPrechunkRows(t *testing.T) {
@@ -94,12 +100,20 @@ func TestXLSXParserDoesNotPrechunkRows(t *testing.T) {
if res.Err != nil {
t.Fatalf("ParseWithResult: %v", res.Err)
}
- if len(res.JSON) != dataRows+1 {
- t.Fatalf("items = %d, want one header plus %d data rows", len(res.JSON), dataRows)
+ if len(res.JSON) != 1 {
+ t.Fatalf("items = %d, want one HTML table item (parser must not pre-split rows)", len(res.JSON))
+ }
+ positions := res.JSON[0]["positions"].([][]float64)
+ if len(positions) != dataRows+1 {
+ t.Fatalf("len(positions) = %d, want one tuple per <tr> (header plus %d rows)", len(positions), dataRows)
+ }
+ last := positions[len(positions)-1]
+ if last[1] != float64(dataRows+1) || last[2] != float64(dataRows+1) {
+ t.Fatalf("last tuple = %v, want source row %d", last, dataRows+1)
}
- last := res.JSON[len(res.JSON)-1]
- if last["ck_type"] != "table_row" || last["row_start"] != dataRows+1 {
- t.Fatalf("last row = %#v, want source row %d", last, dataRows+1)
+ text, _ := res.JSON[0]["text"].(string)
+ if !strings.Contains(text, "<td>258</td>") {
+ t.Fatalf("last data row missing from markup:\n%s", text)
}
}
@@ -119,21 +133,174 @@ func TestXLSXParserPreservesSheetOrderAndIdentity(t *testing.T) {
if res.Err != nil {
t.Fatalf("ParseWithResult: %v", res.Err)
}
- if len(res.JSON) != 4 {
- t.Fatalf("items = %d, want two headers and two rows", len(res.JSON))
+ if len(res.JSON) != 2 {
+ t.Fatalf("items = %d, want one HTML table per sheet", len(res.JSON))
}
- seen := map[string]map[string]any{}
- for _, item := range res.JSON {
- if item["ck_type"] == "table_header" {
- seen[item["sheet"].(string)] = item
+ first, second := res.JSON[0], res.JSON[1]
+ if first["sheet"] != "Sheet1" || first["sheet_index"] != 1 {
+ t.Fatalf("first sheet identity = %#v", first)
+ }
+ if second["sheet"] != "Orders" || second["sheet_index"] != 2 {
+ t.Fatalf("second sheet identity = %#v", second)
+ }
+ // Table grouping collapses into sheet_index; table_id no longer exists
+ // on HTML table items (segmentation is done by the parser itself).
+ if _, ok := first["table_id"]; ok {
+ t.Errorf("table_id must not be emitted on segmented table items")
+ }
+}
+
+// TestXLSXParserEmitsHeaderOnlySegment: a sheet with no data rows still
+// emits its header as the only searchable representation of the column
+// schema — one segment whose markup holds the single <tr> of <th> cells.
+func TestXLSXParserEmitsHeaderOnlySegment(t *testing.T) {
+ data := newTestXLSX(t, func(f *excelize.File) {
+ mustSetCell(t, f, "Sheet1", "A1", "Name")
+ mustSetCell(t, f, "Sheet1", "B1", "Amount")
+ })
+ p, _ := NewXLSXParser("")
+ res := p.ParseWithResult(t.Context(), "headers.xlsx", data)
+ if res.Err != nil {
+ t.Fatalf("ParseWithResult: %v", res.Err)
+ }
+ if len(res.JSON) != 1 {
+ t.Fatalf("items = %d, want one header-only segment", len(res.JSON))
+ }
+ item := res.JSON[0]
+ wantText := "<table><caption>Sheet1</caption>\n" +
+ "<tr><th>Name</th><th>Amount</th></tr>\n" +
+ "</table>\n"
+ if item["text"] != wantText {
+ t.Fatalf("text = %q, want %q", item["text"], wantText)
+ }
+ positions, _ := item["positions"].([][]float64)
+ if len(positions) != 1 || !reflect.DeepEqual(positions[0], []float64{1, 1, 1, 1, 2}) {
+ t.Fatalf("positions = %v, want the header tuple only", positions)
+ }
+}
+
+// TestBuildSheetItemsEscapesCellText: cell text carrying markup characters is
+// escaped in the wire and reads back byte-identical from the markup, so
+// user content can never become markup.
+func TestBuildSheetItemsEscapesCellText(t *testing.T) {
+ const tricky = `<b>bold</b> & "quoted" > 1`
+ items := buildSheetItems([][]string{{"Name"}, {tricky}}, "Sheet1", 1, 1, nil, nil)
+ if len(items) != 1 {
+ t.Fatalf("items = %d, want one segment", len(items))
+ }
+ text, _ := items[0]["text"].(string)
+ if strings.Contains(text, "<b>") || !strings.Contains(text, "&lt;b&gt;") {
+ t.Fatalf("cell text was not escaped: %q", text)
+ }
+ rows := markupRows(t, text)
+ if len(rows) != 2 || rows[1][0] != tricky {
+ t.Fatalf("round trip = %#v, want %q back", rows, tricky)
+ }
+}
+
+// TestXLSXParserImageAnchorsSplitSegments pins the segmentation semantics: an
+// image anchored between rows ends the current segment at its anchor, every
+// segment repeats the header row, and a data row sharing the anchor's row is
+// ordered by column — cells at or left of the anchor column stay in front of
+// the image, cells to its right follow it.
+func TestXLSXParserImageAnchorsSplitSegments(t *testing.T) {
+ const pngBase64 = "iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAQAAAC1HAwCAAAAC0lEQVR42mNk+M8AAAMBAQDJ/pLvAAAAAElFTkSuQmCC"
+
+ build := func(t *testing.T, fillRow4 func(*excelize.File)) []map[string]any {
+ t.Helper()
+ data := newTestXLSX(t, func(f *excelize.File) {
+ mustSetCell(t, f, "Sheet1", "A1", "Name")
+ mustSetCell(t, f, "Sheet1", "B1", "Value")
+ // Every non-anchor row carries two columns so the header
+ // detector never sees row 4 as the only wide row.
+ for row, value := range map[int]string{2: "row2", 3: "row3", 5: "row5", 6: "row6"} {
+ mustSetCell(t, f, "Sheet1", cellAxis(row, 1), value)
+ mustSetCell(t, f, "Sheet1", cellAxis(row, 2), "x")
+ }
+ fillRow4(f)
+ if err := f.AddPictureFromBytes("Sheet1", "A4", &excelize.Picture{
+ Extension: ".png",
+ File: mustDecodeBase64(t, pngBase64),
+ Format: &excelize.GraphicOptions{AltText: "fig"},
+ }); err != nil {
+ t.Fatalf("AddPictureFromBytes: %v", err)
+ }
+ })
+ p, _ := NewXLSXParser("")
+ res := p.ParseWithResult(t.Context(), "anchors.xlsx", data)
+ if res.Err != nil {
+ t.Fatalf("ParseWithResult: %v", res.Err)
}
+ return res.JSON
}
- if first := seen["Sheet1"]; first == nil || first["table_id"] != "sheet-1" || first["sheet_index"] != 1 {
- t.Fatalf("Sheet1 identity = %#v", first)
+
+ assertSegments := func(t *testing.T, items []map[string]any, wantFirst, wantSecond []string) {
+ t.Helper()
+ if len(items) != 3 {
+ t.Fatalf("items = %d, want table, image, table", len(items))
+ }
+ if items[1]["doc_type_kwd"] != "image" || items[1]["row_start"] != 4 {
+ t.Fatalf("image item = %#v", items[1])
+ }
+ for segIdx, want := range [][]string{wantFirst, wantSecond} {
+ item := items[segIdx*2]
+ text, _ := item["text"].(string)
+ rows := markupRows(t, text)
+ if len(rows) != len(want)+1 {
+ t.Fatalf("segment %d rows = %d, want header plus %d", segIdx, len(rows), len(want))
+ }
+ if rows[0][0] != "Name" {
+ t.Errorf("segment %d lost the replicated header: %#v", segIdx, rows[0])
+ }
+ for i, value := range want {
+ found := false
+ for _, cell := range rows[i+1] {
+ if cell == value {
+ found = true
+ }
+ }
+ if !found {
+ t.Errorf("segment %d row %d = %#v, want %q in it", segIdx, i, rows[i+1], value)
+ }
+ }
+ positions, _ := item["positions"].([][]float64)
+ if len(positions) != len(rows) {
+ t.Errorf("segment %d positions = %d, want one tuple per <tr> (%d)", segIdx, len(positions), len(rows))
+ }
+ }
}
- if second := seen["Orders"]; second == nil || second["table_id"] != "sheet-2" || second["sheet_index"] != 2 {
- t.Fatalf("Orders identity = %#v", second)
+
+ t.Run("row at the anchor cell stays in front", func(t *testing.T) {
+ items := build(t, func(f *excelize.File) {
+ mustSetCell(t, f, "Sheet1", "A4", "row4")
+ mustSetCell(t, f, "Sheet1", "B4", "x")
+ })
+ assertSegments(t, items, []string{"row2", "row3", "row4"}, []string{"row5", "row6"})
+ })
+
+ t.Run("row right of the anchor cell follows the image", func(t *testing.T) {
+ items := build(t, func(f *excelize.File) { mustSetCell(t, f, "Sheet1", "B4", "row4") })
+ assertSegments(t, items, []string{"row2", "row3"}, []string{"row4", "row5", "row6"})
+ })
+}
+
+// markupRows extracts the cell texts of every <tr> in rendered spreadsheet
+// markup, unescaped. It is a deliberately minimal test-local reader: these
+// tests assert the parser's rendered output, so the markup itself is the
+// contract under test.
+func markupRows(t *testing.T, markup string) [][]string {
+ t.Helper()
+ rowRe := regexp.MustCompile(`(?s)<tr>(.*?)</tr>`)
+ cellRe := regexp.MustCompile(`(?s)<t[hd]>(.*?)</t[hd]>`)
+ var rows [][]string
+ for _, row := range rowRe.FindAllStringSubmatch(markup, -1) {
+ var cells []string
+ for _, cell := range cellRe.FindAllStringSubmatch(row[1], -1) {
+ cells = append(cells, html.UnescapeString(cell[1]))
+ }
+ rows = append(rows, cells)
}
+ return rows
}
func spreadsheetText(res ParseResult) string {
@@ -145,13 +312,33 @@ func spreadsheetText(res ParseResult) string {
return out.String()
}
-func spreadsheetHeaderItem(res ParseResult) map[string]any {
+// spreadsheetHeaderText returns the leading <table> item's header labels
+// joined by "; " — the rendered <th> cells of its first <tr> — or "" when no
+// table item exists.
+func spreadsheetHeaderText(res ParseResult) string {
for _, item := range res.JSON {
- if item["ck_type"] == "table_header" {
- return item
+ text, _ := item["text"].(string)
+ if !strings.Contains(strings.ToLower(text), "<table") {
+ continue
+ }
+ row := text
+ if i := strings.Index(row, "<tr>"); i >= 0 {
+ row = row[i+len("<tr>"):]
+ }
+ if i := strings.Index(row, "</tr>"); i >= 0 {
+ row = row[:i]
+ }
+ var labels []string
+ for _, cell := range strings.Split(row, "<th>")[1:] {
+ if i := strings.Index(cell, "</th>"); i >= 0 {
+ labels = append(labels, cell[:i])
+ }
+ }
+ if len(labels) > 0 {
+ return strings.Join(labels, "; ")
}
}
- return nil
+ return ""
}
func TestXLSXImageMIMEType(t *testing.T) {
@@ -238,8 +425,9 @@ func mustAddTable(t *testing.T, f *excelize.File, sheet string, table *excelize.
}
}
-// TestXLSXParser_HeaderAndCaption asserts the XLSX parser emits a typed header
-// row and data row with the header labels preserved separately from values.
+// TestXLSXParser_HeaderAndCaption asserts the XLSX parser renders the typed
+// header row as <th> cells and data rows as <td> cells inside one captioned
+// table, with header labels never duplicated as values.
func TestXLSXParser_HeaderAndCaption(t *testing.T) {
data := newTestXLSX(t, func(f *excelize.File) {
mustSetCell(t, f, "Sheet1", "A1", "Product")
@@ -255,14 +443,14 @@ func TestXLSXParser_HeaderAndCaption(t *testing.T) {
t.Fatalf("ParseWithResult: %v", res.Err)
}
text := spreadsheetText(res)
- if !strings.Contains(text, "Product; Price") {
- t.Fatalf("want header text, got:\n%s", text)
+ if !strings.Contains(text, "<tr><th>Product</th><th>Price</th></tr>") {
+ t.Fatalf("want header row, got:\n%s", text)
}
- if !strings.Contains(text, "Product:Widget; Price:9.99") {
- t.Fatalf("want row text, got:\n%s", text)
+ if !strings.Contains(text, "<tr><td>Widget</td><td>9.99</td></tr>") {
+ t.Fatalf("want data row, got:\n%s", text)
}
- // The header label must not be duplicated as a value in the row text.
- if strings.Contains(text, "Product:Product") {
+ // The header label must not be duplicated as a value in any row.
+ if strings.Contains(text, "<td>Product</td>") {
t.Fatalf("header value leaked into row text:\n%s", text)
}
}
@@ -291,7 +479,7 @@ func TestXLSXParser_MergedHeaderInheritance(t *testing.T) {
t.Fatalf("ParseWithResult: %v", res.Err)
}
text := spreadsheetText(res)
- if !strings.Contains(text, "Sales Report; Sales Report; Sales Report") {
+ if !strings.Contains(text, "<th>Sales Report</th><th>Sales Report</th><th>Sales Report</th>") {
t.Fatalf("merged master text not inherited into header cells, got:\n%s", text)
}
}
@@ -313,14 +501,10 @@ func TestDetectHeaderRow_ListObject(t *testing.T) {
if res.Err != nil {
t.Fatalf("ParseWithResult: %v", res.Err)
}
- if !strings.Contains(spreadsheetText(res), "Name; Age") {
+ if !strings.Contains(spreadsheetText(res), "<th>Name</th><th>Age</th>") {
t.Fatalf("want ListObject header row detected, got:\n%s", spreadsheetText(res))
}
- header := spreadsheetHeaderItem(res)
- if header == nil {
- t.Fatal("missing table_header item")
- }
- if got := header["text"]; got != "Name; Age" {
+ if got := spreadsheetHeaderText(res); got != "Name; Age" {
t.Fatalf("header text = %v, want Name; Age", got)
}
}
@@ -344,10 +528,10 @@ func TestDetectHeaderRow_Lightweight(t *testing.T) {
if res.Err != nil {
t.Fatalf("ParseWithResult: %v", res.Err)
}
- if !strings.Contains(spreadsheetText(res), "Item; Count") {
+ if !strings.Contains(spreadsheetText(res), "<th>Item</th><th>Count</th>") {
t.Fatalf("want row-2 header detected, got:\n%s", spreadsheetText(res))
}
- if strings.Contains(spreadsheetText(res), "100; 200") {
+ if strings.Contains(spreadsheetText(res), "<th>100</th><th>200</th>") {
t.Fatalf("numeric row 1 must not be the header:\n%s", spreadsheetText(res))
}
}
@@ -366,7 +550,7 @@ func TestXLSXParser_CommonCaseNoRegression(t *testing.T) {
if res.Err != nil {
t.Fatalf("ParseWithResult: %v", res.Err)
}
- if !strings.Contains(spreadsheetText(res), "col_a; col_b") {
+ if !strings.Contains(spreadsheetText(res), "<th>col_a</th><th>col_b</th>") {
t.Fatalf("common-case header must be preserved:\n%s", spreadsheetText(res))
}
}
@@ -394,10 +578,10 @@ func TestDetectHeaderRow_BoldSubtotalNotHeader(t *testing.T) {
if res.Err != nil {
t.Fatalf("ParseWithResult: %v", res.Err)
}
- if !strings.Contains(spreadsheetText(res), "2023; 2024") {
+ if !strings.Contains(spreadsheetText(res), "<th>2023</th><th>2024</th>") {
t.Fatalf("numeric row 1 must remain the header:\n%s", spreadsheetText(res))
}
- if strings.Contains(spreadsheetText(res), "Total; Summary") {
+ if strings.Contains(spreadsheetText(res), "<th>Total</th><th>Summary</th>") {
t.Fatalf("bold subtotal row must not be promoted to header:\n%s", spreadsheetText(res))
}
}
@@ -427,14 +611,10 @@ func TestDetectHeaderRow_StyledHeaderPastFarMerge(t *testing.T) {
if res.Err != nil {
t.Fatalf("ParseWithResult: %v", res.Err)
}
- if !strings.Contains(spreadsheetText(res), "Name; Desc") {
+ if !strings.Contains(spreadsheetText(res), "<th>Name</th><th>Desc</th>") {
t.Fatalf("narrow bold header past far merge must be detected:\n%s", spreadsheetText(res))
}
- header := spreadsheetHeaderItem(res)
- if header == nil {
- t.Fatal("missing table_header item")
- }
- if got := header["text"]; got != "Name; Desc" {
+ if got := spreadsheetHeaderText(res); got != "Name; Desc" {
t.Fatalf("header text = %v, want Name; Desc", got)
}
}
@@ -461,14 +641,10 @@ func TestDetectHeaderRow_StyledTextHeaderOverNumeric(t *testing.T) {
if res.Err != nil {
t.Fatalf("ParseWithResult: %v", res.Err)
}
- if !strings.Contains(spreadsheetText(res), "Product; Units") {
+ if !strings.Contains(spreadsheetText(res), "<th>Product</th><th>Units</th>") {
t.Fatalf("styled text header over numeric data must be detected:\n%s", spreadsheetText(res))
}
- header := spreadsheetHeaderItem(res)
- if header == nil {
- t.Fatal("missing table_header item")
- }
- if got := header["text"]; got != "Product; Units" {
+ if got := spreadsheetHeaderText(res); got != "Product; Units" {
t.Fatalf("header text = %v, want Product; Units", got)
}
}
diff --git a/internal/parser/parser/pdf_parser_common.go b/internal/parser/parser/pdf_parser_common.go
index 4cb485a94b..8534d72f7f 100644
--- a/internal/parser/parser/pdf_parser_common.go
+++ b/internal/parser/parser/pdf_parser_common.go
@@ -784,10 +784,10 @@ func normalizePDFPositions(raw any) [][]any {
// normalizePDFPageNumber converts a DeepDoc 0-indexed page number to the
// 1-indexed form stored in _pdf_positions / positions. It is the SINGLE
// 0→1 conversion point: DeepDoc (pdf_oxide/pdfium) emits 0-indexed pages,
-// and every downstream consumer (AddPositions for ES storage,
+// and every downstream consumer (addPDFPositions for ES storage,
// PositionsFromMatrix for the PDFium render path) expects 1-indexed input.
// Adding +1 unconditionally — instead of only for v<=0 — keeps all pages
-// consistent; the old heuristic left page>=1 unconverted, which AddPositions
+// consistent; the old heuristic left page>=1 unconverted, which addPDFPositions
// then double-incremented and PositionsFromMatrix mis-decremented.
func normalizePDFPageNumber(raw any) (int, bool) {
switch v := raw.(type) {
diff --git a/internal/parser/parser/pdf_parser_common_test.go b/internal/parser/parser/pdf_parser_common_test.go
index 46358f7285..584171bc68 100644
--- a/internal/parser/parser/pdf_parser_common_test.go
+++ b/internal/parser/parser/pdf_parser_common_test.go
@@ -156,7 +156,7 @@ func TestPDFParseResultToJSON_ClassifiesFigureCaptionWithImage(t *testing.T) {
// TestNormalizePDFPageNumber_UnconditionalIncrement pins the contract that
// DeepDoc emits 0-indexed page numbers and normalizePDFPageNumber is the
// SINGLE conversion point to 1-indexed. It must add +1 unconditionally —
-// not just for v<=0 — so that downstream AddPositions (a passthrough) and
+// not just for v<=0 — so that downstream addPDFPositions (a passthrough) and
// PositionsFromMatrix (which subtracts 1 for the 0-indexed PDFium engine)
// each see a consistent 1-indexed value.
func TestNormalizePDFPageNumber_UnconditionalIncrement(t *testing.T) {
diff --git a/internal/parser/parser/pdf_parser_tcadp.go b/internal/parser/parser/pdf_parser_tcadp.go
index 4aa517bac9..faafe813e8 100644
--- a/internal/parser/parser/pdf_parser_tcadp.go
+++ b/internal/parser/parser/pdf_parser_tcadp.go
@@ -84,7 +84,7 @@ func tcadpAnyToItems(raw any) []map[string]any {
emit := func(text, docType, layout string) []map[string]any {
m := map[string]any{"text": text, "doc_type_kwd": docType, "layout": layout}
if page > 0 {
- // 1-indexed 5-tuple. AddPositions is a passthrough so
+ // 1-indexed 5-tuple. addPDFPositions is a passthrough so
// the final position_int / page_num_int carry the same
// 1-indexed page number the caller passes. Mirrors
// Python presentation.py:148-149.
diff --git a/internal/parser/parser/pdf_parser_tcadp_test.go b/internal/parser/parser/pdf_parser_tcadp_test.go
index bf517546a7..f3775e72b3 100644
--- a/internal/parser/parser/pdf_parser_tcadp_test.go
+++ b/internal/parser/parser/pdf_parser_tcadp_test.go
@@ -92,7 +92,7 @@ func TestPDFParser_ParseWithResult_TCADPRequiresAPIServer(t *testing.T) {
// : per-slide page numbers present in the raw TCADP response
// must be attached to each chunk item as "positions" (a 1-indexed 5-tuple
// [page, 0, 0, 0, 0]). The shared ingestion pipeline
-// (processChunkPositions -> AddPositions, a passthrough) derives
+// (processChunkPositions -> addPDFPositions, a passthrough) derives
// top_int=[0], position_int=[[page,0,0,0,0]] and page_num_int=[page] from
// this field. Elements without a page number (e.g. spreadsheet TCADP)
// must NOT receive a positions field.
diff --git a/internal/parser/parser/xls_parser.go b/internal/parser/parser/xls_parser.go
index e807ea8b63..7dc42a4269 100644
--- a/internal/parser/parser/xls_parser.go
+++ b/internal/parser/parser/xls_parser.go
@@ -22,22 +22,16 @@ import (
)
type XLSParser struct {
- libType string
ParseMethod string
OutputFormat string
- HTML4Excel bool
TCADPAPIServer string
TCADPAPIKey string
TCADPTableResultType string
TCADPMarkdownImageResponseType string
}
-func NewXLSParser(libType string) (*XLSParser, error) {
- if libType == "" {
- libType = "excelize"
- }
+func NewXLSParser(_ string) (*XLSParser, error) {
return &XLSParser{
- libType: libType,
TCADPTableResultType: "1",
TCADPMarkdownImageResponseType: "1",
}, nil
@@ -57,9 +51,7 @@ func (p *XLSParser) ConfigureFromSetup(setup map[string]any) {
if v, ok := setup["output_format"].(string); ok && v != "" {
p.OutputFormat = v
}
- if v, ok := setup["html4excel"].(bool); ok {
- p.HTML4Excel = v
- }
+ deprecatedHTML4Excel(setup, p.String())
deprecatedChunkRows(setup, p.String())
if v, ok := setup["tcadp_apiserver"].(string); ok && v != "" {
p.TCADPAPIServer = v
@@ -93,7 +85,7 @@ func (p *XLSParser) ParseWithResult(ctx context.Context, filename string, data [
}
}
- items, warnings, sheetsCount, err := parseXLSXBytes(data, p.HTML4Excel)
+ items, warnings, sheetsCount, err := parseXLSXBytes(data)
if err != nil {
return ParseResult{Err: fmt.Errorf("xls parse: %w", err)}
}
diff --git a/internal/parser/parser/xls_parser_test.go b/internal/parser/parser/xls_parser_test.go
index c9ac44e4db..6f5d8dbf54 100644
--- a/internal/parser/parser/xls_parser_test.go
+++ b/internal/parser/parser/xls_parser_test.go
@@ -3,6 +3,7 @@ package parser
import (
"bytes"
"context"
+ "strings"
"testing"
"github.com/xuri/excelize/v2"
@@ -46,14 +47,16 @@ func TestXLSParser_SpreadsheetJSONOutput(t *testing.T) {
t.Fatalf("res.JSON should have structured table items")
}
item := res.JSON[0]
- if item["ck_type"] != "table_header" {
- t.Errorf("header ck_type = %v, want table_header", item["ck_type"])
+ if item["ck_type"] != "table" {
+ t.Errorf("table item ck_type = %v, want table", item["ck_type"])
}
- if len(res.JSON) < 2 || res.JSON[1]["text"] != "Header1:Val1; Header2:Val2" {
- t.Errorf("row item = %#v", res.JSON)
+ text, _ := item["text"].(string)
+ if !strings.Contains(text, "<tr><th>Header1</th><th>Header2</th></tr>") ||
+ !strings.Contains(text, "<tr><td>Val1</td><td>Val2</td></tr>") {
+ t.Errorf("table markup = %q, want captioned header and data row", text)
}
if item["doc_type_kwd"] != "table" {
- t.Errorf("header doc_type_kwd = %v, want 'table'", item["doc_type_kwd"])
+ t.Errorf("doc_type_kwd = %v, want 'table'", item["doc_type_kwd"])
}
// Configure output format to "json"
@@ -87,11 +90,11 @@ func TestXLSXParser_JSONOutput(t *testing.T) {
if res.OutputFormat != "json" {
t.Errorf("res.OutputFormat = %q, want 'json'", res.OutputFormat)
}
- if len(res.JSON) < 2 {
- t.Fatalf("res.JSON = %#v, want header and row", res.JSON)
+ if len(res.JSON) != 1 {
+ t.Fatalf("res.JSON = %#v, want one segmented HTML table item", res.JSON)
}
- if res.JSON[0]["ck_type"] != "table_header" || res.JSON[1]["ck_type"] != "table_row" {
- t.Errorf("items should be header and row, got %#v", res.JSON)
+ if res.JSON[0]["ck_type"] != "table" {
+ t.Errorf("item ck_type = %v, want table", res.JSON[0]["ck_type"])
}
// Even if configured to legacy "html", format is unified to "json"
diff --git a/internal/parser/parser/xls_tcadp_test.go b/internal/parser/parser/xls_tcadp_test.go
index f72a517dbb..f623c3ce98 100644
--- a/internal/parser/parser/xls_tcadp_test.go
+++ b/internal/parser/parser/xls_tcadp_test.go
@@ -225,11 +225,11 @@ func TestCSVParser_ParseWithResult_DefaultCSVBehavior(t *testing.T) {
if got, want := res.OutputFormat, "json"; got != want {
t.Fatalf("OutputFormat = %q, want %q", got, want)
}
- if len(res.JSON) < 2 {
- t.Fatalf("JSON items = %d, want at least header and one data row", len(res.JSON))
+ if len(res.JSON) != 1 {
+ t.Fatalf("JSON items = %d, want one HTML table item", len(res.JSON))
}
- if res.JSON[0]["ck_type"] != "table_header" || res.JSON[1]["ck_type"] != "table_row" {
- t.Fatalf("JSON items = %#v; want header and row", res.JSON)
+ if res.JSON[0]["ck_type"] != "table" {
+ t.Fatalf("JSON item = %#v; want a table item", res.JSON[0])
}
}
diff --git a/internal/parser/parser/xlsx_parse_method_test.go b/internal/parser/parser/xlsx_parse_method_test.go
index c5e9ae57ba..9374087ac1 100644
--- a/internal/parser/parser/xlsx_parse_method_test.go
+++ b/internal/parser/parser/xlsx_parse_method_test.go
@@ -160,7 +160,8 @@ func mustDecodeBase64(t *testing.T, encoded string) []byte {
}
// TestCSVParser_DeepDocParseMethod asserts the CSV parser accepts the
-// default "deepdoc" parse_method and emits row IR.
+// default "deepdoc" parse_method and emits the segmented HTML table wire
+// shape.
func TestCSVParser_DeepDocParseMethod(t *testing.T) {
p := NewCSVParser()
p.ConfigureFromSetup(map[string]any{"parse_method": "deepdoc"})
@@ -173,11 +174,11 @@ func TestCSVParser_DeepDocParseMethod(t *testing.T) {
if got, want := res.OutputFormat, "json"; got != want {
t.Fatalf("OutputFormat = %q, want %q", got, want)
}
- if len(res.JSON) < 2 {
- t.Fatalf("JSON items = %d, want at least header and one data row", len(res.JSON))
+ if len(res.JSON) != 1 {
+ t.Fatalf("JSON items = %d, want one HTML table item", len(res.JSON))
}
- if res.JSON[0]["ck_type"] != "table_header" || res.JSON[1]["ck_type"] != "table_row" {
- t.Fatalf("JSON items = %#v, want header and row", res.JSON)
+ if res.JSON[0]["ck_type"] != "table" {
+ t.Fatalf("JSON item = %#v, want a table item", res.JSON[0])
}
}
diff --git a/internal/parser/parser/xlsx_parser.go b/internal/parser/parser/xlsx_parser.go
index aff077bd90..d71ca9c170 100644
--- a/internal/parser/parser/xlsx_parser.go
+++ b/internal/parser/parser/xlsx_parser.go
@@ -26,22 +26,16 @@ import (
)
type XLSXParser struct {
- libType string
ParseMethod string
OutputFormat string
- HTML4Excel bool
TCADPAPIServer string
TCADPAPIKey string
TCADPTableResultType string
TCADPMarkdownImageResponseType string
}
-func NewXLSXParser(libType string) (*XLSXParser, error) {
- if libType == "" {
- libType = "excelize"
- }
+func NewXLSXParser(_ string) (*XLSXParser, error) {
return &XLSXParser{
- libType: libType,
TCADPTableResultType: "1",
TCADPMarkdownImageResponseType: "1",
}, nil
@@ -61,9 +55,7 @@ func (p *XLSXParser) ConfigureFromSetup(setup map[string]any) {
if v, ok := setup["output_format"].(string); ok && v != "" {
p.OutputFormat = v
}
- if v, ok := setup["html4excel"].(bool); ok {
- p.HTML4Excel = v
- }
+ deprecatedHTML4Excel(setup, p.String())
deprecatedChunkRows(setup, p.String())
if v, ok := setup["tcadp_apiserver"].(string); ok && v != "" {
p.TCADPAPIServer = v
@@ -115,7 +107,7 @@ func (p *XLSXParser) ParseWithResult(ctx context.Context, filename string, data
// for spreadsheet processing.
}
- items, warnings, sheets, err := parseXLSXBytes(data, p.HTML4Excel)
+ items, warnings, sheets, err := parseXLSXBytes(data)
if err == nil {
return xlsxParseResult(filename, items, warnings, sheets)
}
@@ -127,7 +119,7 @@ func (p *XLSXParser) ParseWithResult(ctx context.Context, filename string, data
if !changed {
return ParseResult{Err: fmt.Errorf("xlsx parse: %w", err)}
}
- items, warnings, sheets, retryErr := parseXLSXBytes(normalized, p.HTML4Excel)
+ items, warnings, sheets, retryErr := parseXLSXBytes(normalized)
if retryErr != nil {
return ParseResult{Err: fmt.Errorf("xlsx parse: %w; retry after normalization: %v", err, retryErr)}
}
@@ -135,7 +127,7 @@ func (p *XLSXParser) ParseWithResult(ctx context.Context, filename string, data
return xlsxParseResult(filename, items, warnings, sheets)
}
-func parseXLSXBytes(data []byte, html4excel bool) ([]map[string]any, []string, int, error) {
+func parseXLSXBytes(data []byte) ([]map[string]any, []string, int, error) {
f, err := excelize.OpenReader(bytes.NewReader(data))
if err != nil {
return nil, nil, 0, fmt.Errorf("open XLSX: %w", err)
@@ -151,25 +143,16 @@ func parseXLSXBytes(data []byte, html4excel bool) ([]map[string]any, []string, i
return nil, warnings, len(sheets), err
}
warnings = append(warnings, sheetWarnings...)
- var sheetItems []map[string]any
- if html4excel {
- if table := recordsToHTMLTableItem(records, sheet, sheetIdx+1, headerRow, dataRows); table != nil {
- sheetItems = []map[string]any{table}
- }
- } else {
- sheetItems = recordsToSpreadsheetItems(records, sheet, sheetIdx+1, headerRow, dataRows)
- }
images, imageWarnings := extractXLSXImages(f, sheet)
for _, image := range images {
row, _ := numericItemInt(image["row_start"])
col, _ := numericItemInt(image["col_start"])
image["sheet_index"] = sheetIdx + 1
- image["table_id"] = fmt.Sprintf("sheet-%d", sheetIdx+1)
image["positions"] = [][]float64{{float64(sheetIdx + 1), float64(row), float64(row), float64(col), float64(col)}}
}
- sheetItems = append(sheetItems, images...)
- sortSpreadsheetItems(sheetItems)
- items = append(items, sheetItems...)
+ // One wire shape for every spreadsheet sheet: segmented HTML tables
+ // with row-aligned positions, images interleaved at their anchors.
+ items = append(items, buildSheetItems(records, sheet, sheetIdx+1, headerRow, dataRows, images)...)
warnings = append(warnings, imageWarnings...)
}
return items, warnings, len(sheets), nil
diff --git a/internal/service/file/file_content.go b/internal/service/file/file_content.go
index 0abb3a6929..551c968316 100644
--- a/internal/service/file/file_content.go
+++ b/internal/service/file/file_content.go
@@ -5,7 +5,6 @@ import (
"encoding/base64"
"encoding/json"
"fmt"
- "html"
"ragflow/internal/dao"
"ragflow/internal/entity"
"ragflow/internal/parser/parser"
@@ -234,9 +233,6 @@ func parseResultText(res parser.ParseResult) (string, error) {
return res.HTML, nil
case "json":
if len(res.JSON) > 0 {
- if rendered, ok := renderSpreadsheetJSON(res.JSON); ok {
- return rendered, nil
- }
parts := make([]string, 0, len(res.JSON))
for _, item := range res.JSON {
if text, ok := item["text"].(string); ok {
@@ -265,164 +261,6 @@ func parseResultText(res parser.ParseResult) (string, error) {
}
}
-func renderSpreadsheetJSON(items []map[string]any) (string, bool) {
- parts := make([]string, 0, len(items))
- rendered := false
- for i := 0; i < len(items); {
- if !isSpreadsheetRowItem(items[i]) {
- if text, ok := items[i]["text"].(string); ok {
- parts = append(parts, text)
- } else {
- raw, err := json.Marshal(items[i])
- if err != nil {
- return "", false
- }
- parts = append(parts, string(raw))
- }
- i++
- continue
- }
-
- start := i
- i = spreadsheetTableEnd(items, start)
- parts = append(parts, renderSpreadsheetTable(items[start:i]))
- rendered = true
- }
- if !rendered {
- return "", false
- }
- return strings.Join(parts, "\n"), true
-}
-
-func isSpreadsheetRowItem(item map[string]any) bool {
- kind, _ := item["ck_type"].(string)
- return kind == "table_header" || kind == "table_row"
-}
-
-func spreadsheetTableKey(item map[string]any) string {
- if tableID, _ := item["table_id"].(string); strings.TrimSpace(tableID) != "" {
- return "table:" + tableID
- }
- if sheetIndex, ok := item["sheet_index"].(float64); ok {
- return fmt.Sprintf("sheet-index:%g", sheetIndex)
- }
- if sheetIndex, ok := item["sheet_index"].(int); ok {
- return fmt.Sprintf("sheet-index:%d", sheetIndex)
- }
- if sheet, _ := item["sheet"].(string); strings.TrimSpace(sheet) != "" {
- return "sheet:" + sheet
- }
- return ""
-}
-
-func spreadsheetTableEnd(items []map[string]any, start int) int {
- key := spreadsheetTableKey(items[start])
- if key != "" {
- i := start
- for i < len(items) && isSpreadsheetRowItem(items[i]) && spreadsheetTableKey(items[i]) == key {
- i++
- }
- return i
- }
-
- // Without a stable table identity, do not merge unrelated row sequences.
- // A header can still own the following anonymous rows until the next
- // anonymous header; anonymous row-only inputs are kept as separate tables
- // because their table boundary cannot be inferred safely.
- if kind, _ := items[start]["ck_type"].(string); kind == "table_header" {
- i := start + 1
- for i < len(items) && isSpreadsheetRowItem(items[i]) {
- if nextKind, _ := items[i]["ck_type"].(string); nextKind == "table_header" {
- break
- }
- if spreadsheetTableKey(items[i]) != "" {
- break
- }
- i++
- }
- return i
- }
- return start + 1
-}
-
-func renderSpreadsheetTable(items []map[string]any) string {
- if len(items) == 0 {
- return ""
- }
- var header []string
- if kind, _ := items[0]["ck_type"].(string); kind == "table_header" {
- header = spreadsheetStringSlice(items[0]["cells"])
- if len(header) == 0 {
- header = spreadsheetStringSlice(items[0]["text"])
- }
- } else {
- // Row IR carries the logical column names separately from its cells.
- // Use those names as a preview header without consuming the first row.
- header = spreadsheetStringSlice(items[0]["headers"])
- }
-
- sheet, _ := items[0]["sheet"].(string)
- var builder strings.Builder
- builder.WriteString("<table>")
- if strings.TrimSpace(sheet) != "" {
- builder.WriteString("<caption>")
- builder.WriteString(html.EscapeString(strings.TrimSpace(sheet)))
- builder.WriteString("</caption>")
- }
- if len(header) > 0 {
- builder.WriteString("<tr>")
- for _, cell := range header {
- builder.WriteString("<th>")
- builder.WriteString(html.EscapeString(cell))
- builder.WriteString("</th>")
- }
- builder.WriteString("</tr>")
- }
- for _, item := range items {
- kind, _ := item["ck_type"].(string)
- if kind == "table_header" {
- continue
- }
- cells := spreadsheetStringSlice(item["cells"])
- if len(cells) == 0 {
- cells = spreadsheetStringSlice(item["text"])
- }
- builder.WriteString("<tr>")
- for _, cell := range cells {
- builder.WriteString("<td>")
- builder.WriteString(html.EscapeString(cell))
- builder.WriteString("</td>")
- }
- builder.WriteString("</tr>")
- }
- builder.WriteString("</table>")
- return builder.String()
-}
-
-func spreadsheetStringSlice(value any) []string {
- switch values := value.(type) {
- case []string:
- out := make([]string, len(values))
- for i, value := range values {
- out[i] = strings.TrimSpace(value)
- }
- return out
- case []any:
- out := make([]string, 0, len(values))
- for _, value := range values {
- out = append(out, strings.TrimSpace(fmt.Sprint(value)))
- }
- return out
- case string:
- if strings.TrimSpace(values) == "" {
- return nil
- }
- return []string{strings.TrimSpace(values)}
- default:
- return nil
- }
-}
-
// parseFileContent tries to parse a file's contents using the appropriate parser.
// Falls back to returning raw text if no parser is available.
func parseFileContent(ctx context.Context, filename string, data []byte) string {
diff --git a/internal/service/file/file_content_test.go b/internal/service/file/file_content_test.go
index 5a70e6edf3..b0421c0681 100644
--- a/internal/service/file/file_content_test.go
+++ b/internal/service/file/file_content_test.go
@@ -75,88 +75,22 @@ func TestParseResultText_JSONItemsPreservesOrderAndFallsBackToJSON(t *testing.T)
}
}
-func TestParseResultText_SpreadsheetRowsRenderReadableHTMLTable(t *testing.T) {
+// TestParseResultText_SpreadsheetSegmentsAreTheirOwnMarkup: the wire emits
+// self-describing HTML table segments, so the readable preview is exactly
+// the item text — no parser-side table re-rendering.
+func TestParseResultText_SpreadsheetSegmentsAreTheirOwnMarkup(t *testing.T) {
+ segment := "<table><caption>Data</caption>\n<tr><th>a</th><th>b</th></tr>\n<tr><td>1</td><td>2</td></tr>\n</table>\n"
result, err := parseResultText(parser.ParseResult{
OutputFormat: "json",
JSON: []map[string]any{
- {
- "text": "a; b",
- "doc_type_kwd": "table",
- "ck_type": "table_header",
- "table_id": "sheet-1",
- "sheet": "Data",
- "cells": []string{"a", "b"},
- },
- {
- "text": "a:1; b:2 ——Data",
- "doc_type_kwd": "text",
- "ck_type": "table_row",
- "table_id": "sheet-1",
- "sheet": "Data",
- "headers": []string{"a", "b"},
- "cells": []string{"1", "2"},
- },
+ {"text": segment, "doc_type_kwd": "table", "ck_type": "table", "sheet": "Data", "sheet_index": 1},
},
})
if err != nil {
t.Fatalf("parseResultText: %v", err)
}
- for _, want := range []string{"<table", "<caption>Data</caption>", "<th>a</th>", "<td>1</td>"} {
- if !strings.Contains(result, want) {
- t.Errorf("parseResultText = %q, want %q", result, want)
- }
- }
-}
-
-func TestParseResultText_SpreadsheetRowsWithoutHeaderDoNotDuplicateFirstRow(t *testing.T) {
- result, err := parseResultText(parser.ParseResult{
- OutputFormat: "json",
- JSON: []map[string]any{
- {
- "ck_type": "table_row",
- "headers": []string{"name", "value"},
- "cells": []string{"alpha", "1"},
- },
- },
- })
- if err != nil {
- t.Fatalf("parseResultText: %v", err)
- }
- if !strings.Contains(result, "<th>name</th>") {
- t.Fatalf("row headers were not rendered: %q", result)
- }
- if got := strings.Count(result, "<td>alpha</td>"); got != 1 {
- t.Fatalf("first row rendered %d times, want once: %q", got, result)
- }
-}
-
-func TestParseResultText_SpreadsheetRowsWithoutIdentityStaySeparate(t *testing.T) {
- result, err := parseResultText(parser.ParseResult{
- OutputFormat: "json",
- JSON: []map[string]any{
- {
- "ck_type": "table_header",
- "cells": []string{"first"},
- },
- {
- "ck_type": "table_row",
- "cells": []string{"one"},
- },
- {
- "ck_type": "table_header",
- "cells": []string{"second"},
- },
- {
- "ck_type": "table_row",
- "cells": []string{"two"},
- },
- },
- })
- if err != nil {
- t.Fatalf("parseResultText: %v", err)
- }
- if got := strings.Count(result, "<table>"); got != 2 {
- t.Fatalf("unknown spreadsheet tables were merged: got %d tables in %q", got, result)
+ if result != segment {
+ t.Fatalf("parseResultText = %q, want the segment markup whole", result)
}
}