RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.
Go agent-harnessagent-native-databaseagentic-aiagentic-retrievalagentic-searchaiai-agentscontext-enginecontext-engineeringcontext-managementharness-engineeringknowledge-compilationragretrieval-augmented-generationretrieval-harnesssearch-harness
refactor(parser): unify spreadsheet tables on a segmented HTML wire (#20358)
jay777219b67a4112a70
diff --git a/internal/engine/infinity/chunk.go b/internal/engine/infinity/chunk.goindex dae7635a7b..28b4a6f5b2 100644--- a/internal/engine/infinity/chunk.go+++ b/internal/engine/infinity/chunk.go@@ -2983,7 +2983,7 @@ func numericRow(row interface{}, out *[]interface{}) bool {// flat []interface{} the hex encoder takes. Two shapes reach the engine: values// decoded from JSON arrive as []interface{} (numbers as float64), while a chunk// built IN PROCESS carries Go's typed slices (the ingestion pipeline's-// AddPositions emits []int / [][]int). Both must be encoded, because these+// addPDFPositions emits []int / [][]int). Both must be encoded, because these// columns are VARCHAR holding the hex form: handing Infinity the typed slice// makes it try to store an int64 tensor and fail with// "Not support to convert Tensor(int64,5) to Varchar" (InfinityException 3049).@@ -3154,7 +3154,7 @@ func transformChunkFields(chunk map[string]interface{}, embeddingCols [][2]intercase "position_int", "page_num_int", "top_int":// Python flattens the position rows and hex-encodes every number// (infinity_conn.py: `[num for row in v for num in row]`, "%08x").- // The input may be Go-NATIVE: the ingestion pipeline's AddPositions+ // The input may be Go-NATIVE: the ingestion pipeline's addPDFPositions// emits []int / [][]int, and those used to miss the// []interface{}-only branch and reach Infinity as a raw tensor// ("Not support to convert Tensor(int64,5) to Varchar", 3049).diff --git a/internal/engine/infinity/chunk_transform_test.go b/internal/engine/infinity/chunk_transform_test.goindex 69afb7f302..6543b34f49 100644--- a/internal/engine/infinity/chunk_transform_test.go+++ b/internal/engine/infinity/chunk_transform_test.go@@ -93,7 +93,7 @@ func TestTransformChunkFields_IngestionShape(t *testing.T) {}// TestTransformChunkFieldsHexEncodesNativeIntSlices pins the Go-native slice-// shapes the ingestion pipeline produces: AddPositions emits []int / [][]int+// shapes the ingestion pipeline produces: addPDFPositions emits []int / [][]int// (internal/ingestion/task/indexdoc/position.go), and the Infinity columns are// VARCHAR holding the hex form. Left unconverted, Infinity rejects the insert// with "Not support to convert Tensor(int64,5) to Varchar" (3049).diff --git a/internal/ingestion/component/chunker/coverage_blind_spots_test.go b/internal/ingestion/component/chunker/coverage_blind_spots_test.goindex ed800d4f83..c1c9d4023f 100644--- a/internal/ingestion/component/chunker/coverage_blind_spots_test.go+++ b/internal/ingestion/component/chunker/coverage_blind_spots_test.go@@ -160,14 +160,13 @@ func TestGeneralChunkerKeepsHeaderOnlySheetsSeparate(t *testing.T) {if err != nil {t.Fatalf("NewGeneralChunker: %v", err)}+ segOne := spreadsheetSegmentItem("Sheet1", []string{"Name"}, nil, 1, 1)+ segTwo := spreadsheetSegmentItem("Sheet2", []string{"Amount"}, nil, 2, 1)out, err := chunker.Invoke(t.Context(), nil, map[string]any{"name": "empty-sheets.xlsx","file_type": "xlsx","output_format": "json",- "json": []map[string]any{- {"text": "Name", "doc_type_kwd": "table", "ck_type": "table_header", "sheet_index": 1, "table_id": "sheet-1"},- {"text": "Amount", "doc_type_kwd": "table", "ck_type": "table_header", "sheet_index": 2, "table_id": "sheet-2"},- },+ "json": []map[string]any{segOne, segTwo},})if err != nil {t.Fatalf("GeneralChunker.Invoke: %v", err)@@ -176,7 +175,7 @@ func TestGeneralChunkerKeepsHeaderOnlySheetsSeparate(t *testing.T) {if len(chunks) != 2 {t.Fatalf("chunks = %#v, want one header-only chunk per sheet", chunks)}- if chunks[0]["text"] != "Name" || chunks[1]["text"] != "Amount" {+ if chunks[0]["text"] != segOne["text"] || chunks[1]["text"] != segTwo["text"] {t.Fatalf("header-only chunks = %#v", chunks)}}diff --git a/internal/ingestion/component/chunker/general.go b/internal/ingestion/component/chunker/general.goindex ba9c6747ea..9700c159b2 100644--- a/internal/ingestion/component/chunker/general.go+++ b/internal/ingestion/component/chunker/general.go@@ -463,11 +463,11 @@ func collectGeneralMediaContext(units []schema.ChunkDoc, index, budget int, abovif generalMediaContextCrossesSpreadsheetBoundary(units[index], units[cursor]) {break}- if itemDocType(units[cursor]) != "text" {+ text, ok := generalContextSourceText(units[cursor])+ if !ok {continue}- text := units[cursor].Text- tokens := generalUnitTokens(units[cursor])+ tokens := tokenizeStr(text)if tokens >= remaining {piece := takeContextSentences(text, remaining, above)if above {@@ -487,6 +487,41 @@ func collectGeneralMediaContext(units []schema.ChunkDoc, index, budget int, abovreturn strings.Join(parts, "\n")}+// generalContextSourceText returns the readable text a unit contributes as+// media context. Text units contribute their body; spreadsheet table+// segments contribute their data rows' cell text — the row IR's loose text+// rows were the image context before the HTML wire unification, and the+// repeating header stays out of the context exactly as it did then. Only+// spreadsheet segments qualify: attachGeneralMediaContext is shared with the+// PDF and DOCX paths, where a table item carries no identity and must keep+// its previous behavior (no context contribution).+func generalContextSourceText(doc schema.ChunkDoc) (string, bool) {+ switch itemDocType(doc) {+ case "text":+ return doc.Text, true+ case "table":+ if !hasSpreadsheetIdentity(doc) || !isTableStrictHTML(doc.Text) {+ return "", false+ }+ rows, headerCount := tableRowsWithHeader(doc.Text)+ lines := make([]string, 0, len(rows)-headerCount)+ for _, row := range rows[headerCount:] {+ cells := make([]string, 0, len(row))+ for _, cell := range row {+ if cell != "" {+ cells = append(cells, cell)+ }+ }+ if len(cells) > 0 {+ lines = append(lines, strings.Join(cells, " "))+ }+ }+ return strings.Join(lines, "\n"), true+ default:+ return "", false+ }+}+func generalMediaContextCrossesSpreadsheetBoundary(media, candidate schema.ChunkDoc) bool {if !hasSpreadsheetIdentity(media) && !hasSpreadsheetIdentity(candidate) {return false@@ -495,7 +530,7 @@ func generalMediaContextCrossesSpreadsheetBoundary(media, candidate schema.Chunk}func hasSpreadsheetIdentity(doc schema.ChunkDoc) bool {- return doc.TableID != "" || doc.Sheet != "" || doc.SheetIndex != nil+ return doc.Sheet != "" || doc.SheetIndex != nil}// takeContextSentences returns the whole sentences at one end of text whose@@ -815,44 +850,13 @@ func (c *GeneralChunkerComponent) chunkSpreadsheet(ctx context.Context, upstream}attachGeneralMediaContext(units, c.param.TableContextSize, c.param.ImageContextSize)chunks := make([]schema.ChunkDoc, 0, len(units))- pendingRows := make([]schema.ChunkDoc, 0)- var pendingHeader *schema.ChunkDoc- flushRows := func() {- if len(pendingRows) == 0 {- return- }- chunks = append(chunks, mergeSpreadsheetRows(pendingRows, c.param.ChunkTokenSize)...)- pendingRows = pendingRows[:0]- }- flushHeader := func() {- if pendingHeader == nil {- return- }- chunks = append(chunks, cloneChunkDoc(*pendingHeader))- pendingHeader = nil- }for _, unit := range units {- if unit.CKType == "table_header" {- flushRows()- flushHeader()- header := cloneChunkDoc(unit)- pendingHeader = &header- continue- }- if unit.CKType == "table_row" {- // A header immediately followed by row units is parser metadata for- // that table, not an additional data chunk. If no row follows, the- // deferred header is emitted as a header-only table below.- pendingHeader = nil- pendingRows = append(pendingRows, unit)+ if itemDocType(unit) == "table" && isTableStrictHTML(unit.Text) {+ chunks = append(chunks, c.splitSpreadsheetTable(unit)...)continue}- flushRows()- flushHeader()chunks = append(chunks, cloneChunkDoc(unit))}- flushRows()- flushHeader()childrenPattern := compileChildrenPattern(c.param.ChildrenDelimiters)chunks = finalizeGeneralChunks(chunks, childrenPattern)if len(chunks) == 0 {@@ -861,6 +865,64 @@ func (c *GeneralChunkerComponent) chunkSpreadsheet(ctx context.Context, upstreamreturn chunkOutputs(chunks), nil}+// splitSpreadsheetTable enforces the ChunkTokenSize budget on a table+// segment: sub-tables replicate the caption and header rows, and the+// row-aligned positions matrix is sliced per sub-table (R11) so each chunk+// carries exactly its own rows' tuples. A segment that fits is emitted+// whole; a split whose matrix cannot be aligned to the markup rows drops+// positions rather than pointing every sub-table row at the full matrix (R1).+//+// The legacy path merged adjacent rows by token budget into multi-row chunks+// and deferred the header; a segment now passes through whole unless the+// budget forces a cut, and the header repeats in every part.+func (c *GeneralChunkerComponent) splitSpreadsheetTable(unit schema.ChunkDoc) []schema.ChunkDoc {+ parts, ranges, headerRows := splitLargeHTMLTable(unit.Text, c.param.ChunkTokenSize, tokenizeStr)+ if ranges == nil {+ return []schema.ChunkDoc{cloneChunkDoc(unit)}+ }+ var matrix [][]float64+ // Spreadsheet items only: PDF items write layout boxes into the same+ // field, and those must never be sliced into sub-tables as row tuples.+ if unit.SheetIndex != nil && len(unit.Positions) > 0 {+ if err := json.Unmarshal(unit.Positions, &matrix); err != nil {+ matrix = nil+ }+ }+ // The matrix carries one five-field tuple per <tr>; the splitter cut those+ // same rows, so its last range end is the total data-row count — no second+ // full parse of the markup is needed to compare lengths.+ aligned := false+ if len(ranges) > 0 {+ if expected := headerRows + ranges[len(ranges)-1][1]; expected > 0 && len(matrix) == expected {+ aligned = true+ for _, tuple := range matrix {+ if len(tuple) != 5 {+ aligned = false+ break+ }+ }+ }+ }+ out := make([]schema.ChunkDoc, 0, len(parts))+ for i, part := range parts {+ piece := cloneChunkDoc(unit)+ piece.Text = part+ piece.Positions = nil+ lo := headerRows + ranges[i][0]+ hi := headerRows + ranges[i][1]+ if aligned && hi <= len(matrix) {+ sub := make([][]float64, 0, headerRows+hi-lo)+ sub = append(sub, matrix[:headerRows]...)+ sub = append(sub, matrix[lo:hi]...)+ if encoded, err := json.Marshal(sub); err == nil {+ piece.Positions = encoded+ }+ }+ out = append(out, piece)+ }+ return out+}+func (c *GeneralChunkerComponent) chunkGeneral(ctx context.Context, upstream schema.ChunkerFromUpstream) (map[string]any, error) {if err := ctx.Err(); err != nil {return nil, fmt.Errorf("GeneralChunker: %w", err)@@ -933,32 +995,7 @@ func mergeGeneralUnits(units []schema.ChunkDoc, target int, overlapPct float64,return applyGeneralOverlap(merged, overlapPct, joinSep)}-func mergeSpreadsheetRows(rows []schema.ChunkDoc, target int) []schema.ChunkDoc {- merged := make([]schema.ChunkDoc, 0, len(rows))- current := -1- for _, row := range rows {- if row.CKType != "table_row" || itemDocType(row) != "text" {- merged = append(merged, cloneChunkDoc(row))- current = -1- continue- }- count := generalUnitTokens(row)- if current < 0 || count > target || !sameSpreadsheetTable(merged[current], row) || intValue(merged[current].TKNums)+count > target {- row.TKNums = intPtr(count)- merged = append(merged, cloneChunkDoc(row))- current = len(merged) - 1- continue- }- mergeGeneralChunk(&merged[current], row, "\n")- mergeSpreadsheetRowRange(&merged[current], row)- }- return merged-}-func sameSpreadsheetTable(first, second schema.ChunkDoc) bool {- if first.TableID != "" || second.TableID != "" {- return first.TableID == second.TableID- }if first.SheetIndex != nil || second.SheetIndex != nil {return first.SheetIndex != nil && second.SheetIndex != nil && *first.SheetIndex == *second.SheetIndex}@@ -980,32 +1017,6 @@ func spreadsheetPositionSheet(doc schema.ChunkDoc) (float64, bool) {return row[0], true}-func mergeSpreadsheetRowRange(dst *schema.ChunkDoc, src schema.ChunkDoc) {- dst.RowStart = minSpreadsheetInt(dst.RowStart, src.RowStart, false)- dst.RowEnd = minSpreadsheetInt(dst.RowEnd, src.RowEnd, true)- dst.ColStart = minSpreadsheetInt(dst.ColStart, src.ColStart, false)- dst.ColEnd = minSpreadsheetInt(dst.ColEnd, src.ColEnd, true)-}--func minSpreadsheetInt(first, second *int, maximum bool) *int {- if first == nil {- if second == nil {- return nil- }- value := *second- return &value- }- if second == nil {- value := *first- return &value- }- value := *first- if (maximum && *second > value) || (!maximum && *second < value) {- value = *second- }- return &value-}-func generalUnitTokens(unit schema.ChunkDoc) int {if unit.TKNums != nil && *unit.TKNums > 0 {return *unit.TKNumsdiff --git a/internal/ingestion/component/chunker/general_merge_test.go b/internal/ingestion/component/chunker/general_merge_test.goindex b8bf8d5702..53d8e21d1d 100644--- a/internal/ingestion/component/chunker/general_merge_test.go+++ b/internal/ingestion/component/chunker/general_merge_test.go@@ -42,65 +42,17 @@ func TestMergeGeneralUnitsAllowsOneBoundaryOverflow(t *testing.T) {}}-func TestMergeSpreadsheetRowsUsesProjectedCapAndSheetBoundary(t *testing.T) {- sheetOne := 1- sheetTwo := 2- rows := []schema.ChunkDoc{- {Text: "row-1", DocType: "text", CKType: "table_row", TKNums: intPtr(2), SheetIndex: &sheetOne, RowStart: intPtr(2), RowEnd: intPtr(2), ColStart: intPtr(1), ColEnd: intPtr(2)},- {Text: "row-2", DocType: "text", CKType: "table_row", TKNums: intPtr(1), SheetIndex: &sheetOne, RowStart: intPtr(3), RowEnd: intPtr(3), ColStart: intPtr(1), ColEnd: intPtr(2)},- {Text: "row-3", DocType: "text", CKType: "table_row", TKNums: intPtr(1), SheetIndex: &sheetTwo, RowStart: intPtr(2), RowEnd: intPtr(2), ColStart: intPtr(1), ColEnd: intPtr(2)},- }-- got := mergeSpreadsheetRows(rows, 3)- if texts := generalChunkTexts(got); !reflect.DeepEqual(texts, []string{"row-1\nrow-2", "row-3"}) {- t.Fatalf("texts = %q", texts)- }- if len(got) != 2 || got[0].TKNums == nil || *got[0].TKNums != 3 {- t.Fatalf("merged row token count = %#v, want 3", got)- }- if got[0].RowStart == nil || *got[0].RowStart != 2 || got[0].RowEnd == nil || *got[0].RowEnd != 3 {- t.Fatalf("merged row range = start:%v end:%v", got[0].RowStart, got[0].RowEnd)- }- if got[0].SheetIndex == nil || *got[0].SheetIndex != 1 || got[1].SheetIndex == nil || *got[1].SheetIndex != 2 {- t.Fatalf("sheet boundaries not preserved: %#v", got)- }-}--func TestMergeSpreadsheetRowsKeepsOversizedRowAndCapZeroAtomic(t *testing.T) {- sheet := 1- rows := []schema.ChunkDoc{- {Text: "small", DocType: "text", CKType: "table_row", TKNums: intPtr(1), SheetIndex: &sheet, RowStart: intPtr(2), RowEnd: intPtr(2)},- {Text: "oversized", DocType: "text", CKType: "table_row", TKNums: intPtr(8), SheetIndex: &sheet, RowStart: intPtr(3), RowEnd: intPtr(3)},- {Text: "after", DocType: "text", CKType: "table_row", TKNums: intPtr(1), SheetIndex: &sheet, RowStart: intPtr(4), RowEnd: intPtr(4)},- }- if got := mergeSpreadsheetRows(rows, 3); !reflect.DeepEqual(generalChunkTexts(got), []string{"small", "oversized", "after"}) {- t.Fatalf("oversized texts = %q", generalChunkTexts(got))- }- if got := mergeSpreadsheetRows(rows, 0); !reflect.DeepEqual(generalChunkTexts(got), []string{"small", "oversized", "after"}) {- t.Fatalf("zero-cap texts = %q", generalChunkTexts(got))- }-}--func TestGeneralChunkerSpreadsheetConsumesRowIR(t *testing.T) {- component, err := NewGeneralChunker(map[string]any{"chunk_token_size": 3})- if err != nil {- t.Fatalf("NewGeneralChunker: %v", err)- }- out, err := component.Invoke(t.Context(), nil, map[string]any{- "name": "orders.xlsx",- "file_type": "xlsx",- "output_format": "json",- "json": []map[string]any{- {"text": "ID; Status", "doc_type_kwd": "table", "ck_type": "table_header", "sheet_index": 1, "table_id": "sheet-1"},- {"text": "ID:A-100; Status:paid", "doc_type_kwd": "text", "ck_type": "table_row", "tk_nums": 2, "sheet_index": 1, "table_id": "sheet-1", "row_start": 2, "row_end": 2},- {"text": "ID:A-101; Status:pending", "doc_type_kwd": "text", "ck_type": "table_row", "tk_nums": 1, "sheet_index": 1, "table_id": "sheet-1", "row_start": 3, "row_end": 3},- },- })- if err != nil {- t.Fatalf("Invoke: %v", err)- }- if texts := outputTexts(t, out); !reflect.DeepEqual(texts, []string{"ID:A-100; Status:paid\nID:A-101; Status:pending"}) {- t.Fatalf("spreadsheet chunks = %q", texts)+// TestSameSpreadsheetTableUsesPositionSheet covers the media-context sheet+// guard once carried by merged row runs: identity falls through to the+// positions matrix when the sheet fields are absent.+func TestSameSpreadsheetTableUsesPositionSheet(t *testing.T) {+ one := schema.ChunkDoc{Text: "sheet-1", Positions: json.RawMessage(`[[1,2,2,1,2]]`)}+ two := schema.ChunkDoc{Text: "sheet-2", Positions: json.RawMessage(`[[2,2,2,1,2]]`)}+ if !sameSpreadsheetTable(one, one) {+ t.Errorf("same position sheet rejected")+ }+ if sameSpreadsheetTable(one, two) {+ t.Errorf("different position sheets accepted")}}@@ -171,18 +123,6 @@ func TestApplyGeneralOverlapDoesNotAccumulatePositions(t *testing.T) {}}-func TestMergeSpreadsheetRowsUsesPositionSheetWhenIdentityFieldsAreMissing(t *testing.T) {- rows := []schema.ChunkDoc{- {Text: "sheet-1-row", DocType: "text", CKType: "table_row", TKNums: intPtr(1), Positions: json.RawMessage(`[[1,2,2,1,2]]`)},- {Text: "sheet-2-row", DocType: "text", CKType: "table_row", TKNums: intPtr(1), Positions: json.RawMessage(`[[2,2,2,1,2]]`)},- }-- got := mergeSpreadsheetRows(rows, 10)- if texts := generalChunkTexts(got); !reflect.DeepEqual(texts, []string{"sheet-1-row", "sheet-2-row"}) {- t.Fatalf("position-only sheet boundary was lost: texts = %q", texts)- }-}-func TestMergeMarkdownUnitsCarriesCharacterOverlapIntoNextBudget(t *testing.T) {units := []schema.ChunkDoc{{Text: "alpha beta", DocType: "text", CKType: "text"},diff --git a/internal/ingestion/component/chunker/general_test.go b/internal/ingestion/component/chunker/general_test.goindex e77939bb7a..455fa6f1cf 100644--- a/internal/ingestion/component/chunker/general_test.go+++ b/internal/ingestion/component/chunker/general_test.go@@ -38,6 +38,43 @@ import ("ragflow/internal/ingestion/task/indexdoc")+// spreadsheetSegmentItem builds one wire-format spreadsheet table segment:+// the markup the parser renders (captioned <table>, <th> header, <td> rows)+// plus the row-aligned positions matrix (one tuple per <tr>, header first).+// firstRow is the sheet row of the first DATA row; the header tuple covers+// firstRow-1.+func spreadsheetSegmentItem(sheet string, header []string, rows [][]string, sheetIndex, firstRow int) map[string]any {+ var markup strings.Builder+ markup.WriteString("<table><caption>" + sheet + "</caption>\n<tr>")+ for _, cell := range header {+ markup.WriteString("<th>" + cell + "</th>")+ }+ markup.WriteString("</tr>\n")+ for _, row := range rows {+ markup.WriteString("<tr>")+ for _, cell := range row {+ markup.WriteString("<td>" + cell + "</td>")+ }+ markup.WriteString("</tr>\n")+ }+ markup.WriteString("</table>\n")++ tuple := func(row int) []float64 {+ return []float64{float64(sheetIndex), float64(row), float64(row), 1, float64(len(header))}+ }+ matrix := [][]float64{tuple(firstRow - 1)}+ for i := range rows {+ matrix = append(matrix, tuple(firstRow+i))+ }+ return map[string]any{+ "text": markup.String(),+ "doc_type_kwd": "table",+ "ck_type": "table",+ "sheet_index": sheetIndex,+ "positions": matrix,+ }+}+func TestGeneralChunkerRegistered(t *testing.T) {factory, category, metadata, ok := runtime.DefaultRegistry.Lookup(ComponentNameGeneralChunker)if !ok {@@ -753,18 +790,12 @@ func TestGeneralChunkerSpreadsheetHeaderOnlyPreservesHeaderChunk(t *testing.T) {if err != nil {t.Fatalf("NewGeneralChunker: %v", err)}+ item := spreadsheetSegmentItem("Sheet1", []string{"Name", "Amount"}, nil, 1, 1)out, err := component.Invoke(t.Context(), nil, map[string]any{"name": "headers.xlsx","file_type": "xlsx","output_format": "json",- "json": []map[string]any{{- "text": "Name; Amount",- "doc_type_kwd": "table",- "ck_type": "table_header",- "sheet_index": 1,- "table_id": "sheet-1",- "positions": []any{[]any{1.0, 1.0, 1.0, 1.0, 2.0}},- }},+ "json": []map[string]any{item},})if err != nil {t.Fatalf("Invoke: %v", err)@@ -773,14 +804,14 @@ func TestGeneralChunkerSpreadsheetHeaderOnlyPreservesHeaderChunk(t *testing.T) {if len(chunks) != 1 {t.Fatalf("chunks = %#v, want one header-only chunk", chunks)}- if chunks[0]["text"] != "Name; Amount" || chunks[0]["ck_type"] != "table_header" {+ if chunks[0]["text"] != item["text"] || chunks[0]["ck_type"] != "table" {t.Fatalf("header-only chunk = %#v", chunks[0])}}func TestGeneralChunkerSpreadsheetAttachesImageContext(t *testing.T) {component, err := NewGeneralChunker(map[string]any{- "chunk_token_size": 10,+ "chunk_token_size": 100,"image_context_size": 10,})if err != nil {@@ -791,9 +822,9 @@ func TestGeneralChunkerSpreadsheetAttachesImageContext(t *testing.T) {"file_type": "xlsx","output_format": "json","json": []map[string]any{- {"text": "Revenue", "doc_type_kwd": "text", "ck_type": "table_row", "table_id": "sheet-1", "sheet_index": 1, "tk_nums": 1},- {"text": "B2", "doc_type_kwd": "image", "ck_type": "image", "image": "figure", "table_id": "sheet-1", "sheet_index": 1},- {"text": "Growth", "doc_type_kwd": "text", "ck_type": "table_row", "table_id": "sheet-1", "sheet_index": 1, "tk_nums": 1},+ spreadsheetSegmentItem("Sheet1", []string{"Name"}, [][]string{{"Revenue"}}, 1, 2),+ {"text": "B2", "doc_type_kwd": "image", "ck_type": "image", "image": "figure", "sheet_index": 1},+ spreadsheetSegmentItem("Sheet1", []string{"Name"}, [][]string{{"Growth"}}, 1, 3),},})if err != nil {@@ -801,7 +832,7 @@ func TestGeneralChunkerSpreadsheetAttachesImageContext(t *testing.T) {}chunks := outputChunks(t, out)if len(chunks) != 3 {- t.Fatalf("chunks = %#v, want row, image, row", chunks)+ t.Fatalf("chunks = %#v, want segment, image, segment", chunks)}if chunks[1]["ck_type"] != "image" {t.Fatalf("image chunk = %#v", chunks[1])@@ -810,7 +841,7 @@ func TestGeneralChunkerSpreadsheetAttachesImageContext(t *testing.T) {}func TestGeneralChunkerSpreadsheetImageContextStopsAtSheetBoundary(t *testing.T) {- component, err := NewGeneralChunker(map[string]any{"image_context_size": 10})+ component, err := NewGeneralChunker(map[string]any{"chunk_token_size": 100, "image_context_size": 10})if err != nil {t.Fatalf("NewGeneralChunker: %v", err)}@@ -819,9 +850,9 @@ func TestGeneralChunkerSpreadsheetImageContextStopsAtSheetBoundary(t *testing.T)"file_type": "xlsx","output_format": "json","json": []map[string]any{- {"text": "Sheet one", "doc_type_kwd": "text", "ck_type": "table_row", "table_id": "sheet-1", "sheet_index": 1, "tk_nums": 1},- {"text": "B2", "doc_type_kwd": "image", "ck_type": "image", "image": "figure", "table_id": "sheet-1", "sheet_index": 1},- {"text": "Sheet two", "doc_type_kwd": "text", "ck_type": "table_row", "table_id": "sheet-2", "sheet_index": 2, "tk_nums": 1},+ spreadsheetSegmentItem("Sheet1", []string{"Name"}, [][]string{{"Sheet one"}}, 1, 2),+ {"text": "B2", "doc_type_kwd": "image", "ck_type": "image", "image": "figure", "sheet_index": 1},+ spreadsheetSegmentItem("Sheet2", []string{"Name"}, [][]string{{"Sheet two"}}, 2, 2),},})if err != nil {@@ -829,11 +860,104 @@ func TestGeneralChunkerSpreadsheetImageContextStopsAtSheetBoundary(t *testing.T)}chunks := outputChunks(t, out)if len(chunks) != 3 {- t.Fatalf("chunks = %#v, want row, image, row", chunks)+ t.Fatalf("chunks = %#v, want segment, image, segment", chunks)}assertMaterializedMediaContext(t, chunks[1], "Sheet oneB2")}+// TestGeneralChunkerSpreadsheetSplitsLargeSegmentIntoBoundedChunks pins the+// chunker-side size budget: a segment over ChunkTokenSize is cut into+// sub-tables that each replicate the header row and carry only their own+// rows' position tuples (R11). Without the cut a wide/long row runs past the+// embedding truncation point and its tail rows silently disappear.+func TestGeneralChunkerSpreadsheetSplitsLargeSegmentIntoBoundedChunks(t *testing.T) {+ component, err := NewGeneralChunker(map[string]any{"chunk_token_size": 20})+ if err != nil {+ t.Fatalf("NewGeneralChunker: %v", err)+ }+ var rows [][]string+ for i := 0; i < 6; i++ {+ rows = append(rows, []string{fmt.Sprintf("value-%d-with-some-padding", i)})+ }+ item := spreadsheetSegmentItem("Sheet1", []string{"Name"}, rows, 1, 2)+ out, err := component.Invoke(t.Context(), nil, map[string]any{+ "name": "long.xlsx",+ "file_type": "xlsx",+ "output_format": "json",+ "json": []map[string]any{item},+ })+ if err != nil {+ t.Fatalf("Invoke: %v", err)+ }+ chunks := outputChunks(t, out)+ if len(chunks) < 2 {+ t.Fatalf("chunks = %d, want the segment split under the token budget", len(chunks))+ }+ covered := 0+ for i, chunk := range chunks {+ text, _ := chunk["text"].(string)+ if !strings.Contains(text, "<th>Name</th>") {+ t.Errorf("chunk %d lost the replicated header", i)+ }+ var matrix [][]float64+ raw, _ := json.Marshal(chunk["positions"])+ if err := json.Unmarshal(raw, &matrix); err != nil {+ t.Fatalf("chunk %d positions = %v, want a row-aligned matrix", i, chunk["positions"])+ }+ // One header tuple + this sub-table's own data tuples, never the+ // full segment matrix.+ if len(matrix) < 2 {+ t.Fatalf("chunk %d matrix = %v, want header plus data rows", i, matrix)+ }+ if matrix[0][1] != 1 {+ t.Errorf("chunk %d header tuple = %v, want sheet 1 row 1", i, matrix[0])+ }+ trCount := strings.Count(text, "<tr>")+ if trCount != len(matrix) {+ t.Errorf("chunk %d has %d rows but %d position tuples", i, trCount, len(matrix))+ }+ covered += len(matrix) - 1+ }+ if covered != len(rows) {+ t.Errorf("sub-tables cover %d data rows, want %d", covered, len(rows))+ }+}++// TestGeneralChunkerSpreadsheetDropsMisalignedPositionsOnSplit: when the+// matrix cannot be aligned one tuple per row, a split drops positions+// entirely rather than pointing every sub-table row at the full matrix (R1).+func TestGeneralChunkerSpreadsheetDropsMisalignedPositionsOnSplit(t *testing.T) {+ component, err := NewGeneralChunker(map[string]any{"chunk_token_size": 20})+ if err != nil {+ t.Fatalf("NewGeneralChunker: %v", err)+ }+ var rows [][]string+ for i := 0; i < 6; i++ {+ rows = append(rows, []string{fmt.Sprintf("value-%d-with-some-padding", i)})+ }+ item := spreadsheetSegmentItem("Sheet1", []string{"Name"}, rows, 1, 2)+ // One whole-segment tuple instead of the per-row matrix.+ item["positions"] = [][]float64{{1, 1, 7, 1, 1}}+ out, err := component.Invoke(t.Context(), nil, map[string]any{+ "name": "long.xlsx",+ "file_type": "xlsx",+ "output_format": "json",+ "json": []map[string]any{item},+ })+ if err != nil {+ t.Fatalf("Invoke: %v", err)+ }+ chunks := outputChunks(t, out)+ if len(chunks) < 2 {+ t.Fatalf("chunks = %d, want the segment split under the token budget", len(chunks))+ }+ for i, chunk := range chunks {+ if _, exists := chunk["positions"]; exists {+ t.Errorf("chunk %d keeps a misaligned positions payload: %v", i, chunk["positions"])+ }+ }+}+// TestGeneralChunkerMediaContextReachesChunkIDAndIndexContent pins the// downstream half of the fold: once the context is part of the chunk body, it// is also part of the persisted content_with_weight and of the chunk id@@ -955,3 +1079,96 @@ func TestGeneralChunkerPDFAttachesOutlineOnce(t *testing.T) {t.Errorf("outline entry = %#v", entry)}}++// TestGeneralChunkerNonSpreadsheetTableNotMediaContext: a table item without+// spreadsheet identity must not become a media-context source. The DOCX and+// PDF strategies share the collector, so reading their tables as context would+// change those documents' image chunks (and their chunk ids) as a side effect+// of the spreadsheet wire switch.+func TestGeneralChunkerNonSpreadsheetTableNotMediaContext(t *testing.T) {+ component, err := NewGeneralChunker(map[string]any{+ "chunk_token_size": 100,+ "image_context_size": 10,+ })+ if err != nil {+ t.Fatalf("NewGeneralChunker: %v", err)+ }+ table := map[string]any{+ "text": "<table><caption>Doc</caption>\n<tr><th>Name</th></tr>\n<tr><td>Revenue</td></tr>\n</table>\n",+ "doc_type_kwd": "table",+ "ck_type": "table",+ }+ out, err := component.Invoke(t.Context(), nil, map[string]any{+ "name": "document.docx",+ "file_type": "docx",+ "output_format": "json",+ "json": []map[string]any{+ table,+ {"text": "B2", "doc_type_kwd": "image", "ck_type": "image", "image": "figure"},+ table,+ },+ })+ if err != nil {+ t.Fatalf("Invoke: %v", err)+ }+ chunks := outputChunks(t, out)+ if len(chunks) != 3 {+ t.Fatalf("chunks = %#v, want table, image, table", chunks)+ }+ if chunks[1]["ck_type"] != "image" {+ t.Fatalf("image chunk = %#v", chunks[1])+ }+ if got, _ := chunks[1]["text"].(string); got != "B2" {+ t.Errorf("image chunk text = %q, want B2 with no table text folded in", got)+ }+ for _, key := range []string{"context_above", "context_below"} {+ if _, exists := chunks[1][key]; exists {+ t.Errorf("image chunk must not carry %s: %v", key, chunks[1][key])+ }+ }+}++// TestGeneralChunkerSpreadsheetSplitDropsNonSpreadsheetPositions: sub-tables+// only receive sliced positions when the item carries spreadsheet identity and+// the matrix is five-field tuples aligned to the splitter's rows; a PDF-style+// matrix or a short tuple is dropped rather than misattributed.+func TestGeneralChunkerSpreadsheetSplitDropsNonSpreadsheetPositions(t *testing.T) {+ rows := [][]string{{"v0", "p0"}, {"v1", "p1"}, {"v2", "p2"}, {"v3", "p3"}, {"v4", "p4"}, {"v5", "p5"}}+ cases := []struct {+ name string+ mutate func(item map[string]any)+ }{+ {"no spreadsheet identity", func(item map[string]any) { delete(item, "sheet_index") }},+ {"short tuples", func(item map[string]any) {+ item["positions"] = [][]float64{{1}, {1}, {1}, {1}, {1}, {1}, {1}}+ }},+ }+ for _, tc := range cases {+ t.Run(tc.name, func(t *testing.T) {+ component, err := NewGeneralChunker(map[string]any{"chunk_token_size": 20})+ if err != nil {+ t.Fatalf("NewGeneralChunker: %v", err)+ }+ item := spreadsheetSegmentItem("Sheet1", []string{"name", "price"}, rows, 1, 2)+ tc.mutate(item)+ out, err := component.Invoke(t.Context(), nil, map[string]any{+ "name": "long.xlsx",+ "file_type": "xlsx",+ "output_format": "json",+ "json": []map[string]any{item},+ })+ if err != nil {+ t.Fatalf("Invoke: %v", err)+ }+ chunks := outputChunks(t, out)+ if len(chunks) < 2 {+ t.Fatalf("expected the segment to be split, got %d chunks", len(chunks))+ }+ for i, chunk := range chunks {+ if p, ok := chunk["positions"]; ok && p != nil && fmt.Sprintf("%v", p) != "[]" && fmt.Sprintf("%v", p) != "<nil>" {+ t.Errorf("chunk %d kept positions without a usable spreadsheet matrix: %v", i, p)+ }+ }+ })+ }+}diff --git a/internal/ingestion/component/chunker/html_rows.go b/internal/ingestion/component/chunker/html_rows.gonew file mode 100644index 0000000000..a1874deac0--- /dev/null+++ b/internal/ingestion/component/chunker/html_rows.go@@ -0,0 +1,168 @@+//+// Copyright 2026 The InfiniFlow Authors. All Rights Reserved.+//+// Licensed under the Apache License, Version 2.0 (the "License");+// you may not use this file except in compliance with the License.+// You may obtain a copy of the License at+//+// http://www.apache.org/licenses/LICENSE-2.0+//+// Unless required by applicable law or agreed to in writing, software+// distributed under the License is distributed on an "AS IS" BASIS,+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+// See the License for the specific language governing permissions and+// limitations under the License.+//++package chunker++import (+ "strings"++ "golang.org/x/net/html"+)++// ---------------------------------------------------------------------------+// HTML table row extraction (shared by the Table, QA and General chunkers)+// ---------------------------------------------------------------------------++// tableRows returns every row's cell text, header rows included.+func tableRows(htmlStr string) [][]string {+ rows, _ := tableRowsWithHeader(htmlStr)+ return rows+}++// tableRowsWithHeader returns every row's cell text in document order, plus+// the number of leading header rows. A row counts as a header row when any of+// its cells is a <th>. When no row uses <th> at all the first row is treated+// as the header — the convention spreadsheet and markup tables are read with.+//+// The markup is parsed into a tree rather than matched with a regex because+// this input is not guaranteed to be well formed: seven parsers render table+// items (xlsx, csv, docx, html, pdf, …) and some of that markup originates+// from user-supplied documents. A tree also settles the cases a tag-level+// scan gets wrong: a nested <table> no longer terminates its enclosing row+// early — that row keeps its own cells, with the nested table's text folded+// into the cell holding it — and a row or cell missing its closing tag is+// recovered rather than dropped.+func tableRowsWithHeader(htmlStr string) (rows [][]string, headerCount int) {+ // A <tr> outside a <table> is discarded by the HTML5 "in body" insertion+ // mode, so a bare row fragment would yield nothing. Give the parser the+ // table context it needs instead of dropping the rows silently.+ lower := strings.ToLower(htmlStr)+ if strings.Contains(lower, "<tr") && !strings.Contains(lower, "<table") {+ htmlStr = "<table>" + htmlStr + "</table>"+ }+ doc, err := html.Parse(strings.NewReader(htmlStr))+ if err != nil {+ return nil, 0+ }+ var headerFlags []bool+ var walk func(*html.Node)+ walk = func(n *html.Node) {+ // An inert subtree is parsed but never rendered. <template> puts its+ // content straight into the ordinary child list (the parser has no+ // separate template-contents field), so without this its rows would+ // be read as rows of the enclosing table.+ if n.Type == html.ElementNode && isInertElement(n.Data) {+ return+ }+ if isHTMLElement(n, "tr") {+ var cells []string+ anyTh := false+ for c := n.FirstChild; c != nil; c = c.NextSibling {+ if isHTMLElement(c, "td") || isHTMLElement(c, "th") {+ if isHTMLElement(c, "th") {+ anyTh = true+ }+ cells = append(cells, cellText(c))+ }+ }+ // Return without descending: the cells above already collected+ // the nested table's text, so its rows must not be reported a+ // second time as rows of the enclosing table.+ rows = append(rows, cells)+ headerFlags = append(headerFlags, anyTh)+ return+ }+ for c := n.FirstChild; c != nil; c = c.NextSibling {+ walk(c)+ }+ }+ walk(doc)+ for len(headerFlags) > headerCount && headerFlags[headerCount] {+ headerCount+++ }+ if len(rows) > 0 && headerCount == 0 {+ headerCount = 1+ }+ return rows, headerCount+}++// isTableStrictHTML reports whether block text is an outer <table> element+// (the inlined GFM/HTML table). Only such blocks are emitted as structured+// table items; other raw HTML (e.g. <div>, <style>) is plain text. The+// spreadsheet wire is the markup this reports on (see the parser's+// renderSpreadsheetTable), so callers that slice a positions matrix use+// this strict form rather than the candidate filter.+func isTableStrictHTML(s string) bool {+ return strings.HasPrefix(strings.TrimSpace(strings.ToLower(s)), "<table")+}++// isTableHTML is the cheap candidate filter for table-vs-prose routing: text+// that opens an outer <table> element, or a bare-row fragment holding <tr>.+// It only narrows the candidates — the caller decides on tableRows' result —+// so a block that merely opens with "<table" text and holds no row stays on+// the prose path instead of silently pairing nothing.+func isTableHTML(s string) bool {+ lower := strings.ToLower(s)+ return strings.HasPrefix(strings.TrimSpace(lower), "<table") || strings.Contains(lower, "<tr")+}++// cellText returns the visible text of a table cell. The parser hands text+// nodes over already unescaped, nested markup contributes its text without+// its tags (a nested table's cells are concatenated, not separated), and a+// <br> becomes a newline instead of silently gluing the two halves of the+// cell together.+func cellText(cell *html.Node) string {+ var sb strings.Builder+ var walk func(*html.Node)+ walk = func(n *html.Node) {+ switch {+ case n.Type == html.TextNode:+ sb.WriteString(n.Data)+ case isHTMLElement(n, "br"):+ sb.WriteByte('\n')+ case n.Type == html.ElementNode && isInertElement(n.Data):+ // Stop here rather than descending: the content is parsed but+ // never rendered, so it is not text a reader of the cell sees.+ return+ }+ for c := n.FirstChild; c != nil; c = c.NextSibling {+ walk(c)+ }+ }+ walk(cell)+ return strings.TrimSpace(sb.String())+}++// isHTMLElement reports whether n is an element with the given tag name in+// the HTML namespace. Foreign content reuses HTML tag names for unrelated+// elements — an <svg><tr> is not a table row — so matching on the tag name+// alone would read markup that carries no table semantics.+func isHTMLElement(n *html.Node, tag string) bool {+ return n.Type == html.ElementNode && n.Namespace == "" && n.Data == tag+}++// isInertElement reports whether an element's content is inert — parsed, but+// never rendered as visible text. An inline <script> or <style> inside a+// table cell of a user-supplied document would otherwise be embedded into the+// Q&A pair as if it were part of the sentence, and a <template>'s placeholder+// rows would be read as real ones.+func isInertElement(tag string) bool {+ switch tag {+ case "script", "style", "noscript", "template":+ return true+ }+ return false+}diff --git a/internal/ingestion/component/chunker/html_rows_test.go b/internal/ingestion/component/chunker/html_rows_test.gonew file mode 100644index 0000000000..76690052bb--- /dev/null+++ b/internal/ingestion/component/chunker/html_rows_test.go@@ -0,0 +1,70 @@+//+// Copyright 2026 The InfiniFlow Authors. All Rights Reserved.+//+// Licensed under the Apache License, Version 2.0 (the "License");+// you may not use this file except in compliance with the License.+// You may obtain a copy of the License at+//+// http://www.apache.org/licenses/LICENSE-2.0+//+// Unless required by applicable law or agreed to in writing, software+// distributed under the License is distributed on an "AS IS" BASIS,+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+// See the License for the specific language governing permissions and+// limitations under the License.+//++package chunker++import "testing"++// TestIsTableStrictHTML pins the predicate's prefix semantics: it reports+// whether the block opens with the "<table" text, exactly as the markdown+// parser has always classified its HTML blocks. It is deliberately loose (a+// "<tableau>" token passes), which is why routing never treats it as the final+// word — the row walker decides.+func TestIsTableStrictHTML(t *testing.T) {+ cases := []struct {+ text string+ want bool+ }{+ {"<table>", true},+ {" <TABLE class=\"x\">", true},+ {"<table-x>", true}, // historical prefix semantics: "<table" prefix only+ {"<tableau>", true},+ {"<div><table>", false},+ {"plain text", false},+ {"", false},+ }+ for _, tc := range cases {+ if got := isTableStrictHTML(tc.text); got != tc.want {+ t.Errorf("isTableStrictHTML(%q) = %v, want %v", tc.text, got, tc.want)+ }+ }+}++// TestIsTableHTMLCoversWhatTheWalkerReads guards the routing invariant: the+// candidate filter must never deny a payload the row walker can read, or a+// readable table would silently fall through to the prose extractor.+func TestIsTableHTMLCoversWhatTheWalkerReads(t *testing.T) {+ texts := []string{+ "<table><tr><th>q</th><th>a</th></tr></table>",+ "<table><caption>c</caption>\n<tr><td>q</td><td>a</td></tr>\n</table>\n",+ "<tr><td>q</td><td>a</td></tr>", // bare fragment+ "<tr><td>q</td><td>a</td></tr><tr><td>q2</td><td>a2</td></tr>", // bare rows+ "<tbody><tr><td>q</td><td>a</td></tr></tbody>", // rows wrapped in tbody+ "<table><tr><td>q<table><tr><td>inner</td></tr></table></td></tr></table>", // nested+ "<table><template><tr><td>x</td></tr></template></table>", // template rows are inert+ "<table></table>", // nothing to read+ "plain text",+ "",+ }+ for _, text := range texts {+ if len(tableRows(text)) == 0 {+ continue+ }+ if !isTableHTML(text) {+ t.Errorf("tableRows can read %q but isTableHTML denies it", text)+ }+ }+}diff --git a/internal/ingestion/component/chunker/pdfcrop_cgo.go b/internal/ingestion/component/chunker/pdfcrop_cgo.goindex 315a047b47..0bd5105cfd 100644--- a/internal/ingestion/component/chunker/pdfcrop_cgo.go+++ b/internal/ingestion/component/chunker/pdfcrop_cgo.go@@ -240,7 +240,7 @@ func cropImageChunks(ctx context.Context, engine deepdoctype.PDFEngine, chunks [// region (Python restore_pdf_text_previews). A pre-existing Image is never// re-cropped — cropImageChunks honors that separately.//-// CKType is the chunker-layer refinement (heading/table_header/table_row/text/+// CKType is the chunker-layer refinement (heading/text/image/table/// image/table). General and token chunkers always set it before cropping, so// they are handled by the CKType branch above. Group and hierarchy chunkers// forward the parser's output verbatim, which carries only the coarserdiff --git a/internal/ingestion/component/chunker/pdfcrop_cgo_test.go b/internal/ingestion/component/chunker/pdfcrop_cgo_test.goindex 784ebbe4e4..15447b1512 100644--- a/internal/ingestion/component/chunker/pdfcrop_cgo_test.go+++ b/internal/ingestion/component/chunker/pdfcrop_cgo_test.go@@ -86,7 +86,7 @@ func TestNeedsCrop(t *testing.T) {{"doc image fallback", schema.ChunkDoc{DocType: "image", PDFPositions: jsonPositions(t, []float64{1, 10, 100, 10, 100})}, true},{"doc table fallback", schema.ChunkDoc{DocType: "table", Positions: jsonPositions(t, []float64{1, 10, 100, 10, 100})}, true},{"doc text fallback", schema.ChunkDoc{DocType: "text", PDFPositions: jsonPositions(t, []float64{1, 10, 100, 10, 100})}, true},- // Heading / table_header / table_row all carry doc_type_kwd "text" but+ // Heading-tagged blocks carry doc_type_kwd "text" but// must NOT be previewed when CKType is empty — only a plain text body// chunk legitimately has CKType empty AND should be previewed. Since// group/hierarchy cannot distinguish heading from body at this layer,diff --git a/internal/ingestion/component/chunker/qa.go b/internal/ingestion/component/chunker/qa.goindex aa819d5127..9e9d36d785 100644--- a/internal/ingestion/component/chunker/qa.go+++ b/internal/ingestion/component/chunker/qa.go@@ -38,7 +38,6 @@ import ("github.com/gomarkdown/markdown""github.com/gomarkdown/markdown/parser"- "golang.org/x/net/html""gorm.io/gorm""ragflow/internal/agent/runtime"@@ -208,145 +207,6 @@ func isCSV(name string) bool {return strings.HasSuffix(strings.ToLower(name), ".csv")}-// ----------------------------------------------------------------------------// HTML / spreadsheet QA extraction-// -----------------------------------------------------------------------------// tableRows walks the parsed HTML and returns the <td>/<th> text of every-// <tr>, in document order.-//-// The markup is parsed into a tree rather than matched with a regex because-// this input is not guaranteed to be well formed: seven parsers render table-// items (xlsx, csv, docx, html, pdf, …) and some of that markup originates-// from user-supplied documents. A tree also settles the cases a tag-level-// scan gets wrong: a nested <table> no longer terminates its enclosing row-// early — that row keeps its own cells, with the nested table's text folded-// into the cell holding it — and a row or cell missing its closing tag is-// recovered rather than dropped.-func tableRows(htmlStr string) [][]string {- // A <tr> outside a <table> is discarded by the HTML5 "in body" insertion- // mode, so a bare row fragment would yield nothing. Give the parser the- // table context it needs instead of dropping the rows silently.- lower := strings.ToLower(htmlStr)- if strings.Contains(lower, "<tr") && !strings.Contains(lower, "<table") {- htmlStr = "<table>" + htmlStr + "</table>"- }- doc, err := html.Parse(strings.NewReader(htmlStr))- if err != nil {- return nil- }- var rows [][]string- var walk func(*html.Node)- walk = func(n *html.Node) {- // An inert subtree is parsed but never rendered. <template> puts its- // content straight into the ordinary child list (the parser has no- // separate template-contents field), so without this its rows would- // be read as rows of the enclosing table.- if n.Type == html.ElementNode && isInertElement(n.Data) {- return- }- if isHTMLElement(n, "tr") {- var cells []string- for c := n.FirstChild; c != nil; c = c.NextSibling {- if isHTMLElement(c, "td") || isHTMLElement(c, "th") {- cells = append(cells, cellText(c))- }- }- // Return without descending: the cells above already collected- // the nested table's text, so its rows must not be reported a- // second time as rows of the enclosing table.- rows = append(rows, cells)- return- }- for c := n.FirstChild; c != nil; c = c.NextSibling {- walk(c)- }- }- walk(doc)- return rows-}--// cellText returns the visible text of a table cell. The parser hands text-// nodes over already unescaped, nested markup contributes its text without-// its tags (a nested table's cells are concatenated, not separated), and a-// <br> becomes a newline instead of silently gluing the two halves of the-// cell together.-func cellText(cell *html.Node) string {- var sb strings.Builder- var walk func(*html.Node)- walk = func(n *html.Node) {- switch {- case n.Type == html.TextNode:- sb.WriteString(n.Data)- case isHTMLElement(n, "br"):- sb.WriteByte('\n')- case n.Type == html.ElementNode && isInertElement(n.Data):- // Stop here rather than descending: the content is parsed but- // never rendered, so it is not text a reader of the cell sees.- return- }- for c := n.FirstChild; c != nil; c = c.NextSibling {- walk(c)- }- }- walk(cell)- return strings.TrimSpace(sb.String())-}--// isHTMLElement reports whether n is an element with the given tag name in-// the HTML namespace. Foreign content reuses HTML tag names for unrelated-// elements — an <svg><tr> is not a table row — so matching on the tag name-// alone would read markup that carries no table semantics.-func isHTMLElement(n *html.Node, tag string) bool {- return n.Type == html.ElementNode && n.Namespace == "" && n.Data == tag-}--// isInertElement reports whether an element's content is inert — parsed, but-// never rendered as visible text. An inline <script> or <style> inside a-// table cell of a user-supplied document would otherwise be embedded into the-// Q&A pair as if it were part of the sentence, and a <template>'s placeholder-// rows would be read as real ones.-func isInertElement(tag string) bool {- switch tag {- case "script", "style", "noscript", "template":- return true- }- return false-}--// extractQATable turns table markup into Q&A pairs: the first two non-empty-// cells of a row become the question and the answer. strictPairs is the CSV-// contract (Python qa.py:365) and requires a row to have exactly two cells-// instead of taking the first two.-func extractQATable(htmlStr string, strictPairs bool) []qaPair {- if htmlStr == "" {- return nil- }- rows := tableRows(htmlStr)- pairs := make([]qaPair, 0, len(rows))- for _, cells := range rows {- // Python qa.py:365 requires exactly two fields for CSV pairs.- if strictPairs && len(cells) != 2 {- if len(pairs) > 0 {- pairs[len(pairs)-1].Answer += "\n" + strings.Join(cells, ",")- }- continue- }- var texts []string- for _, cell := range cells {- if cell != "" {- texts = append(texts, cell)- }- }- if len(texts) >= 2 {- // RowNum mirrors Python qa.py's enumerate over the extracted- // pairs (beAdoc(..., row_num=ii)) → top_int.- pairs = append(pairs, qaPair{Question: texts[0], Answer: texts[1], RowNum: len(pairs)})- }- }- return pairs-}-// ---------------------------------------------------------------------------// Markdown QA extraction// ---------------------------------------------------------------------------@@ -543,59 +403,118 @@ func extractQAJSON(items []schema.ChunkDoc, fileType string) []qaPair {strictCSV := strings.EqualFold(fileType, "csv")for _, item := range items {var tmp []qaPair- if item.CKType == "table_header" || item.CKType == "table_row" {- // QA spreadsheets have no header row: the parser's structural- // table_header is still the first question/answer record.- if strictCSV && len(item.Cells) != 2 {- if len(pairs) > 0 {- pairs[len(pairs)-1].Answer += "\n" + strings.Join(item.Cells, ",")- }- continue- }- tmp = extractQARowCells(item.Cells, strictCSV, item.RowStart)+ txt, _ := itemText(item)+ if txt == "" {+ continue+ }+ // Route on what the row walker can read, not on the type label.+ // doc_type:"table" is not truthful for every producer: the+ // opendataloader, tcadp and somark PDF parsers tag text that holds+ // no <table> markup, so such an item used to enter the table+ // extractor, find no rows, and silently lose every pair; conversely+ // a text-labelled block holding real table markup is read as a+ // table. isTableHTML is only the cheap candidate filter: the+ // walker's result decides, so nothing it can read is denied, and a+ // block that merely opens with "<table" text (no row) stays on the+ // prose path instead of silently pairing nothing. (Python's qa.py+ // drops plain-text table payloads such as PDF tables entirely; the+ // walker keeps their pairs.) Pre-upgrade row-IR payloads+ // (ck_type: table_row/table_header with cells) hold no markup and+ // fall through to the text extractor, so documents from before this+ // wire must be re-parsed rather than re-chunked.+ var rows [][]string+ if isTableHTML(txt) {+ rows = tableRows(txt)+ }+ if len(rows) > 0 {+ tmp = qaPairsFromRows(rows, strictCSV, item.Positions, item.SheetIndex != nil)} else {- txt, _ := itemText(item)- if txt == "" {- continue- }- // Non-spreadsheet table items may still carry HTML markup. Keep- // the HTML fallback for parsers that do not expose typed cells.- if itemDocType(item) == "table" {- tmp = extractQATable(txt, strictCSV)- } else {- tmp = extractQAText(txt)- }+ tmp = extractQAText(txt)}// Preserve the source item's image id and coordinates on each- // extracted pair+ // extracted pair. A row-aligned positions matrix already gave each+ // pair its own tuple (R1: a row chunk must not carry the whole+ // table's matrix in positions[0]).for _, p := range tmp {p.Image = item.Imagep.PDFPositions = item.PDFPositions- p.Positions = item.Positions+ if len(p.Positions) == 0 {+ p.Positions = item.Positions+ }pairs = append(pairs, p)}}return pairs}-func extractQARowCells(cells []string, strict bool, rowStart *int) []qaPair {- if strict && len(cells) != 2 {+// extractQATable turns table markup from an HTML payload into Q&A pairs: the+// JSON dispatch routes on the same walker, so both paths share+// qaPairsFromRows. These items carry no positions payload of their own.+func extractQATable(htmlStr string, strictPairs bool) []qaPair {+ if htmlStr == "" {return nil}- texts := make([]string, 0, len(cells))- for _, cell := range cells {- if cell = strings.TrimSpace(cell); cell != "" {- texts = append(texts, cell)+ return qaPairsFromRows(tableRows(htmlStr), strictPairs, nil, false)+}++// qaPairsFromRows builds the pairs of one table: the first two non-empty+// cells of a row become the question and the answer. strictPairs is the CSV+// contract (Python qa.py:365) and requires a row to have exactly two cells+// instead of taking the first two.+//+// positions is the item's own positions payload, and spreadsheetPositions+// says whether it may be read as the spreadsheet wire's matrix. When it is+// (identity present, one five-field tuple per <tr>), each pair carries only+// its row's tuple and takes its 0-based record index from that tuple's+// rowStart (top_int = positions[i][1] - 1), which reproduces the legacy+// row-IR numbering. Otherwise pairs are numbered by extraction order,+// mirroring Python qa.py's enumerate over the extracted pairs — PDF items+// write layout boxes into the same field, so they must not be reinterpreted+// as spreadsheet rows.+func qaPairsFromRows(rows [][]string, strictPairs bool, positions json.RawMessage, spreadsheetPositions bool) []qaPair {+ var matrix [][]float64+ if len(positions) > 0 {+ if err := json.Unmarshal(positions, &matrix); err != nil {+ matrix = nil+ }+ }+ rowAligned := spreadsheetPositions && len(matrix) == len(rows) && len(rows) > 0+ if rowAligned {+ for _, tuple := range matrix {+ if len(tuple) != 5 {+ rowAligned = false+ break+ }}}- if len(texts) < 2 {- return nil- }- rowNum := -1- if rowStart != nil && *rowStart > 0 {- rowNum = *rowStart - 1+ pairs := make([]qaPair, 0, len(rows))+ for i, cells := range rows {+ // Python qa.py:365 requires exactly two fields for CSV pairs.+ if strictPairs && len(cells) != 2 {+ if len(pairs) > 0 {+ pairs[len(pairs)-1].Answer += "\n" + strings.Join(cells, ",")+ }+ continue+ }+ var texts []string+ for _, cell := range cells {+ if cell != "" {+ texts = append(texts, cell)+ }+ }+ if len(texts) < 2 {+ continue+ }+ pair := qaPair{Question: texts[0], Answer: texts[1], RowNum: len(pairs)}+ if rowAligned {+ pair.RowNum = int(matrix[i][1]) - 1+ if encoded, err := json.Marshal([][]float64{matrix[i]}); err == nil {+ pair.Positions = encoded+ }+ }+ pairs = append(pairs, pair)}- return []qaPair{{Question: texts[0], Answer: texts[1], RowNum: rowNum}}+ return pairs}func init() {diff --git a/internal/ingestion/component/chunker/qa_spreadsheet_contract_test.go b/internal/ingestion/component/chunker/qa_spreadsheet_contract_test.goindex 4c5161d91f..becbdc7934 100644--- a/internal/ingestion/component/chunker/qa_spreadsheet_contract_test.go+++ b/internal/ingestion/component/chunker/qa_spreadsheet_contract_test.go@@ -46,10 +46,9 @@ func TestCSVTypedRowsFoldMalformedRowIntoPreviousAnswer(t *testing.T) {"name": "qa.csv","file_type": "csv","output_format": "json",- "json": []map[string]any{- {"doc_type_kwd": "table", "ck_type": "table_header", "cells": []string{"q1", "a1"}},- {"doc_type_kwd": "text", "ck_type": "table_row", "cells": []string{"question", "", "extra"}},- },+ "json": []map[string]any{spreadsheetSegmentItem("Sheet1",+ []string{"q1", "a1"},+ [][]string{{"question", "", "extra"}}, 1, 2)},})if err != nil {t.Fatal(err)diff --git a/internal/ingestion/component/chunker/qa_test.go b/internal/ingestion/component/chunker/qa_test.goindex 472771afd8..468edcf9c3 100644--- a/internal/ingestion/component/chunker/qa_test.go+++ b/internal/ingestion/component/chunker/qa_test.go@@ -17,10 +17,12 @@package chunkerimport (+ "encoding/json""strings""testing""ragflow/internal/agent/runtime"+ "ragflow/internal/ingestion/component/schema")func TestQAChunker_Registered(t *testing.T) {@@ -180,9 +182,9 @@ func TestQAChunker_JSONCSVNameUsesStrictRowShapeWithoutFileType(t *testing.T) {"name": "questions.csv","output_format": "json","json": []map[string]any{{- "doc_type_kwd": "text",- "ck_type": "table_row",- "cells": []string{"question", "answer", "unexpected"},+ "text": "<table><tr><td>question</td><td></td><td>unexpected</td></tr></table>",+ "doc_type_kwd": "table",+ "ck_type": "table",}},}out, err := comp.Invoke(t.Context(), nil, inputs)@@ -419,7 +421,7 @@ func TestQAChunker_XLSXJSONRegression(t *testing.T) {}}-func TestQAChunkerSpreadsheetRowIRTreatsFirstRowAsQAData(t *testing.T) {+func TestQAChunkerSpreadsheetWireTreatsFirstRowAsQAData(t *testing.T) {comp, err := NewQAChunker(map[string]any{"lang": "english"})if err != nil {t.Fatal(err)@@ -429,8 +431,7 @@ func TestQAChunkerSpreadsheetRowIRTreatsFirstRowAsQAData(t *testing.T) {"file_type": "xlsx","output_format": "json","json": []map[string]any{- {"text": "ID; Status", "doc_type_kwd": "table", "ck_type": "table_header", "cells": []string{"ID", "Status"}},- {"text": "ID:A-100; Status:paid", "doc_type_kwd": "text", "ck_type": "table_row", "cells": []string{"A-100", "paid"}},+ spreadsheetSegmentItem("Sheet1", []string{"ID", "Status"}, [][]string{{"A-100", "paid"}}, 1, 2),},})if err != nil {@@ -446,4 +447,116 @@ func TestQAChunkerSpreadsheetRowIRTreatsFirstRowAsQAData(t *testing.T) {if got, _ := chunks[1]["text"].(string); got != "Question: A-100\tAnswer: paid" {t.Fatalf("second-row QA = %q", got)}+ // top_int keeps the legacy 0-based record index: rowStart - 1.+ for i, want := range []any{float64(0), float64(1)} {+ top, _ := chunks[i]["top_int"].([]any)+ if len(top) != 1 || top[0] != want {+ t.Errorf("chunk[%d] top_int = %v, want [%v]", i, chunks[i]["top_int"], want)+ }+ }+ // R1: each pair carries only its own row's tuple, not the segment matrix.+ for i, wantRow := range []float64{1, 2} {+ raw, _ := json.Marshal(chunks[i]["positions"])+ var matrix [][]float64+ if err := json.Unmarshal(raw, &matrix); err != nil {+ t.Fatalf("chunk[%d] positions = %v: %v", i, chunks[i]["positions"], err)+ }+ if len(matrix) != 1 {+ t.Fatalf("chunk[%d] positions = %v, want one tuple", i, chunks[i]["positions"])+ }+ if tuple := matrix[0]; len(tuple) != 5 || tuple[1] != wantRow {+ t.Errorf("chunk[%d] tuple = %v, want rowStart %v", i, tuple, wantRow)+ }+ }+}++// TestQAChunker_PlainTablePayloadNotSwallowed: a doc_type:"table" item whose+// payload is plain delimited text (the shape of a PDF table item) used to+// enter the HTML table extractor, find no <tr>, and lose every pair silently.+// Routing on what the row walker can read sends it to the text extractor+// instead.+func TestQAChunker_PlainTablePayloadNotSwallowed(t *testing.T) {+ items := []schema.ChunkDoc{+ {Text: "what is it\tit is a thing\nwho comes\tbob", DocType: "table"},+ }+ pairs := extractQAJSON(items, "")+ if len(pairs) != 2 {+ t.Fatalf("got %d pairs, want 2 (plain payload under a table label must still yield pairs)", len(pairs))+ }+ if pairs[0].Question != "what is it" || pairs[0].Answer != "it is a thing" {+ t.Errorf("pair0 = %q/%q", pairs[0].Question, pairs[0].Answer)+ }+}++// TestQAChunker_TextLabelledHTMLTableReadAsTable documents the accepted+// non-monotonic side of payload routing: a text-labelled block that+// genuinely holds table markup is now read as a table.+func TestQAChunker_TextLabelledHTMLTableReadAsTable(t *testing.T) {+ items := []schema.ChunkDoc{+ {Text: "<table><tr><td>q1</td><td>a1</td></tr><tr><td>q2</td><td>a2</td></tr></table>", DocType: "text"},+ }+ pairs := extractQAJSON(items, "")+ if len(pairs) != 2 {+ t.Fatalf("got %d pairs, want 2", len(pairs))+ }+ if pairs[1].Question != "q2" || pairs[1].Answer != "a2" {+ t.Errorf("pair1 = %q/%q", pairs[1].Question, pairs[1].Answer)+ }+}++// TestQAChunker_RowlessTableTextStaysProse: text that merely opens with+// "<table" — a table element without rows, or a "<tableau>" token — must+// stay on the prose path. The table extractor would find nothing to pair+// there, so routing it by prefix alone would silently turn readable text+// into zero pairs.+func TestQAChunker_RowlessTableTextStaysProse(t *testing.T) {+ items := []schema.ChunkDoc{+ {Text: "<table><caption>schema</caption></table>\nwhat is it\tit is a thing", DocType: "table"},+ {Text: "<tableau>\nwhat is it\tit is a thing", DocType: "table"},+ }+ pairs := extractQAJSON(items, "")+ if len(pairs) != 2 {+ t.Fatalf("got %d pairs, want one from each item via the prose path", len(pairs))+ }+ for i, pair := range pairs {+ if pair.Question != "what is it" || pair.Answer != "it is a thing" {+ t.Errorf("pair%d = %q/%q", i, pair.Question, pair.Answer)+ }+ }+}++// TestQAChunkerPositionsRequireSpreadsheetIdentityAndFiveFields: a positions+// matrix is read as per-row spreadsheet tuples only when the item carries+// spreadsheet identity and every tuple has the five wire fields. A short tuple+// must not panic the pair builder, and a matrix without identity must not move+// the row numbers (PDF items write layout boxes into the same field).+func TestQAChunkerPositionsRequireSpreadsheetIdentityAndFiveFields(t *testing.T) {+ table := "<table><tr><th>q</th><th>a</th></tr></table>"+ cases := []struct {+ name string+ positions string+ sheetIndex *int+ wantRowNum int+ }{+ {"aligned sheet matrix", `[[1,5,5,1,2]]`, intPtr(1), 4},+ {"short tuple", `[[1]]`, intPtr(1), 0},+ {"sheet matrix without identity", `[[1,5,5,1,2]]`, nil, 0},+ }+ for _, tc := range cases {+ t.Run(tc.name, func(t *testing.T) {+ item := schema.ChunkDoc{+ Text: table,+ DocType: "table",+ Positions: json.RawMessage(tc.positions),+ SheetIndex: tc.sheetIndex,+ }+ pairs := extractQAJSON([]schema.ChunkDoc{item}, "")+ if len(pairs) != 1 {+ t.Fatalf("got %d pairs, want 1", len(pairs))+ }+ if pairs[0].RowNum != tc.wantRowNum {+ t.Errorf("RowNum = %d, want %d", pairs[0].RowNum, tc.wantRowNum)+ }+ })+ }}diff --git a/internal/ingestion/component/chunker/table.go b/internal/ingestion/component/chunker/table.goindex 0dc53979fb..dc405675bb 100644--- a/internal/ingestion/component/chunker/table.go+++ b/internal/ingestion/component/chunker/table.go@@ -20,16 +20,19 @@//// Unlike TokenChunker (which token-shreds a row's text into multiple// pieces) or OneChunker (which merges many rows into a single chunk),-// TableChunker keeps the row as the unit of chunking. The table parser is-// responsible for producing one structured record (ChunkDoc) per row with-// the formatted "- field: value" content and any column-role / field_map-// metadata; this chunker passes each record through unchanged, one chunk-// per row.+// TableChunker keeps the row as the unit of chunking. Every table arrives as+// rendered HTML — the spreadsheet parsers emit captioned table segments — and+// each data row becomes one chunk whose text repeats the column names in the+// Python "- field: value" line format, carrying its own position tuple when+// the item is a spreadsheet item with a row-aligned matrix. A table whose only+// row is the header stays one whole chunk.package chunkerimport ("context"+ "encoding/json""fmt"+ "strings""ragflow/internal/agent/runtime""ragflow/internal/ingestion/component/schema"@@ -111,7 +114,11 @@ func (c *TableChunkerComponent) invoke(_ context.Context, inputs map[string]any)}// tableItems returns the per-row records, preferring JSONResult and-// falling back to Chunks. Each record becomes exactly one chunk.+// falling back to Chunks. Each HTML table row becomes exactly one chunk;+// every other payload record passes through as one chunk — including+// pre-upgrade row-IR records (ck_type: table_row/table_header with cells),+// which hold no markup and therefore keep no per-row positions; documents+// from before this wire must be re-parsed rather than re-chunked.func tableItems(items, chunks []schema.ChunkDoc) []schema.ChunkDoc {source := itemsif len(source) == 0 {@@ -120,42 +127,89 @@ func tableItems(items, chunks []schema.ChunkDoc) []schema.ChunkDoc {if len(source) == 0 {return nil}- dataTables := make(map[string]struct{})- for _, item := range source {- if item.CKType == "table_row" {- dataTables[spreadsheetTableKey(item)] = struct{}{}- }- }filtered := make([]schema.ChunkDoc, 0, len(source))for _, item := range source {- // Spreadsheet parsers expose the header as schema metadata. It is- // already represented in each table_row and must not become a data- // chunk of its own. A table with no data rows still needs its header- // as the only searchable representation.- if item.CKType == "table_header" {- if _, hasRows := dataTables[spreadsheetTableKey(item)]; hasRows {- continue- }- }- filtered = append(filtered, item)+ filtered = append(filtered, expandHTMLTableRows(item)...)}return filtered}-func spreadsheetTableKey(item schema.ChunkDoc) string {- if item.TableID != "" {- return "table:" + item.TableID+// expandHTMLTableRows turns one HTML <table> payload into one chunk per data+// row. Non-table payloads pass through unchanged. A table whose only row is+// the header keeps the whole markup as its single chunk: the header line is+// then the only searchable representation.+func expandHTMLTableRows(item schema.ChunkDoc) []schema.ChunkDoc {+ if !isTableHTML(item.Text) {+ return []schema.ChunkDoc{item}}- if item.SheetIndex != nil {- return fmt.Sprintf("sheet-index:%d", *item.SheetIndex)+ rows, headerCount := tableRowsWithHeader(item.Text)+ if len(rows) <= headerCount {+ return []schema.ChunkDoc{item}}- if item.Sheet != "" {- return "sheet:" + item.Sheet+ names := rows[0]+ // R1: a row chunk must carry only its own tuple. That is only sound when+ // the item carries spreadsheet identity and its matrix was built+ // row-aligned (one five-field tuple per <tr>, header included); a+ // whole-table tuple must not be copied onto every row, and PDF items write+ // layout boxes into the same field, so misaligned or non-spreadsheet+ // payloads get no positions at all.+ var matrix [][]float64+ aligned := false+ if item.SheetIndex != nil && len(item.Positions) > 0 {+ if err := json.Unmarshal(item.Positions, &matrix); err == nil && len(matrix) == len(rows) {+ aligned = true+ for _, tuple := range matrix {+ if len(tuple) != 5 {+ aligned = false+ break+ }+ }+ }}- if sheet, ok := spreadsheetPositionSheet(item); ok {- return fmt.Sprintf("position-sheet:%g", sheet)+ out := make([]schema.ChunkDoc, 0, len(rows)-headerCount)+ for i, row := range rows[headerCount:] {+ text := tableRowRecordText(names, row)+ if text == "" {+ continue+ }+ doc := item+ doc.Text = text+ doc.TKNums = nil+ doc.Positions = nil+ if aligned {+ if tuple, err := json.Marshal([][]float64{matrix[headerCount+i]}); err == nil {+ doc.Positions = tuple+ }+ }+ out = append(out, doc)+ }+ if len(out) == 0 {+ return []schema.ChunkDoc{item}+ }+ return out+}++// tableRowRecordText renders one row in rag/app/table.py's line format:+// "- {field}: {value}" per non-empty cell, newline joined. A cell whose+// column has no header name keeps its value alone rather than being dropped.+func tableRowRecordText(names, cells []string) string {+ lines := make([]string, 0, len(cells))+ for j, cell := range cells {+ value := strings.TrimSpace(cell)+ if value == "" {+ continue+ }+ name := ""+ if j < len(names) {+ name = strings.TrimSpace(names[j])+ }+ if name != "" {+ lines = append(lines, "- "+name+": "+value)+ continue+ }+ lines = append(lines, "- "+value)}- return "unknown"+ return strings.Join(lines, "\n")}func init() {diff --git a/internal/ingestion/component/chunker/table_split.go b/internal/ingestion/component/chunker/table_split.gonew file mode 100644index 0000000000..0957114cef--- /dev/null+++ b/internal/ingestion/component/chunker/table_split.go@@ -0,0 +1,189 @@+//+// Copyright 2026 The InfiniFlow Authors. All Rights Reserved.+//+// Licensed under the Apache License, Version 2.0 (the "License");+// you may not use this file except in compliance with the License.+// You may obtain a copy of the License at+//+// http://www.apache.org/licenses/LICENSE-2.0+//+// Unless required by applicable law or agreed to in writing, software+// distributed under the License is distributed on an "AS IS" BASIS,+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+// See the License for the specific language governing permissions and+// limitations under the License.+//++package chunker++import (+ "regexp"+ "strings"+)++var trRegex = regexp.MustCompile(`(?is)<tr\b[^>]*>.*?</tr>`)++// tableOpenTagEnd returns the index of the ">" that closes the open tag whose+// name s begins after, skipping ">" inside quoted attribute values. It returns+// -1 when the tag never closes before the next "<": a malformed open tag must+// not be cut at the next ">", which may belong to a closing tag — slicing+// "<table </table>" there would invert the slice bounds.+func tableOpenTagEnd(s string) int {+ quote := byte(0)+ for i := 0; i < len(s); i++ {+ switch c := s[i]; {+ case quote != 0:+ if c == quote {+ quote = 0+ }+ case c == '"' || c == '\'':+ quote = c+ case c == '>':+ return i+ case c == '<':+ return -1+ }+ }+ return -1+}++// splitLargeHTMLTable splits an HTML table into sub-tables that fit within+// maxTokens, replicating the caption and header row(s) in each sub-table.+// Token counting is injected (countTokens) so the caller's tokenizer decides+// what "size" means. This is the chunker-side size budget: Python's flow path+// never splits a spreadsheet sheet, but honouring chunk_token_size keeps a+// wide or long table from losing its tail rows to embedding truncation.+//+// The text is expected to hold one outer <table> with flat rows — the shape+// the spreadsheet parsers render. Markup this function cannot cut safely (an+// open tag that never closes, a ">" inside an attribute value, a nested or+// second table) is returned unchanged, so callers fall back to one whole+// chunk instead of cutting at a guessed boundary.+//+// Alongside the parts it returns dataRanges and headerRows: for part i, the+// half-open range [start, end) of DATA rows (rows below the header in the+// original table) the part contains, and the number of leading header rows+// every part replicates. A caller carrying a per-row position matrix slices+// matrix[:headerRows] + matrix[headerRows+start : headerRows+end] to stay+// aligned with the sub-table markup. headerRows and ranges are 0/nil when+// the text is returned unchanged.+func splitLargeHTMLTable(text string, maxTokens int, countTokens func(string) int) (parts []string, dataRanges [][2]int, headerRows int) {+ lower := strings.ToLower(text)+ startIdx := strings.Index(lower, "<table")+ endIdx := strings.LastIndex(lower, "</table>")+ if startIdx < 0 || endIdx < 0 || endIdx <= startIdx {+ return []string{text}, nil, 0+ }++ prefix := text[:startIdx]+ afterOpen := text[startIdx:]+ const openTag = "<table"+ // The match must be a tag name, not the prefix of a longer one such as+ // "<tableau>".+ if len(afterOpen) <= len(openTag) || !strings.ContainsRune(" \t\n\f\r/>", rune(afterOpen[len(openTag)])) {+ return []string{text}, nil, 0+ }+ tagRel := tableOpenTagEnd(afterOpen[len(openTag):])+ if tagRel < 0 {+ return []string{text}, nil, 0+ }+ tagEnd := len(openTag) + tagRel+ innerStart := startIdx + tagEnd + 1+ if innerStart > endIdx {+ return []string{text}, nil, 0+ }+ tableOpenTag := afterOpen[:tagEnd+1]+ inner := text[innerStart:endIdx]+ // A second table (or a nested one) has no modelled boundary here; leaving+ // the text alone keeps the caller on the single-chunk path.+ if innerLower := strings.ToLower(inner); strings.Contains(innerLower, "<table") || strings.Contains(innerLower, "</table>") {+ return []string{text}, nil, 0+ }+ suffix := text[endIdx+len("</table>"):]++ // Extract caption if present.+ lowerInner := strings.ToLower(inner)+ capStart := strings.Index(lowerInner, "<caption")+ var captionHTML string+ if capStart >= 0 {+ capEndRel := strings.Index(lowerInner[capStart:], "</caption>")+ if capEndRel >= 0 {+ capEnd := capStart + capEndRel + len("</caption>")+ captionHTML = inner[capStart:capEnd]+ }+ }++ // Extract all rows.+ rows := trRegex.FindAllString(inner, -1)+ if len(rows) <= 1 {+ return []string{text}, nil, 0+ }++ // Determine header rows vs data rows.+ headerEnd := 0+ for i, r := range rows {+ if strings.Contains(strings.ToLower(r), "<th") {+ headerEnd = i + 1+ } else {+ break+ }+ }+ if headerEnd == 0 || headerEnd >= len(rows) {+ headerEnd = 1+ }+ headerRowHTML := rows[:headerEnd]+ dataRows := rows[headerEnd:]++ baseHeader := tableOpenTag + captionHTML + strings.Join(headerRowHTML, "")+ prefixTokens := countTokens(prefix)+ baseTokens := countTokens(baseHeader + "</table>")++ // prefix (text before the table, e.g. a short Markdown heading) belongs to+ // the table's position in the document: repeat only the table wrapper,+ // caption and header rows in every chunk, and prepend prefix to the first.+ var chunks []string+ var ranges [][2]int+ var currentDataRows []string+ currentTokens := baseTokens + prefixTokens+ currentStart := 0+ firstChunk := true++ flush := func(end int) {+ if len(currentDataRows) == 0 {+ return+ }+ var b strings.Builder+ if firstChunk {+ b.WriteString(prefix)+ firstChunk = false+ }+ b.WriteString(baseHeader)+ for _, dr := range currentDataRows {+ b.WriteString(dr)+ }+ b.WriteString("</table>")+ chunks = append(chunks, b.String())+ ranges = append(ranges, [2]int{currentStart, end})+ currentDataRows = nil+ currentTokens = baseTokens+ currentStart = end+ }++ for i, row := range dataRows {+ rowTokens := countTokens(row)+ if len(currentDataRows) > 0 && currentTokens+rowTokens > maxTokens {+ flush(i)+ }+ currentDataRows = append(currentDataRows, row)+ currentTokens += rowTokens+ }+ flush(len(dataRows))++ if len(chunks) <= 1 {+ return []string{text}, nil, 0+ }+ if suffix != "" {+ chunks[len(chunks)-1] += suffix+ }+ return chunks, ranges, headerEnd+}diff --git a/internal/ingestion/component/chunker/table_split_test.go b/internal/ingestion/component/chunker/table_split_test.gonew file mode 100644index 0000000000..652bdba8d4--- /dev/null+++ b/internal/ingestion/component/chunker/table_split_test.go@@ -0,0 +1,165 @@+//+// Copyright 2026 The InfiniFlow Authors. All Rights Reserved.+//+// Licensed under the Apache License, Version 2.0 (the "License");+// you may not use this file except in compliance with the License.+// You may obtain a copy of the License at+//+// http://www.apache.org/licenses/LICENSE-2.0+//+// Unless required by applicable law or agreed to in writing, software+// distributed under the License is distributed on an "AS IS" BASIS,+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+// See the License for the specific language governing permissions and+// limitations under the License.+//++package chunker++import (+ "strings"+ "testing"+)++// charTokens approximates a tokenizer for tests: one token per byte.+func charTokens(s string) int { return len(s) }++func buildTable(header string, data []string) string {+ var b strings.Builder+ b.WriteString("<table><caption>cap</caption>\n")+ b.WriteString("<tr>" + header + "</tr>\n")+ for _, row := range data {+ b.WriteString("<tr>" + row + "</tr>\n")+ }+ b.WriteString("</table>\n")+ return b.String()+}++func TestSplitLargeHTMLTableNoSplit(t *testing.T) {+ small := buildTable("<th>a</th><th>b</th>", []string{"<td>1</td><td>2</td>"})+ parts, ranges, headerRows := splitLargeHTMLTable(small, len(small)+100, charTokens)+ if len(parts) != 1 || parts[0] != small || ranges != nil || headerRows != 0 {+ t.Fatalf("small table must return unchanged with nil ranges, got %d parts ranges=%v headerRows=%d", len(parts), ranges, headerRows)+ }+ parts, ranges, headerRows = splitLargeHTMLTable("no table here", 1, charTokens)+ if len(parts) != 1 || parts[0] != "no table here" || ranges != nil || headerRows != 0 {+ t.Fatalf("non-table text must pass through unchanged")+ }+}++func TestSplitLargeHTMLTableReplicatesHeaderAndSlicesRanges(t *testing.T) {+ data := make([]string, 6)+ for i := range data {+ data[i] = "<td>d" + string(rune('0'+i)) + "</td><td>x</td>"+ }+ text := buildTable("<th>h1</th><th>h2</th>", data)+ // Budget that holds header + 2 data rows at most.+ rowLen := len("<td>d0</td><td>x</td>")+ budget := len("<table><caption>cap</caption><tr><th>h1</th><th>h2</th></tr></table>") + 2*rowLen+ parts, ranges, headerRows := splitLargeHTMLTable(text, budget, charTokens)+ if headerRows != 1 {+ t.Fatalf("headerRows = %d, want 1 (the single <th> row)", headerRows)+ }+ if len(parts) < 3 {+ t.Fatalf("expected several sub-tables, got %d", len(parts))+ }+ if len(ranges) != len(parts) {+ t.Fatalf("ranges count %d != parts count %d", len(ranges), len(parts))+ }+ seen := 0+ for i, p := range parts {+ if !strings.HasPrefix(p, "<table><caption>cap</caption>") {+ t.Errorf("part %d lost its caption: %q", i, p)+ }+ if !strings.Contains(p, "<th>h1</th>") {+ t.Errorf("part %d lost the header row", i)+ }+ lo, hi := ranges[i][0], ranges[i][1]+ if hi <= lo || hi > len(data) {+ t.Fatalf("part %d range [%d,%d) out of bounds", i, lo, hi)+ }+ for r := lo; r < hi; r++ {+ if !strings.Contains(p, data[r]) {+ t.Errorf("part %d missing row %d", i, r)+ }+ seen+++ }+ // Rows outside the range must not leak into this part.+ for r := 0; r < len(data); r++ {+ if (r < lo || r >= hi) && strings.Contains(p, data[r]) {+ t.Errorf("part %d leaked row %d outside [%d,%d)", i, r, lo, hi)+ }+ }+ }+ if seen != len(data) {+ t.Errorf("rows covered %d, want %d", seen, len(data))+ }+}++func TestSplitLargeHTMLTableHeaderlessReplicatesFirstRow(t *testing.T) {+ // Original behaviour: with no <th> row the first row is treated as the+ // repeating header.+ var b strings.Builder+ b.WriteString("<table>")+ b.WriteString("<tr><td>r0</td></tr>")+ for i := 1; i < 5; i++ {+ b.WriteString("<tr><td>row" + string(rune('0'+i)) + "</td></tr>")+ }+ b.WriteString("</table>")+ parts, ranges, headerRows := splitLargeHTMLTable(b.String(), len("<table><tr><td>r0</td></tr><tr><td>row1</td></tr></table>")+10, charTokens)+ if headerRows != 1 {+ t.Fatalf("headerRows = %d, want 1 (first row treated as header)", headerRows)+ }+ if len(parts) < 2 {+ t.Fatalf("expected split, got %d parts", len(parts))+ }+ for i, p := range parts {+ if !strings.Contains(p, "<td>r0</td>") {+ t.Errorf("part %d lost replicated first row", i)+ }+ }+ if ranges[0][0] != 0 {+ t.Errorf("ranges index DATA rows below the treated header, want first range to start at 0, got %v", ranges[0])+ }+}++// TestSplitLargeHTMLTableRefusesUncuttableMarkup: an open tag that never+// closes, a longer tag name, or a nested table is markup this splitter must+// not cut at a guessed boundary — the text comes back unchanged. The+// "<table </table>" case used to invert the slice bounds.+func TestSplitLargeHTMLTableRefusesUncuttableMarkup(t *testing.T) {+ unchanged := []string{+ "<table </table>",+ "<tableau><tr><td>a</td></tr><tr><td>b</td></tr></table>",+ "<table><tr><td>a</td></tr><table><tr><td>b</td></tr></table></table>",+ }+ for _, text := range unchanged {+ parts, ranges, headerRows := splitLargeHTMLTable(text, 1, charTokens)+ if len(parts) != 1 || parts[0] != text || ranges != nil || headerRows != 0 {+ t.Errorf("text %q must pass through unchanged, got %d parts ranges=%v headerRows=%d", text, len(parts), ranges, headerRows)+ }+ }+}++// TestSplitLargeHTMLTableQuotedAngleBracketStaysInOpenTag: a ">" inside a+// quoted attribute value must not end the open tag — every part repeats the+// complete tag instead of being cut at the attribute.+func TestSplitLargeHTMLTableQuotedAngleBracketStaysInOpenTag(t *testing.T) {+ text := "<table data-meta=\"a>b\">" ++ "<tr><td>r0</td></tr><tr><td>r1</td></tr><tr><td>r2</td></tr></table>"+ parts, ranges, headerRows := splitLargeHTMLTable(text, 60, charTokens)+ if headerRows != 1 {+ t.Fatalf("headerRows = %d, want 1 (first row treated as header)", headerRows)+ }+ if len(parts) < 2 {+ t.Fatalf("expected a split, got %d parts", len(parts))+ }+ if len(ranges) != len(parts) {+ t.Fatalf("ranges count %d != parts count %d", len(ranges), len(parts))+ }+ for i, p := range parts {+ if !strings.HasPrefix(p, "<table data-meta=\"a>b\">") {+ t.Errorf("part %d lost the complete open tag: %q", i, p)+ }+ }+}diff --git a/internal/ingestion/component/chunker/table_test.go b/internal/ingestion/component/chunker/table_test.goindex c5cde7d98f..99e653c4da 100644--- a/internal/ingestion/component/chunker/table_test.go+++ b/internal/ingestion/component/chunker/table_test.go@@ -17,6 +17,8 @@package chunkerimport (+ "fmt"+ "strings""testing")@@ -80,53 +82,173 @@ func TestTableChunker_EmptyRows(t *testing.T) {}}-func TestTableChunkerSkipsSpreadsheetHeaderRecord(t *testing.T) {+// TestTableChunkerPreservesHeaderOnlyTable: the header row only becomes a+// chunk itself when the segment has no data rows — then the whole markup is+// the table's only searchable representation.+func TestTableChunkerPreservesHeaderOnlyTable(t *testing.T) {+ segment := spreadsheetSegmentItem("Sheet1", []string{"ID", "Name"}, nil, 1, 1)+ chunks := tableChunksOf(t, map[string]any{+ "name": "empty.xlsx",+ "output_format": "json",+ "json": []map[string]any{segment},+ })+ if len(chunks) != 1 {+ t.Fatalf("got %d chunks, want the header-only table preserved", len(chunks))+ }+ if chunks[0]["ck_type"] != "table" || chunks[0]["text"] != segment["text"] {+ t.Fatalf("chunk = %#v, want the header-only segment whole", chunks[0])+ }+}++func TestTableChunkerPreservesHeaderOnlySheetAlongsideDataSheet(t *testing.T) {+ dataSegment := spreadsheetSegmentItem("Sheet1", []string{"ID", "Name"}, [][]string{{"A-1", "paid"}}, 1, 2)+ headerSegment := spreadsheetSegmentItem("Sheet2", []string{"ID", "Status"}, nil, 2, 1)+ chunks := tableChunksOf(t, map[string]any{+ "name": "mixed.xlsx",+ "output_format": "json",+ "json": []map[string]any{dataSegment, headerSegment},+ })+ if len(chunks) != 2 {+ t.Fatalf("got %d chunks, want data row plus header-only segment", len(chunks))+ }+ // The header row of the data segment never becomes a chunk of its own;+ // its column names live in the row record text instead.+ if chunks[0]["text"] != "- ID: A-1\n- Name: paid" {+ t.Fatalf("chunk0 = %#v", chunks[0])+ }+ if chunks[1]["text"] != headerSegment["text"] {+ t.Fatalf("chunk1 = %#v, want the header-only segment whole", chunks[1])+ }+}++// TestTableChunker_ExpandsHTMLRowsWithAlignedPositions covers the P2+// expander: one HTML <table> item with a row-aligned position matrix (one+// tuple per <tr>, header included) becomes one chunk per data row, each+// carrying its own tuple and the Python "- field: value" line format.+func TestTableChunker_ExpandsHTMLRowsWithAlignedPositions(t *testing.T) {+ table := "<table><caption>orders</caption>\n" ++ "<tr><th>ID</th><th>Status</th></tr>\n" ++ "<tr><td>A-1</td><td>paid</td></tr>\n" ++ "<tr><td>A-2</td><td></td></tr>\n" ++ "</table>\n"chunks := tableChunksOf(t, map[string]any{"name": "orders.xlsx","output_format": "json","json": []map[string]any{- {"text": "ID; Status", "doc_type_kwd": "table", "ck_type": "table_header", "cells": []string{"ID", "Status"}},- {"text": "A-100; paid", "doc_type_kwd": "table", "ck_type": "table_row", "cells": []string{"A-100", "paid"}},+ {+ "text": table,+ "doc_type_kwd": "table",+ "ck_type": "table",+ "sheet_index": 1,+ "positions": [][]float64{{1, 1, 1, 1, 2}, {1, 2, 2, 1, 2}, {1, 3, 3, 1, 2}},+ },},})- if len(chunks) != 1 {- t.Fatalf("got %d chunks, want only the data row", len(chunks))+ if len(chunks) != 2 {+ t.Fatalf("got %d chunks, want one per data row", len(chunks))+ }+ if chunks[0]["text"] != "- ID: A-1\n- Status: paid" {+ t.Errorf("chunk0 text = %v", chunks[0]["text"])}- if chunks[0]["ck_type"] != "table_row" || chunks[0]["text"] != "A-100; paid" {- t.Fatalf("chunks = %#v, want the table row only", chunks)+ // The empty cell is skipped, exactly like rag/app/table.py:656.+ if chunks[1]["text"] != "- ID: A-2" {+ t.Errorf("chunk1 text = %v", chunks[1]["text"])+ }+ if got := fmt.Sprintf("%v", chunks[0]["positions"]); !strings.Contains(got, "2") || strings.Contains(got, "3") {+ t.Errorf("chunk0 positions should hold only its own tuple, got %v", chunks[0]["positions"])+ }+ if got := fmt.Sprintf("%v", chunks[1]["positions"]); !strings.Contains(got, "3") {+ t.Errorf("chunk1 positions should hold its own tuple, got %v", chunks[1]["positions"])}}-func TestTableChunkerPreservesHeaderOnlyTable(t *testing.T) {+// TestTableChunker_WholeTableTupleNotCopiedToRows guards R1: when the+// position matrix is not row-aligned (a single whole-table tuple), row+// chunks must not inherit it — every row would otherwise locate to the same+// first coordinate. Row chunks get no positions instead.+func TestTableChunker_WholeTableTupleNotCopiedToRows(t *testing.T) {+ table := "<table><tr><th>ID</th></tr><tr><td>a</td></tr><tr><td>b</td></tr></table>"chunks := tableChunksOf(t, map[string]any{- "name": "empty.xlsx",+ "name": "sheet.csv","output_format": "json","json": []map[string]any{- {"text": "ID; Name", "doc_type_kwd": "table", "ck_type": "table_header", "table_id": "sheet-1", "sheet_index": 1},+ {+ "text": table,+ "doc_type_kwd": "table",+ "sheet_index": 1,+ "positions": [][]float64{{1, 1, 2, 1, 1}},+ },},})- if len(chunks) != 1 {- t.Fatalf("got %d chunks, want the header-only table preserved", len(chunks))+ if len(chunks) != 2 {+ t.Fatalf("got %d chunks, want 2", len(chunks))}- if chunks[0]["ck_type"] != "table_header" || chunks[0]["text"] != "ID; Name" {- t.Fatalf("chunk = %#v, want the header record", chunks[0])+ for i, c := range chunks {+ if p, ok := c["positions"]; ok && p != nil && fmt.Sprintf("%v", p) != "[]" && fmt.Sprintf("%v", p) != "<nil>" {+ t.Errorf("chunk %d leaked whole-table positions: %v", i, p)+ }}}-func TestTableChunkerPreservesHeaderOnlySheetAlongsideDataSheet(t *testing.T) {+// TestTableChunker_NonSpreadsheetPositionsNotAttached: a markup item without+// spreadsheet identity must not have its positions read as per-row tuples —+// PDF items write layout boxes into the same field, so a five-field matrix+// that happens to align must still be ignored.+func TestTableChunker_NonSpreadsheetPositionsNotAttached(t *testing.T) {+ table := "<table><tr><th>ID</th></tr><tr><td>a</td></tr><tr><td>b</td></tr></table>"chunks := tableChunksOf(t, map[string]any{- "name": "mixed.xlsx",+ "name": "document.docx","output_format": "json","json": []map[string]any{- {"text": "ID; Name", "doc_type_kwd": "table", "ck_type": "table_header", "table_id": "sheet-1", "sheet_index": 1},- {"text": "ID; Status", "doc_type_kwd": "table", "ck_type": "table_header", "table_id": "sheet-2", "sheet_index": 2},- {"text": "A-1; paid", "doc_type_kwd": "text", "ck_type": "table_row", "table_id": "sheet-2", "sheet_index": 2},+ {+ "text": table,+ "doc_type_kwd": "table",+ "ck_type": "table",+ "positions": [][]float64{{1, 1, 1, 1, 1}, {1, 2, 2, 1, 1}, {1, 3, 3, 1, 1}},+ },},})if len(chunks) != 2 {- t.Fatalf("got %d chunks, want header-only sheet plus data row", len(chunks))+ t.Fatalf("got %d chunks, want one per data row", len(chunks))+ }+ for i, c := range chunks {+ if p, ok := c["positions"]; ok && p != nil && fmt.Sprintf("%v", p) != "[]" && fmt.Sprintf("%v", p) != "<nil>" {+ t.Errorf("chunk %d kept positions without spreadsheet identity: %v", i, p)+ }+ }+}++// TestTableChunker_HeaderOnlyHTMLTableKeepsWholePayload: a table whose only+// row is the header has nothing to expand; the whole markup stays one chunk.+func TestTableChunker_HeaderOnlyHTMLTableKeepsWholePayload(t *testing.T) {+ table := "<table><caption>t</caption>\n<tr><th>A</th><th>B</th></tr>\n</table>\n"+ chunks := tableChunksOf(t, map[string]any{+ "name": "template.xlsx",+ "output_format": "json",+ "json": []map[string]any{+ {"text": table, "doc_type_kwd": "table", "ck_type": "table"},+ },+ })+ if len(chunks) != 1 || chunks[0]["text"] != table {+ t.Fatalf("header-only table must stay a single whole-payload chunk, got %#v", chunks)+ }+}++// TestTableChunker_HeaderlessHTMLFirstRowIsHeader: with no <th> row the+// first row names the columns, matching SplitLargeHTMLTable's convention.+func TestTableChunker_HeaderlessHTMLFirstRowIsHeader(t *testing.T) {+ table := "<table><tr><td>k</td><td>v</td></tr><tr><td>a</td><td>b</td></tr></table>"+ chunks := tableChunksOf(t, map[string]any{+ "name": "kv.csv",+ "output_format": "json",+ "json": []map[string]any{+ {"text": table, "doc_type_kwd": "table"},+ },+ })+ if len(chunks) != 1 {+ t.Fatalf("got %d chunks, want the single data row", len(chunks))}- if chunks[0]["ck_type"] != "table_header" || chunks[1]["ck_type"] != "table_row" {- t.Fatalf("chunks = %#v, want header then row", chunks)+ if chunks[0]["text"] != "- k: a\n- v: b" {+ t.Errorf("text = %v", chunks[0]["text"])}}diff --git a/internal/ingestion/component/chunker/table_wire_invariant_test.go b/internal/ingestion/component/chunker/table_wire_invariant_test.gonew file mode 100644index 0000000000..7502bf0557--- /dev/null+++ b/internal/ingestion/component/chunker/table_wire_invariant_test.go@@ -0,0 +1,68 @@+package chunker++import (+ "testing"++ "ragflow/internal/agent/runtime"+)++// TestTableWireHasNoRowIRChunkTypes is the cross-chunker guard of the wire+// contract: the row IR (ck_type table_header/table_row) was deleted, so no+// chunker may ever emit those values again. Every registered ingestion+// chunker is enumerated rather than a hand-picked list, so a new chunker that+// reintroduces the row IR fails here.+func TestTableWireHasNoRowIRChunkTypes(t *testing.T) {+ inputs := map[string]any{+ "name": "orders.xlsx",+ "file_type": "xlsx",+ "output_format": "json",+ "json": []map[string]any{+ spreadsheetSegmentItem("Sheet1", []string{"ID", "Status"}, [][]string{{"A-1", "paid"}, {"A-2", "open"}}, 1, 2),+ },+ }+ tested := 0+ for _, name := range runtime.DefaultRegistry.NamesByCategory(runtime.CategoryIngestion) {+ factory, _, metadata, ok := runtime.DefaultRegistry.Lookup(name)+ if !ok || factory == nil {+ continue+ }+ if _, isChunker := metadata.Outputs["chunks"]; !isChunker {+ continue+ }+ component, err := factory(name, nil)+ if err != nil {+ // Some chunkers (e.g. GroupTitle) declare required parameters;+ // without them there is no component to exercise.+ t.Logf("skipping %s: %v", name, err)+ continue+ }+ out, err := component.Invoke(t.Context(), nil, inputs)+ if err != nil {+ t.Logf("skipping %s: %v", name, err)+ continue+ }+ if out["_ERROR"] != nil {+ t.Logf("skipping %s: %v", name, out["_ERROR"])+ continue+ }+ chunks, ok := out["chunks"].([]map[string]any)+ if !ok {+ t.Errorf("%s returned chunks of type %T, want []map[string]any", name, out["chunks"])+ continue+ }+ if len(chunks) == 0 {+ t.Logf("skipping %s: no chunks for the fixture", name)+ continue+ }+ for i, chunk := range chunks {+ kind, _ := chunk["ck_type"].(string)+ if kind == "table_header" || kind == "table_row" {+ t.Errorf("%s chunk[%d] ck_type = %q: the row IR must not come back", name, i, kind)+ }+ }+ tested+++ }+ if tested < 4 {+ t.Fatalf("exercised %d chunkers, want the registered chunker set", tested)+ }+}diff --git a/internal/ingestion/component/chunker/token.go b/internal/ingestion/component/chunker/token.goindex e58daa9511..a15703104d 100644--- a/internal/ingestion/component/chunker/token.go+++ b/internal/ingestion/component/chunker/token.go@@ -550,12 +550,6 @@ func (c *TokenChunkerComponent) invokeJSONPayload(ctx context.Context, items []s}lanes := partition(len(items), workers)perItem := make([][]schema.ChunkDoc, len(items))- dataTables := make(map[string]struct{})- for _, item := range items {- if item.CKType == "table_row" {- dataTables[spreadsheetTableKey(item)] = struct{}{}- }- }// Computed once before the goroutines so the split path can decide whether// the main delimiter is a custom (backtick) one — those are dropped, bare@@ -573,15 +567,6 @@ func (c *TokenChunkerComponent) invokeJSONPayload(ctx context.Context, items []sperItem[i] = nilcontinue}- if items[i].CKType == "table_header" {- if _, hasRows := dataTables[spreadsheetTableKey(items[i])]; hasRows {- // The typed header is metadata carried by every row. It- // must not become an independent TokenChunker chunk when- // row IR is consumed by a legacy pipeline.- perItem[i] = nil- continue- }- }if isTextParserSentenceFallback(fileType, c.param.Delimiters, items[i]) {perItem[i] = splitTextParserSentences(items[i])} else {@@ -727,14 +712,6 @@ func splitTextParserSentences(item schema.ChunkDoc) []schema.ChunkDoc {// the chunk reproduces the source exactly, but a custom (backtick) delimiter// is dropped (Python-compatible split instruction).func chunkFromItem(it schema.ChunkDoc, delimPattern *regexp.Regexp, keepDelim bool) []schema.ChunkDoc {- if it.CKType == "table_row" {- row := cloneChunkDoc(it)- row.DocType = "text"- row.CKType = "table_row"- row.Text = itemTextOrFallback(it)- row.TKNums = intPtr(tokenizeStr(row.Text))- return []schema.ChunkDoc{row}- }ckType := itemDocType(it)txt := itemTextOrFallback(it)if ckType != "text" {diff --git a/internal/ingestion/component/chunker/token_pdfpos_test.go b/internal/ingestion/component/chunker/token_pdfpos_test.goindex c07a8134eb..9bde5d471b 100644--- a/internal/ingestion/component/chunker/token_pdfpos_test.go+++ b/internal/ingestion/component/chunker/token_pdfpos_test.go@@ -93,7 +93,7 @@ func TestCloneChunkDoc_DeepCopiesPDFPositions(t *testing.T) {// TestMergeByTokenSizeFromJSON_PositionsDecodeToMatrix verifies the// chunker-side contract for diff 1.4: preserved `positions` must decode// (via ChunkDoc.ToMap → decodeStructuredValue) to a [][]float64 matrix so-// the downstream task-layer processChunkPositions → AddPositions can+// the downstream task-layer processChunkPositions → addPDFPositions can// convert it to page_num_int / top_int / position_int. The coordinate// conversion itself lives in internal/ingestion/task (processChunkPositions),// not in the chunker.diff --git a/internal/ingestion/component/chunker/token_test.go b/internal/ingestion/component/chunker/token_test.goindex 12c6827464..b7cb1cce7a 100644--- a/internal/ingestion/component/chunker/token_test.go+++ b/internal/ingestion/component/chunker/token_test.go@@ -70,21 +70,18 @@ func TestTokenChunker_InvokeEmptyInput(t *testing.T) {}}-func TestTokenChunkerPreservesSpreadsheetRowBoundaries(t *testing.T) {+func TestTokenChunkerPreservesSpreadsheetSegmentBoundaries(t *testing.T) {c, err := NewTokenChunker(map[string]any{"chunk_token_size": 512})if err != nil {t.Fatalf("NewTokenChunker: %v", err)}+ segOne := spreadsheetSegmentItem("Sheet1", []string{"ID", "Status"}, [][]string{{"A-1", "paid"}}, 1, 2)+ segTwo := spreadsheetSegmentItem("Sheet2", []string{"ID", "Status"}, [][]string{{"B-1", "open"}}, 2, 2)out, err := c.Invoke(context.Background(), nil, map[string]any{"name": "orders.xlsx","file_type": "xlsx","output_format": "json",- "json": []map[string]any{- {"text": "ID; Status", "doc_type_kwd": "table", "ck_type": "table_header", "sheet_index": 1},- {"text": "ID: A-1; Status: paid", "doc_type_kwd": "text", "ck_type": "table_row", "sheet_index": 1},- {"text": "ID; Status", "doc_type_kwd": "table", "ck_type": "table_header", "sheet_index": 2},- {"text": "ID: B-1; Status: open", "doc_type_kwd": "text", "ck_type": "table_row", "sheet_index": 2},- },+ "json": []map[string]any{segOne, segTwo},})if err != nil {t.Fatalf("Invoke: %v", err)@@ -94,14 +91,14 @@ func TestTokenChunkerPreservesSpreadsheetRowBoundaries(t *testing.T) {t.Fatalf("chunks = %T, want []map[string]any", out["chunks"])}if len(chunks) != 2 {- t.Fatalf("chunks = %#v, want one row chunk per sheet", chunks)+ t.Fatalf("chunks = %#v, want one chunk per segment", chunks)}- if chunks[0]["text"] != "ID: A-1; Status: paid" || chunks[1]["text"] != "ID: B-1; Status: open" {- t.Fatalf("row chunks = %#v", chunks)+ if chunks[0]["text"] != segOne["text"] || chunks[1]["text"] != segTwo["text"] {+ t.Fatalf("segment chunks = %#v", chunks)}for i, chunk := range chunks {- if chunk["ck_type"] != "table_row" {- t.Errorf("chunk[%d] ck_type = %v, want table_row", i, chunk["ck_type"])+ if chunk["ck_type"] != "table" {+ t.Errorf("chunk[%d] ck_type = %v, want table", i, chunk["ck_type"])}}}@@ -111,25 +108,23 @@ func TestTokenChunkerPreservesHeaderOnlySheetAlongsideDataSheet(t *testing.T) {if err != nil {t.Fatalf("NewTokenChunker: %v", err)}+ segData := spreadsheetSegmentItem("Sheet1", []string{"ID", "Status"}, [][]string{{"A-1", "paid"}}, 1, 2)+ segHeaderOnly := spreadsheetSegmentItem("Sheet2", []string{"Name", "Owner"}, nil, 2, 1)out, err := c.Invoke(t.Context(), nil, map[string]any{"name": "orders.xlsx","file_type": "xlsx","output_format": "json",- "json": []map[string]any{- {"text": "ID; Status", "doc_type_kwd": "table", "ck_type": "table_header", "table_id": "sheet-1", "sheet_index": 1},- {"text": "ID: A-1; Status: paid", "doc_type_kwd": "text", "ck_type": "table_row", "table_id": "sheet-1", "sheet_index": 1},- {"text": "Name; Owner", "doc_type_kwd": "table", "ck_type": "table_header", "table_id": "sheet-2", "sheet_index": 2},- },+ "json": []map[string]any{segData, segHeaderOnly},})if err != nil {t.Fatalf("Invoke: %v", err)}chunks, _ := out["chunks"].([]map[string]any)if len(chunks) != 2 {- t.Fatalf("chunks = %#v, want data row and header-only sheet", chunks)+ t.Fatalf("chunks = %#v, want data segment and header-only segment", chunks)}- if chunks[0]["text"] != "ID: A-1; Status: paid" || chunks[1]["text"] != "Name; Owner" {- t.Fatalf("chunks = %#v, want row followed by header-only sheet", chunks)+ if chunks[0]["text"] != segData["text"] || chunks[1]["text"] != segHeaderOnly["text"] {+ t.Fatalf("chunks = %#v, want data segment followed by header-only segment", chunks)}}diff --git a/internal/ingestion/component/chunker/xlsx_qa_regression_test.go b/internal/ingestion/component/chunker/xlsx_qa_regression_test.goindex 2434affcd9..719e6c7a02 100644--- a/internal/ingestion/component/chunker/xlsx_qa_regression_test.go+++ b/internal/ingestion/component/chunker/xlsx_qa_regression_test.go@@ -66,8 +66,8 @@ func qaChunksFromXLSX(t *testing.T, data []byte) []map[string]any {}// TestXLSXQAFirstRowIsData protects the QA spreadsheet contract: workbooks-// contain question/answer rows without a header, so the parser's structural-// table_header item is still the first QA pair.+// contain question/answer rows without a header, so the leading header row+// of the segment (the first <tr>) is still the first QA pair.func TestXLSXQAFirstRowIsData(t *testing.T) {chunks := qaChunksFromXLSX(t, xlsxWorkbook(t, [][]string{{"q1", "a1"},diff --git a/internal/ingestion/component/schema/chunk_types.go b/internal/ingestion/component/schema/chunk_types.goindex 6648dd6b6d..08ec8e9821 100644--- a/internal/ingestion/component/schema/chunk_types.go+++ b/internal/ingestion/component/schema/chunk_types.go@@ -124,15 +124,8 @@ type ChunkDoc struct {TopInt []int `json:"top_int,omitempty"`PDFPositions json.RawMessage `json:"_pdf_positions,omitempty"`Positions json.RawMessage `json:"positions,omitempty"`- TableID string `json:"table_id,omitempty"`Sheet string `json:"sheet,omitempty"`SheetIndex *int `json:"sheet_index,omitempty"`- Headers []string `json:"headers,omitempty"`- Cells []string `json:"cells,omitempty"`- RowStart *int `json:"row_start,omitempty"`- RowEnd *int `json:"row_end,omitempty"`- ColStart *int `json:"col_start,omitempty"`- ColEnd *int `json:"col_end,omitempty"`Extra map[string]json.RawMessage `json:"-"`}@@ -156,7 +149,7 @@ func (d *ChunkDoc) UnmarshalJSON(data []byte) error {"context_above", "context_below", "questions", "keywords", "summary","chunk_order_int", "title_tks", "title_sm_tks", "content_ltks","content_sm_ltks", "tag_kwd", "page_number", "top_int", "_pdf_positions", "positions",- "table_id", "sheet", "sheet_index", "headers", "cells", "row_start", "row_end", "col_start", "col_end",+ "sheet", "sheet_index",} {delete(raw, key)}diff --git a/internal/ingestion/component/schema/schema_test.go b/internal/ingestion/component/schema/schema_test.goindex db1f7ce946..14e5088708 100644--- a/internal/ingestion/component/schema/schema_test.go+++ b/internal/ingestion/component/schema/schema_test.go@@ -237,20 +237,13 @@ func TestChunkerFromUpstreamJSONRoundTrip(t *testing.T) {}func TestChunkDocSpreadsheetFieldsRoundTrip(t *testing.T) {- sheetIndex, rowStart, rowEnd, colStart, colEnd := 2, 42, 42, 1, 3+ sheetIndex := 2original := ChunkDoc{- Text: "ID:A-100; Status:paid",- DocType: "text",- CKType: "table_row",- TableID: "sheet-2",+ Text: "<table><tr><th>ID</th></tr></table>",+ DocType: "table",+ CKType: "table",Sheet: "Orders",SheetIndex: &sheetIndex,- Headers: []string{"ID", "Status", "Note"},- Cells: []string{"A-100", "paid", ""},- RowStart: &rowStart,- RowEnd: &rowEnd,- ColStart: &colStart,- ColEnd: &colEnd,}data, err := json.Marshal(original)@@ -261,20 +254,29 @@ func TestChunkDocSpreadsheetFieldsRoundTrip(t *testing.T) {if err := json.Unmarshal(data, &decoded); err != nil {t.Fatalf("unmarshal: %v", err)}- if decoded.TableID != original.TableID || decoded.Sheet != original.Sheet {- t.Fatalf("spreadsheet identity mismatch: got table=%q sheet=%q", decoded.TableID, decoded.Sheet)+ if decoded.Sheet != original.Sheet {+ t.Fatalf("sheet mismatch: got %q", decoded.Sheet)}if decoded.SheetIndex == nil || *decoded.SheetIndex != sheetIndex {t.Fatalf("sheet index mismatch: got %v", decoded.SheetIndex)}- if decoded.RowStart == nil || *decoded.RowStart != rowStart || decoded.RowEnd == nil || *decoded.RowEnd != rowEnd {- t.Fatalf("row range mismatch: start=%v end=%v", decoded.RowStart, decoded.RowEnd)- }- if decoded.ColStart == nil || *decoded.ColStart != colStart || decoded.ColEnd == nil || *decoded.ColEnd != colEnd {- t.Fatalf("column range mismatch: start=%v end=%v", decoded.ColStart, decoded.ColEnd)+}++// TestChunkDocLegacyRowIRKeysPassThrough: the deleted row-IR keys (table_id,+// headers, cells and the per-row coordinate fields) are no longer typed+// fields, but payloads that still carry them must survive a decode/encode+// round trip through Extra; the index boundary strips them from the store.+func TestChunkDocLegacyRowIRKeysPassThrough(t *testing.T) {+ var decoded ChunkDoc+ raw := `{"text":"row","doc_type_kwd":"text","ck_type":"table_row","table_id":"sheet-2","headers":["ID","Status"],"cells":["A-100","paid"],"row_start":42,"row_end":42,"col_start":1,"col_end":3}`+ if err := json.Unmarshal([]byte(raw), &decoded); err != nil {+ t.Fatalf("unmarshal: %v", err)}- if !reflect.DeepEqual(decoded.Headers, original.Headers) || !reflect.DeepEqual(decoded.Cells, original.Cells) {- t.Fatalf("cells mismatch: headers=%v cells=%v", decoded.Headers, decoded.Cells)+ out := decoded.ToMap()+ for _, key := range []string{"table_id", "headers", "cells", "row_start", "row_end", "col_start", "col_end"} {+ if _, ok := out[key]; !ok {+ t.Errorf("legacy key %q lost in round trip: %#v", key, out)+ }}}@@ -683,5 +685,3 @@ func TestFlattenLegacyParserSetups(t *testing.T) {})}}--func ptrString(s string) *string { return &s }diff --git a/internal/ingestion/component/tokenizer_unit_test.go b/internal/ingestion/component/tokenizer_unit_test.goindex 9118ddf10b..2fcbd4b473 100644--- a/internal/ingestion/component/tokenizer_unit_test.go+++ b/internal/ingestion/component/tokenizer_unit_test.go@@ -430,7 +430,7 @@ func TestValidateTokenizerOutputs_SymbolOnlyContentLtksIsEmptyFails(t *testing.T// Tokenizer-(T)1: the tokenizer emits chunks via schema.ChunkDocsToMaps// (ChunkDoc.ToMap), which must carry the raw `positions` / `_pdf_positions`// through untouched so the downstream executor stage-// (internal/ingestion/task processChunkPositions → AddPositions) can convert+// (internal/ingestion/task processChunkPositions → addPDFPositions) can convert// them into position_int / page_num_int / top_int exactly once. This does NOT// require the C++ analyzer pool, so it runs under plain `go test`.func TestChunkDocsToMaps_PreservesPDFPositions(t *testing.T) {diff --git a/internal/ingestion/task/indexdoc/position.go b/internal/ingestion/task/indexdoc/position.goindex 26ef0a65df..c68bb8d074 100644--- a/internal/ingestion/task/indexdoc/position.go+++ b/internal/ingestion/task/indexdoc/position.go@@ -16,38 +16,63 @@package indexdoc-// AddPositions adds position fields to a chunk map.-// Input positions is a flat []float64 grouped as [pn, left, right, top, bottom]-// every 5 elements. pn is ALREADY 1-indexed — the 0→1 conversion happens-// once, at the parser boundary (normalizePDFPageNumber for the DeepDoc PDF-// path; TCADP writes 1-indexed directly). This function is a passthrough: it-// must NOT add +1, otherwise the PDF path (which already normalized) would-// double-increment page numbers.-//-// Mirrors Python: rag.nlp.add_positions() (Python adds +1 because its-// callers feed 0-indexed values; the Go pipeline normalizes earlier).-func AddPositions(chunk map[string]any, positions []float64) {+// positionsToIntMatrix groups a flat position array by five and truncates each+// value to int — the stored shape of position_int. It returns nil for an empty+// array or one whose length is not a multiple of five: malformed input is+// dropped, never reinterpreted.+func positionsToIntMatrix(positions []float64) [][]int {if len(positions) == 0 || len(positions)%5 != 0 {- return+ return nil}- n := len(positions) / 5- pageNumInt := make([]int, 0, n)- topInt := make([]int, 0, n)- positionInt := make([][]int, 0, n)-+ matrix := make([][]int, 0, len(positions)/5)for i := 0; i < len(positions); i += 5 {- pn := int(positions[i]) // already 1-indexed- left := int(positions[i+1])- right := int(positions[i+2])- top := int(positions[i+3])- bottom := int(positions[i+4])-- pageNumInt = append(pageNumInt, pn)- topInt = append(topInt, top)- positionInt = append(positionInt, []int{pn, left, right, top, bottom})+ matrix = append(matrix, []int{+ int(positions[i]), int(positions[i+1]), int(positions[i+2]),+ int(positions[i+3]), int(positions[i+4]),+ })}+ return matrix+}+// addPDFPositions stores PDF layout boxes: positions is a flat []float64+// grouped by five as [page, left, right, top, bottom] and is written to+// page_num_int / top_int / position_int. Page numbers are ALREADY 1-indexed on+// entry — the 0→1 conversion happens once, at the parser boundary+// (normalizePDFPageNumber for the DeepDoc PDF path; TCADP writes 1-indexed+// directly) — so this is a passthrough and must NOT add +1, or the PDF path+// would double-increment.+//+// Mirrors Python: rag.nlp.add_positions() (Python adds +1 because its callers+// feed 0-indexed values; the Go pipeline normalizes earlier).+func addPDFPositions(chunk map[string]any, positions []float64) {+ matrix := positionsToIntMatrix(positions)+ if matrix == nil {+ return+ }+ pageNumInt := make([]int, 0, len(matrix))+ topInt := make([]int, 0, len(matrix))+ for _, box := range matrix {+ pageNumInt = append(pageNumInt, box[0])+ topInt = append(topInt, box[3])+ }chunk["page_num_int"] = pageNumIntchunk["top_int"] = topInt- chunk["position_int"] = positionInt+ chunk["position_int"] = matrix+}++// addSpreadsheetPositions stores spreadsheet cell ranges: positions is a flat+// []float64 grouped by five as [sheet, rowStart, rowEnd, colStart, colEnd] and+// is written to position_int only — that field is the document preview's+// coordinate carrier (the chunk API serves "positions" from it, and the Excel+// preview reads the five values positionally as sheet / rows / columns).+//+// page_num_int and top_int are PDF layout fields: deriving them from these+// tuples would store a sheet number as a page and a column as a top, and would+// clobber the row index the QA chunker wrote to top_int (the same field+// Python's beAdoc/beAdocDocx set from row_num for its spreadsheet chunks,+// which never carry positions at all).+func addSpreadsheetPositions(chunk map[string]any, positions []float64) {+ if matrix := positionsToIntMatrix(positions); matrix != nil {+ chunk["position_int"] = matrix+ }}diff --git a/internal/ingestion/task/indexdoc/position_test.go b/internal/ingestion/task/indexdoc/position_test.goindex 50464b0d55..551275b4b3 100644--- a/internal/ingestion/task/indexdoc/position_test.go+++ b/internal/ingestion/task/indexdoc/position_test.go@@ -5,7 +5,7 @@ import ()// =============================================================================-// AddPositions+// addPDFPositions// Canonical format: [pageNum, left, right, top, bottom] × N// Contract: input page numbers are ALREADY 1-indexed (the 0→1 conversion// happens once, at the parser boundary in normalizePDFPageNumber). This@@ -14,11 +14,11 @@ import (// double-incremented page number.// =============================================================================-func TestAddPositions_Basic(t *testing.T) {+func TestAddPDFPositions_Basic(t *testing.T) {chunk := map[string]any{}// [pn=1 (first page, 1-indexed), left=100, right=50, top=200, bottom=150]positions := []float64{1, 100, 50, 200, 150}- AddPositions(chunk, positions)+ addPDFPositions(chunk, positions)pageNum, ok := chunk["page_num_int"].([]int)if !ok || len(pageNum) != 1 || pageNum[0] != 1 {@@ -37,13 +37,13 @@ func TestAddPositions_Basic(t *testing.T) {}}-func TestAddPositions_MultiplePositions(t *testing.T) {+func TestAddPDFPositions_MultiplePositions(t *testing.T) {chunk := map[string]any{}positions := []float64{1, 100, 50, 200, 150, // pn=1, left=100, right=50, top=200, bottom=1502, 200, 60, 300, 250, // pn=2, left=200, right=60, top=300, bottom=250}- AddPositions(chunk, positions)+ addPDFPositions(chunk, positions)pageNum := chunk["page_num_int"].([]int)if len(pageNum) != 2 || pageNum[0] != 1 || pageNum[1] != 2 {@@ -59,36 +59,36 @@ func TestAddPositions_MultiplePositions(t *testing.T) {}}-func TestAddPositions_NilPositions(t *testing.T) {+func TestAddPDFPositions_NilPositions(t *testing.T) {chunk := map[string]any{}- AddPositions(chunk, nil)+ addPDFPositions(chunk, nil)if _, exists := chunk["page_num_int"]; exists {t.Error("page_num_int should not be set for nil positions")}}-func TestAddPositions_EmptyPositions(t *testing.T) {+func TestAddPDFPositions_EmptyPositions(t *testing.T) {chunk := map[string]any{}- AddPositions(chunk, []float64{})+ addPDFPositions(chunk, []float64{})if _, exists := chunk["page_num_int"]; exists {t.Error("page_num_int should not be set for empty positions")}}-func TestAddPositions_PartialPositions(t *testing.T) {+func TestAddPDFPositions_PartialPositions(t *testing.T) {chunk := map[string]any{}positions := []float64{1, 100} // only 2 elements, not a complete position- AddPositions(chunk, positions)+ addPDFPositions(chunk, positions)if _, exists := chunk["page_num_int"]; exists {t.Error("page_num_int should not be set for partial positions")}}-func TestAddPositions_PassthroughNoOffset(t *testing.T) {- // page numbers are 1-indexed on entry; AddPositions must not add +1.+func TestAddPDFPositions_PassthroughNoOffset(t *testing.T) {+ // page numbers are 1-indexed on entry; addPDFPositions must not add +1.chunk := map[string]any{}positions := []float64{6, 100, 50, 200, 150} // pn=6, 1-indexed- AddPositions(chunk, positions)+ addPDFPositions(chunk, positions)pageNum := chunk["page_num_int"].([]int)if pageNum[0] != 6 {@@ -99,3 +99,43 @@ func TestAddPositions_PassthroughNoOffset(t *testing.T) {t.Errorf("position_int[0][0] = %d, want 6 (passthrough, no +1)", position[0][0])}}++// =============================================================================+// addSpreadsheetPositions+// Vocabulary: [sheet, rowStart, rowEnd, colStart, colEnd] × N+// Contract: position_int is the preview's coordinate carrier and is written;+// page_num_int/top_int are PDF layout fields and must NOT be derived from these+// tuples, so a chunk's own top_int (the QA row index) survives.+// =============================================================================++func TestAddSpreadsheetPositions_WritesPositionIntOnly(t *testing.T) {+ chunk := map[string]any{"top_int": []int{41}}+ addSpreadsheetPositions(chunk, []float64{2, 42, 42, 1, 3})++ matrix, ok := chunk["position_int"].([][]int)+ if !ok || len(matrix) != 1 || len(matrix[0]) != 5 {+ t.Fatalf("position_int = %v, want one five-field tuple", chunk["position_int"])+ }+ want := []int{2, 42, 42, 1, 3}+ for i, v := range want {+ if matrix[0][i] != v {+ t.Fatalf("position_int[0] = %v, want %v", matrix[0], want)+ }+ }+ if _, exists := chunk["page_num_int"]; exists {+ t.Errorf("page_num_int must not be derived from a spreadsheet tuple: %v", chunk["page_num_int"])+ }+ if top, ok := chunk["top_int"].([]int); !ok || len(top) != 1 || top[0] != 41 {+ t.Errorf("top_int = %v, want the chunk's own row index [41]", chunk["top_int"])+ }+}++func TestAddSpreadsheetPositions_MalformedNoop(t *testing.T) {+ for _, positions := range [][]float64{nil, {}, {2, 42, 42, 1}} {+ chunk := map[string]any{}+ addSpreadsheetPositions(chunk, positions)+ if _, exists := chunk["position_int"]; exists {+ t.Errorf("positions %v must be dropped, got %v", positions, chunk["position_int"])+ }+ }+}diff --git a/internal/ingestion/task/indexdoc/process.go b/internal/ingestion/task/indexdoc/process.goindex 453172dfbd..5e80a9c937 100644--- a/internal/ingestion/task/indexdoc/process.go+++ b/internal/ingestion/task/indexdoc/process.go@@ -94,10 +94,13 @@ func ProcessChunksForPipeline(}cleanupConsumedChunkFields(ck)+ // The spreadsheet identity is keyed off fields the strip is about to+ // remove, so it must be read before it.+ spreadsheet := isSpreadsheetChunk(ck)stripPipelineOnlyFields(ck)metadata = mergeChunkMetadata(metadata, ck)RenameTextToContentWithWeight(ck)- processChunkPositions(ck)+ processChunkPositions(ck, spreadsheet)}return metadata, nil}@@ -188,23 +191,31 @@ func mergeChunkMetadata(metadata map[string]any, ck map[string]any) map[string]a}// processChunkPositions converts the raw "positions" field into indexable-// position fields (page_num_int, top_int, position_int) via AddPositions,-// then removes the raw "positions" and the sibling "_pdf_positions" internal-// field. _pdf_positions is the parser-emitted position matrix consumed by-// the chunker's image-crop pass (pdfcrop_cgo.go); after the chunker it is-// dead weight with no index column, so it is pruned unconditionally —-// independent of "positions" and not gated on the early-return below.+// position fields, then removes the raw "positions" and the sibling+// "_pdf_positions" internal field. _pdf_positions is the parser-emitted+// position matrix consumed by the chunker's image-crop pass (pdfcrop_cgo.go);+// after the chunker it is dead weight with no index column, so it is pruned+// unconditionally — independent of "positions" and not gated on the+// early-return below.+//+// The field carries two vocabularies, and the branch on them is explicit+// rather than inferred from the shape:+// - PDF items: [page, left, right, top, bottom] → page_num_int, top_int and+// position_int (addPDFPositions);+// - spreadsheet items (identity present): [sheet, rowStart, rowEnd,+// colStart, colEnd] → position_int only (addSpreadsheetPositions), which+// leaves the chunk's own top_int — the QA chunker's row index — intact.//// Two source types reach this point:-// - []float64 — flat array of 5-tuples [page,left,right,top,bottom,…] from-// parsers that emit positions directly as a flat float64 slice.+// - []float64 — flat array of 5-tuples from parsers that emit positions+// directly as a flat float64 slice.// - [][]float64 — the production path: positions flow through ChunkDoc// (json.RawMessage → decodeStructuredValue) which produces a slice of// 5-element groups.//-// Both are flattened into a single []float64 for AddPositions, which groups-// by 5 internally. Unexpected types are logged and discarded.-func processChunkPositions(ck map[string]any) {+// Both are flattened into a single []float64, which the store helpers group by+// five. Unexpected types are logged and discarded.+func processChunkPositions(ck map[string]any, spreadsheet bool) {delete(ck, "_pdf_positions")poss, exists := ck["positions"]if !exists {@@ -212,19 +223,38 @@ func processChunkPositions(ck map[string]any) {}switch v := poss.(type) {case []float64:- AddPositions(ck, v)+ storePositions(ck, v, spreadsheet)case [][]float64:flat := make([]float64, 0, len(v)*5)for _, group := range v {flat = append(flat, group...)}- AddPositions(ck, flat)+ storePositions(ck, flat, spreadsheet)default:common.Warn(fmt.Sprintf("chunk positions unexpected type %T; discarding", poss))}delete(ck, "positions")}+func storePositions(ck map[string]any, flat []float64, spreadsheet bool) {+ if spreadsheet {+ addSpreadsheetPositions(ck, flat)+ return+ }+ addPDFPositions(ck, flat)+}++// isSpreadsheetChunk reports whether a chunk's "positions" carry the+// spreadsheet vocabulary, judged by the identity fields the strip removes+// later in the chunk loop.+func isSpreadsheetChunk(ck map[string]any) bool {+ if v, ok := ck["sheet_index"]; ok && v != nil {+ return true+ }+ sheet, ok := ck["sheet"].(string)+ return ok && sheet != ""+}+// AggregateTableDocMetadata collects unique per-column values across all chunks// for columns with role "metadata" or "both", merges them into document metadata.// Mirrors Python: rag/utils/table_es_metadata.py:aggregate_table_doc_metadatadiff --git a/internal/ingestion/task/indexdoc/process_test.go b/internal/ingestion/task/indexdoc/process_test.goindex 4fe492a740..ca453519aa 100644--- a/internal/ingestion/task/indexdoc/process_test.go+++ b/internal/ingestion/task/indexdoc/process_test.go@@ -351,7 +351,7 @@ func TestProcessChunkPositions_FlatFloat64(t *testing.T) {// positions is 1-indexed (parser normalized before we see it)"positions": []float64{1, 100, 50, 200, 150},}- processChunkPositions(chunk)+ processChunkPositions(chunk, false)if _, exists := chunk["positions"]; exists {t.Fatal("positions key must be removed")@@ -369,7 +369,7 @@ func TestProcessChunkPositions_2DFloat64(t *testing.T) {{2, 200, 60, 300, 250},},}- processChunkPositions(chunk)+ processChunkPositions(chunk, false)if _, exists := chunk["positions"]; exists {t.Fatal("positions key must be removed")@@ -391,7 +391,7 @@ func TestProcessChunkPositions_NoPositions(t *testing.T) {"text": "hello","_pdf_positions": []any{[]any{0, 1, 2, 3, 4}},}- processChunkPositions(chunk)+ processChunkPositions(chunk, false)if _, exists := chunk["page_num_int"]; exists {t.Error("page_num_int must not be set when positions is missing")}@@ -497,3 +497,63 @@ func TestProcessChunksForPipeline_StripsPipelineOnlyFields(t *testing.T) {t.Errorf("doc_id = %v, want doc-1 (the strip must not touch persist fields)", ck["doc_id"])}}++// =============================================================================+// processChunkPositions — the two position vocabularies+// =============================================================================++// TestProcessChunkPositions_SpreadsheetKeepsRowIndex: a spreadsheet chunk's+// positions become position_int (the preview carrier) instead of being decoded+// as PDF boxes — no page_num_int, and the chunk's own top_int (the QA row+// index, aligned with Python's beAdoc) survives.+func TestProcessChunkPositions_SpreadsheetKeepsRowIndex(t *testing.T) {+ chunk := map[string]any{+ "positions": [][]float64{{2, 42, 42, 1, 3}},+ "top_int": []int{41},+ }+ processChunkPositions(chunk, true)++ matrix, ok := chunk["position_int"].([][]int)+ if !ok || len(matrix) != 1 || matrix[0][0] != 2 || matrix[0][4] != 3 {+ t.Fatalf("position_int = %v, want the sheet tuple", chunk["position_int"])+ }+ if _, exists := chunk["page_num_int"]; exists {+ t.Errorf("page_num_int must not be derived from spreadsheet tuples: %v", chunk["page_num_int"])+ }+ if top, ok := chunk["top_int"].([]int); !ok || len(top) != 1 || top[0] != 41 {+ t.Errorf("top_int = %v, want the chunk's own row index [41]", chunk["top_int"])+ }+ if _, exists := chunk["positions"]; exists {+ t.Error("raw positions must be removed")+ }+}++// TestProcessChunksForPipeline_SpreadsheetPositionsKeepRowIndex pins the whole+// chunk loop: the spreadsheet identity has to be read before+// stripPipelineOnlyFields removes sheet_index, or the positions of a+// spreadsheet chunk would be decoded as PDF layout boxes.+func TestProcessChunksForPipeline_SpreadsheetPositionsKeepRowIndex(t *testing.T) {+ chunks := []map[string]any{{+ "text": "<table><tr><th>q</th><th>a</th></tr></table>",+ "sheet_index": 1,+ "top_int": []int{41},+ "positions": [][]float64{{1, 42, 42, 1, 3}},+ }}+ if _, err := ProcessChunksForPipeline(chunks, "doc-1", "doc.xlsx", time.Now()); err != nil {+ t.Fatalf("ProcessChunksForPipeline: %v", err)+ }+ ck := chunks[0]+ if _, exists := ck["sheet_index"]; exists {+ t.Error("sheet_index must be stripped at the index boundary")+ }+ matrix, ok := ck["position_int"].([][]int)+ if !ok || len(matrix) != 1 || matrix[0][0] != 1 || matrix[0][1] != 42 {+ t.Fatalf("position_int = %v, want the sheet tuple", ck["position_int"])+ }+ if _, exists := ck["page_num_int"]; exists {+ t.Errorf("page_num_int must not be derived from spreadsheet tuples: %v", ck["page_num_int"])+ }+ if top, ok := ck["top_int"].([]int); !ok || len(top) != 1 || top[0] != 41 {+ t.Errorf("top_int = %v, want the chunk's own row index [41]", ck["top_int"])+ }+}diff --git a/internal/parser/parser/csv_parser.go b/internal/parser/parser/csv_parser.goindex 156e0ae9d5..28b8bbf02e 100644--- a/internal/parser/parser/csv_parser.go+++ b/internal/parser/parser/csv_parser.go@@ -50,7 +50,6 @@ const (type CSVParser struct {ParseMethod stringOutputFormat string- HTML4Excel boolTCADPAPIServer stringTCADPAPIKey stringTCADPTableResultType string@@ -78,9 +77,7 @@ func (p *CSVParser) ConfigureFromSetup(setup map[string]any) {if v, ok := setup["output_format"].(string); ok && v != "" {p.OutputFormat = v}- if v, ok := setup["html4excel"].(bool); ok {- p.HTML4Excel = v- }+ deprecatedHTML4Excel(setup, p.String())deprecatedChunkRows(setup, p.String())if v, ok := setup["tcadp_apiserver"].(string); ok && v != "" {p.TCADPAPIServer = v@@ -122,10 +119,6 @@ func (p *CSVParser) ParseWithResult(ctx context.Context, filename string, data [decoded, encName := DecodeToUTF8(data, "text/csv")text := string(decoded)if strings.TrimSpace(text) == "" {- var emptyJSON []map[string]any- if p.HTML4Excel {- emptyJSON = []map[string]any{NewTableJSONItem("<table><caption>Data</caption></table>", csvSheetName, [][]float64{{1, 1, 1, 1, 1}})}- }return ParseResult{OutputFormat: spreadsheetOutputFormat,File: map[string]any{@@ -135,7 +128,6 @@ func (p *CSVParser) ParseWithResult(ctx context.Context, filename string, data ["format": "csv","sheets": 1,},- JSON: emptyJSON,}}@@ -151,14 +143,9 @@ func (p *CSVParser) ParseWithResult(ctx context.Context, filename string, data [for i := range dataRows {dataRows[i] = i + 2}- var items []map[string]any- if p.HTML4Excel {- if table := recordsToHTMLTableItem(records, csvSheetName, 1, 1, dataRows); table != nil {- items = []map[string]any{table}- }- } else {- items = recordsToSpreadsheetItems(records, csvSheetName, 1, 1, dataRows)- }+ // One wire shape for csv as for xlsx: a single segmented HTML table with+ // row-aligned positions (csv has no in-cell image anchors).+ items := buildSheetItems(records, csvSheetName, 1, 1, dataRows, nil)return ParseResult{OutputFormat: spreadsheetOutputFormat,File: map[string]any{diff --git a/internal/parser/parser/csv_parser_test.go b/internal/parser/parser/csv_parser_test.goindex 8d1a0fc311..4e45e23515 100644--- a/internal/parser/parser/csv_parser_test.go+++ b/internal/parser/parser/csv_parser_test.go@@ -3,6 +3,7 @@ package parserimport ("context""fmt"+ "reflect""strings""testing")@@ -21,42 +22,65 @@ func TestCSVParser_EmitsSpreadsheetRows(t *testing.T) {t.Fatalf("res.OutputFormat = %q, want %q", res.OutputFormat, "json")}- if len(res.JSON) != 3 {- t.Fatalf("len(res.JSON) = %d, want header plus two rows", len(res.JSON))- }- if res.JSON[0]["ck_type"] != "table_header" {- t.Fatalf("header ck_type = %v, want table_header", res.JSON[0]["ck_type"])- }- if res.JSON[1]["ck_type"] != "table_row" {- t.Fatalf("row ck_type = %v, want table_row", res.JSON[1]["ck_type"])- }- if res.JSON[1]["text"] != "Name:Alice; Age:30; City:New York ——Data" {- t.Errorf("row text = %v", res.JSON[1]["text"])- }- if res.JSON[1]["sheet"] != "Data" || res.JSON[1]["sheet_index"] != 1 {- t.Errorf("row sheet metadata = %#v", res.JSON[1])- }- positions, ok := res.JSON[1]["positions"].([][]float64)- if !ok || len(positions) == 0 {- t.Fatalf("item positions invalid: %v", res.JSON[1]["positions"])- }- pos := positions[0]- // [sheet, rowStart, rowEnd, colStart, colEnd]- if len(pos) != 5 || pos[0] != 1 || pos[1] != 2 || pos[2] != 2 || pos[3] != 1 || pos[4] != 3 {- t.Errorf("unexpected positions: %v", pos)+ // One wire shape: the whole sheet is one segmented HTML table item.+ if len(res.JSON) != 1 {+ t.Fatalf("len(res.JSON) = %d, want one HTML table item", len(res.JSON))+ }+ item := res.JSON[0]+ if item["ck_type"] != "table" || item["doc_type_kwd"] != "table" {+ t.Fatalf("item types = %v/%v, want table/table", item["ck_type"], item["doc_type_kwd"])+ }+ wantText := "<table><caption>Data</caption>\n" ++ "<tr><th>Name</th><th>Age</th><th>City</th></tr>\n" ++ "<tr><td>Alice</td><td>30</td><td>New York</td></tr>\n" ++ "<tr><td>Bob</td><td>25</td><td>San Francisco</td></tr>\n" ++ "</table>\n"+ if item["text"] != wantText {+ t.Errorf("text = %q, want %q", item["text"], wantText)+ }+ if item["sheet"] != "Data" || item["sheet_index"] != 1 {+ t.Errorf("sheet metadata = %#v", item)+ }+ positions, ok := item["positions"].([][]float64)+ if !ok {+ t.Fatalf("item positions invalid: %T", item["positions"])+ }+ // Row-aligned matrix: one tuple per <tr> in markup order, header first.+ if len(positions) != 3 {+ t.Fatalf("len(positions) = %d, want 3 (one per <tr>)", len(positions))+ }+ for i, want := range [][]float64{{1, 1, 1, 1, 3}, {1, 2, 2, 1, 3}, {1, 3, 3, 1, 3}} {+ for j := range want {+ if positions[i][j] != want[j] {+ t.Errorf("positions[%d] = %v, want %v", i, positions[i], want)+ break+ }+ }}}-func TestCSVParserHTML4ExcelRemainsAtomic(t *testing.T) {+// TestCSVParserHTML4ExcelIsIgnored pins the retirement: html4excel is still+// accepted at the entry (with a deprecation warning) but selects nothing —+// the wire is identical to the default path.+func TestCSVParserHTML4ExcelIsIgnored(t *testing.T) {+ data := []byte("Question,Answer\nQ1,A1\n")+ plain := NewCSVParser()+ plainRes := plain.ParseWithResult(context.Background(), "qa.csv", data)+ if plainRes.Err != nil {+ t.Fatalf("ParseWithResult failed: %v", plainRes.Err)+ }p := NewCSVParser()p.ConfigureFromSetup(map[string]any{"html4excel": true})- res := p.ParseWithResult(context.Background(), "qa.csv", []byte("Question,Answer\nQ1,A1\n"))+ res := p.ParseWithResult(context.Background(), "qa.csv", data)if res.Err != nil {t.Fatalf("ParseWithResult failed: %v", res.Err)}if len(res.JSON) != 1 || res.JSON[0]["ck_type"] != "table" {t.Fatalf("items = %#v, want one table item", res.JSON)}+ if !reflect.DeepEqual(res.JSON, plainRes.JSON) {+ t.Errorf("html4excel changed the wire: %v vs %v", res.JSON, plainRes.JSON)+ }}func TestCSVParser_ConfigureOutputFormatJSON(t *testing.T) {@@ -74,11 +98,14 @@ func TestCSVParser_ConfigureOutputFormatJSON(t *testing.T) {if res.OutputFormat != "json" {t.Fatalf("res.OutputFormat = %q, want 'json'", res.OutputFormat)}- if len(res.JSON) != 2 {- t.Fatalf("len(res.JSON) = %d, want header plus row", len(res.JSON))+ if len(res.JSON) != 1 {+ t.Fatalf("len(res.JSON) = %d, want one HTML table item", len(res.JSON))+ }+ if res.JSON[0]["ck_type"] != "table" {+ t.Fatalf("ck_type = %v, want table", res.JSON[0]["ck_type"])}- if res.JSON[1]["ck_type"] != "table_row" {- t.Fatalf("row ck_type = %v, want table_row", res.JSON[1]["ck_type"])+ if positions := res.JSON[0]["positions"].([][]float64); len(positions) != 2 {+ t.Fatalf("len(positions) = %d, want one tuple per <tr>", len(positions))}}@@ -110,11 +137,14 @@ func TestCSVParser_PreservesEveryDataRow(t *testing.T) {t.Fatalf("ParseWithResult failed: %v", res.Err)}- if len(res.JSON) != 5 {- t.Fatalf("len(res.JSON) = %d, want header plus four rows", len(res.JSON))+ if len(res.JSON) != 1 {+ t.Fatalf("len(res.JSON) = %d, want one HTML table item", len(res.JSON))}- for i, item := range res.JSON[1:] {- pos := item["positions"].([][]float64)[0]+ positions := res.JSON[0]["positions"].([][]float64)+ if len(positions) != 5 {+ t.Fatalf("len(positions) = %d, want one tuple per <tr> (header plus four rows)", len(positions))+ }+ for i, pos := range positions[1:] {wantRow := float64(i + 2)if pos[1] != wantRow || pos[2] != wantRow {t.Errorf("row %d position = [%v, %v], want [%v, %v]", i, pos[1], pos[2], wantRow, wantRow)@@ -128,14 +158,18 @@ func TestCSVParserKeepsVariableRowWidths(t *testing.T) {if res.Err != nil {t.Fatalf("ParseWithResult failed: %v", res.Err)}- if len(res.JSON) != 3 {- t.Fatalf("items = %d, want header plus two rows", len(res.JSON))+ if len(res.JSON) != 1 {+ t.Fatalf("items = %d, want one HTML table item", len(res.JSON))}- if got := len(res.JSON[1]["cells"].([]string)); got != 2 {- t.Fatalf("first data row width = %d, want 2", got)+ text, _ := res.JSON[0]["text"].(string)+ if !strings.Contains(text, "<tr><td>Q1</td><td>A1</td></tr>") ||+ !strings.Contains(text, "<tr><td>Q2</td><td>A2</td><td>extra</td></tr>") {+ t.Fatalf("row widths not preserved in markup: %q", text)}- if got := len(res.JSON[2]["cells"].([]string)); got != 3 {- t.Fatalf("second data row width = %d, want 3", got)+ positions := res.JSON[0]["positions"].([][]float64)+ // Per-row column ranges: Q1 row spans cols 1-2, Q2 row spans 1-3.+ if len(positions) != 3 || positions[1][4] != 2 || positions[2][4] != 3 {+ t.Fatalf("position matrix lost per-row widths: %v", positions)}}@@ -148,8 +182,13 @@ func TestCSVParser_ReadsTheSeparatorTheFileWasWrittenWith(t *testing.T) {if res.Err != nil {t.Fatalf("separator %q: ParseWithResult failed: %v", sep, res.Err)}- if got := res.JSON[len(res.JSON)-1]["text"]; got != "Name:Widget; Region:EU; Units:12 ——Data" {- t.Errorf("separator %q: row text = %v", sep, got)+ if len(res.JSON) != 1 {+ t.Fatalf("separator %q: items = %d, want one HTML table item", sep, len(res.JSON))+ }+ text, _ := res.JSON[0]["text"].(string)+ if !strings.Contains(text, "<tr><th>Name</th><th>Region</th><th>Units</th></tr>") ||+ !strings.Contains(text, "<tr><td>Widget</td><td>EU</td><td>12</td></tr>") {+ t.Errorf("separator %q: row not split into three columns: %q", sep, text)}}}@@ -214,7 +253,11 @@ func TestCSVParser_KeepsAnEmptyTabSeparatedField(t *testing.T) {if res.Err != nil {t.Fatalf("ParseWithResult failed: %v", res.Err)}- if got := res.JSON[len(res.JSON)-1]["text"]; got != "Name:Widget; Units:12 ——Data" {- t.Errorf("row text = %v", got)+ if len(res.JSON) != 1 {+ t.Fatalf("items = %d, want one HTML table item", len(res.JSON))+ }+ text, _ := res.JSON[0]["text"].(string)+ if !strings.Contains(text, "<tr><td>Widget</td><td></td><td>12</td></tr>") {+ t.Errorf("empty tab-separated field not kept: %q", text)}}diff --git a/internal/parser/parser/office_table_render.go b/internal/parser/parser/office_table_render.goindex 98a86d677c..f8c3db1df5 100644--- a/internal/parser/parser/office_table_render.go+++ b/internal/parser/parser/office_table_render.go@@ -42,100 +42,33 @@ var tableIllegalCharsRe = regexp.MustCompile(`[\x00-\x08]|\x0B|\x0C|[\x0E-\x1F]`// value parsing.var numericCellRe = regexp.MustCompile(`^[\$\+\-]?[\d,]+(\.\d+)?%?$`)-func recordsToSpreadsheetItems(records [][]string, sheet string, sheetIndex, headerRow int, dataRows []int) []map[string]any {- if len(records) == 0 {- return nil- }- if headerRow <= 0 {- headerRow = 1- }- if len(dataRows) != len(records)-1 {- dataRows = make([]int, len(records)-1)- for i := range dataRows {- dataRows[i] = headerRow + i + 1- }- }- maxCols := 0- for _, row := range records {- if len(row) > maxCols {- maxCols = len(row)- }- }- if maxCols == 0 {- return nil- }- header := append([]string(nil), records[0]...)- if len(header) == 0 {- header = padSpreadsheetRow(header, maxCols)- }- headerColEnd := len(header)- tableID := fmt.Sprintf("sheet-%d", sheetIndex)- items := make([]map[string]any, 0, len(records))- headerText := spreadsheetRowText(header, nil, sheet)- items = append(items, map[string]any{- "text": headerText,- "doc_type_kwd": DocTypeTable,- "ck_type": "table_header",- "table_id": tableID,- "sheet": sheet,- "sheet_index": sheetIndex,- "cells": header,- "row_start": headerRow,- "row_end": headerRow,- "col_start": 1,- "col_end": headerColEnd,- "positions": [][]float64{{float64(sheetIndex), float64(headerRow), float64(headerRow), 1, float64(headerColEnd)}},- })- for i, sourceRow := range records[1:] {- cells := append([]string(nil), sourceRow...)- colStart, colEnd := nonEmptyColumnRange(cells)- if colStart == 0 {- continue- }- rowNumber := dataRows[i]- items = append(items, map[string]any{- "text": spreadsheetRowText(header, cells, sheet),- "doc_type_kwd": DocTypeText,- "ck_type": "table_row",- "table_id": tableID,- "sheet": sheet,- "sheet_index": sheetIndex,- "headers": header,- "cells": cells,- "row_start": rowNumber,- "row_end": rowNumber,- "col_start": colStart,- "col_end": colEnd,- "positions": [][]float64{{float64(sheetIndex), float64(rowNumber), float64(rowNumber), float64(colStart), float64(colEnd)}},- })- }- return items+// spreadsheetSegmentRow is one emitted data row: its cells for the markup+// and its row-aligned position tuple.+type spreadsheetSegmentRow struct {+ cells []string+ tuple []float64+ rowNum int+ colStart int}-func recordsToHTMLTableItem(records [][]string, sheet string, sheetIndex, headerRow int, dataRows []int) map[string]any {- if len(records) == 0 {- return nil- }- if headerRow <= 0 {- headerRow = 1- }- colEnd := 1- for _, row := range records {- if len(row) > colEnd {- colEnd = len(row)- }- }+// renderTableHTML renders one sheet segment in the spreadsheet wire format: a+// captioned <table> whose header row is <th> cells and whose data rows are+// <td> cells, one row per line. It is the byte contract every consumer reads —+// the chunkers parse this shape back row by row (see+// internal/ingestion/component/chunker/html_rows.go), and the positions matrix+// emitted next to it is aligned one tuple per <tr>, header included.+func renderSpreadsheetTable(sheet string, header []string, rows [][]string) string {var builder strings.Builderbuilder.WriteString("<table><caption>")builder.WriteString(html.EscapeString(sheet))builder.WriteString("</caption>\n<tr>")- for _, cell := range records[0] {+ for _, cell := range header {builder.WriteString("<th>")builder.WriteString(html.EscapeString(strings.TrimSpace(cell)))builder.WriteString("</th>")}builder.WriteString("</tr>\n")- for _, row := range records[1:] {+ for _, row := range rows {builder.WriteString("<tr>")for _, cell := range row {builder.WriteString("<td>")@@ -145,98 +78,172 @@ func recordsToHTMLTableItem(records [][]string, sheet string, sheetIndex, headerbuilder.WriteString("</tr>\n")}builder.WriteString("</table>\n")- rowStart, rowEnd := headerRow, headerRow- if len(dataRows) > 0 {- rowStart, rowEnd = dataRows[0], dataRows[len(dataRows)-1]- }- return NewTableJSONItem(builder.String(), sheet, [][]float64{{- float64(sheetIndex), float64(rowStart), float64(rowEnd), 1, float64(colEnd),- }})+ return builder.String()}-func padSpreadsheetRow(row []string, width int) []string {- padded := make([]string, width)- copy(padded, row)- return padded-}+// buildSheetItems renders one sheet into its wire items: HTML <table>+// segments split at image anchor boundaries, interleaved with the anchored+// image items in document order. Every segment repeats the header row (so+// segment 2+ never loses column names) and carries a row-aligned position+// matrix — one tuple per <tr> in strict markup order, header included — so+// row-level consumers index positions by <tr> number with no side channel.+//+// An image lands after the last row whose (row, colStart) sorts at or before+// its anchor; rows before the header's anchor keep the header in front of+// the image (a header-only lead segment); fully empty rows are skipped from+// both markup and matrix.+//+// Segmentation is semantic only — sheet boundaries and image anchors, never a+// size budget. The legacy html4excel path pre-cut a fixed 12 rows here, which+// ignored the user's chunk size; that budget belongs to the chunker (see the+// chunker's splitSpreadsheetTable).+//+// Every tuple is a spreadsheet position, [sheet, rowStart, rowEnd, colStart,+// colEnd]. The same `positions` field carries PDF layout boxes+// ([page, left, right, top, bottom]) on PDF items, so a consumer must key the+// vocabulary off sheet_index / ck_type and never reinterpret one as the other.+func buildSheetItems(records [][]string, sheet string, sheetIndex, headerRow int, dataRows []int, images []map[string]any) []map[string]any {+ sortImagesByAnchor(images)+ if len(records) == 0 {+ return images+ }+ if headerRow <= 0 {+ headerRow = 1+ }+ header := append([]string(nil), records[0]...)+ headerColEnd := len(header)+ if headerColEnd == 0 {+ for _, row := range records {+ if len(row) > headerColEnd {+ headerColEnd = len(row)+ }+ }+ }+ if headerColEnd == 0 {+ headerColEnd = 1+ }+ headerTuple := []float64{float64(sheetIndex), float64(headerRow), float64(headerRow), 1, float64(headerColEnd)} // [sheet, rowStart, rowEnd, colStart, colEnd]-func nonEmptyColumnRange(row []string) (int, int) {- start, end := 0, 0- for i, cell := range row {- if strings.TrimSpace(cell) == "" {+ rows := make([]spreadsheetSegmentRow, 0, len(records)-1)+ for i, source := range records[1:] {+ cells := append([]string(nil), source...)+ colStart, colEnd := nonEmptyColumnRange(cells)+ if colStart == 0 {continue}- col := i + 1- if start == 0 {- start = col+ rowNum := headerRow + i + 1+ if len(dataRows) == len(records)-1 {+ rowNum = dataRows[i]}- end = col+ rows = append(rows, spreadsheetSegmentRow{+ cells: cells,+ tuple: []float64{float64(sheetIndex), float64(rowNum), float64(rowNum), float64(colStart), float64(colEnd)},+ rowNum: rowNum,+ colStart: colStart,+ })}- return start, end-}-func spreadsheetRowText(headers, cells []string, sheet string) string {- values := headers- if cells != nil {- values = cells- }- parts := make([]string, 0, len(values))- for i, value := range values {- value = strings.TrimSpace(value)- if value == "" {- continue- }- if cells == nil {- parts = append(parts, value)- continue+ anchored := make([]struct {+ cut int+ img map[string]any+ }, 0, len(images))+ for _, img := range images {+ row, _ := numericItemInt(img["row_start"])+ col, _ := numericItemInt(img["col_start"])+ cut := 0+ for cut < len(rows) {+ r := rows[cut]+ if r.rowNum > row || (r.rowNum == row && r.colStart > col) {+ break+ }+ cut++}- header := ""- if i < len(headers) {- header = strings.TrimSpace(headers[i])+ anchored = append(anchored, struct {+ cut int+ img map[string]any+ }{cut, img})+ }+ sort.SliceStable(anchored, func(i, j int) bool {+ if anchored[i].cut != anchored[j].cut {+ return anchored[i].cut < anchored[j].cut}- if header != "" {- parts = append(parts, header+":"+value)- } else {- parts = append(parts, value)+ ri, _ := numericItemInt(anchored[i].img["row_start"])+ rj, _ := numericItemInt(anchored[j].img["row_start"])+ if ri != rj {+ return ri < rj}+ ci, _ := numericItemInt(anchored[i].img["col_start"])+ cj, _ := numericItemInt(anchored[j].img["col_start"])+ return ci < cj+ })++ items := make([]map[string]any, 0, len(rows)+len(images))+ newSegment := func(body []spreadsheetSegmentRow) map[string]any {+ cells := make([][]string, 0, len(body))+ matrix := make([][]float64, 0, len(body)+1)+ matrix = append(matrix, headerTuple)+ for _, r := range body {+ cells = append(cells, r.cells)+ matrix = append(matrix, r.tuple)+ }+ item := NewTableJSONItem(renderSpreadsheetTable(sheet, header, cells), sheet, matrix)+ item["sheet_index"] = sheetIndex+ return item+ }++ prev := 0+ headerLeadUsed := false+ for i := 0; i < len(anchored); {+ cut := anchored[i].cut+ if cut == 0 && !headerLeadUsed {+ // A cut at 0 keeps the header in front of the image, matching the+ // old IR's always-first header record: emit a header-only lead+ // segment, the data rows follow in the next segment.+ items = append(items, newSegment(nil))+ headerLeadUsed = true+ } else if cut > prev {+ items = append(items, newSegment(rows[prev:cut]))+ prev = cut+ }+ for i < len(anchored) && anchored[i].cut == cut {+ items = append(items, anchored[i].img)+ i+++ }+ }+ if prev < len(rows) || !headerLeadUsed && len(rows) == 0 && len(anchored) == 0 {+ // A header-only sheet (no data rows, no images) still emits its+ // header as the only searchable representation of the column schema.+ items = append(items, newSegment(rows[prev:]))}- text := strings.Join(parts, "; ")- if cells != nil && sheet != "" && !strings.Contains(strings.ToLower(sheet), "sheet") {- text += " ——" + sheet- }- return text+ return items}-func sortSpreadsheetItems(items []map[string]any) {- sort.SliceStable(items, func(i, j int) bool {- headerI := items[i]["ck_type"] == "table_header"- headerJ := items[j]["ck_type"] == "table_header"- if headerI != headerJ {- return headerI- }- ri, ci := spreadsheetItemCoordinate(items[i])- rj, cj := spreadsheetItemCoordinate(items[j])+func sortImagesByAnchor(images []map[string]any) {+ sort.SliceStable(images, func(i, j int) bool {+ ri, _ := numericItemInt(images[i]["row_start"])+ rj, _ := numericItemInt(images[j]["row_start"])if ri != rj {return ri < rj}- if ci != cj {- return ci < cj- }- return spreadsheetItemKindRank(items[i]) < spreadsheetItemKindRank(items[j])+ ci, _ := numericItemInt(images[i]["col_start"])+ cj, _ := numericItemInt(images[j]["col_start"])+ return ci < cj})}-func spreadsheetItemCoordinate(item map[string]any) (int, int) {- row, _ := numericItemInt(item["row_start"])- col, _ := numericItemInt(item["col_start"])- return row, col-}--func spreadsheetItemKindRank(item map[string]any) int {- if item["ck_type"] == "image" {- return 1+func nonEmptyColumnRange(row []string) (int, int) {+ start, end := 0, 0+ for i, cell := range row {+ if strings.TrimSpace(cell) == "" {+ continue+ }+ col := i + 1+ if start == 0 {+ start = col+ }+ end = col}- return 0+ return start, end}func numericItemInt(value any) (int, bool) {@@ -256,9 +263,10 @@ func numericItemInt(value any) (int, bool) {}// deprecatedChunkRows reports the removed parser-side chunking option. Row-// boundaries are now emitted as structured IR and the selected chunker owns-// token-budget merging; silently accepting chunk_rows would make an existing-// configuration look effective when it is not.+// segmentation is now owned by the parser's semantic segments and the+// selected chunker owns token-budget splitting; silently accepting+// chunk_rows would make an existing configuration look effective when it is+// not.func deprecatedChunkRows(setup map[string]any, parserName string) {raw, exists := setup["chunk_rows"]if !exists {@@ -274,6 +282,17 @@ func deprecatedChunkRows(setup map[string]any, parserName string) {zap.String("parser", parserName), zap.Int("chunk_rows", rows))}+// deprecatedHTML4Excel reports the retired html4excel option. Both+// spreadsheet builders collapsed into the segmented-HTML wire, so the flag+// selects nothing anymore. Only an enabled flag warns: false is the default+// every parser config carries, and warning there would be noise.+func deprecatedHTML4Excel(setup map[string]any, parserName string) {+ if enabled, ok := setup["html4excel"].(bool); ok && enabled {+ common.Warn("spreadsheet parser ignored deprecated html4excel; spreadsheet tables are always emitted as segmented HTML",+ zap.String("parser", parserName))+ }+}+// extractXLSXImages returns the floating and in-cell images anchored to a// worksheet as structured parser items. Excelize exposes both kinds through// GetPictureCells/GetPictures; walking the reported anchor cells avoidsdiff --git a/internal/parser/parser/office_table_render_test.go b/internal/parser/parser/office_table_render_test.goindex b66a67ec46..ddbfd3c1f1 100644--- a/internal/parser/parser/office_table_render_test.go+++ b/internal/parser/parser/office_table_render_test.go@@ -1,14 +1,16 @@package parserimport (+ "html""reflect"+ "regexp""strings""testing""github.com/xuri/excelize/v2")-func TestXLSXParserEmitsSpreadsheetRows(t *testing.T) {+func TestXLSXParserEmitsSegmentedHTMLTable(t *testing.T) {data := newTestXLSX(t, func(f *excelize.File) {mustSetCell(t, f, "Sheet1", "A1", "ID")mustSetCell(t, f, "Sheet1", "B1", "Status")@@ -22,51 +24,52 @@ func TestXLSXParserEmitsSpreadsheetRows(t *testing.T) {if res.Err != nil {t.Fatalf("ParseWithResult: %v", res.Err)}- if len(res.JSON) != 3 {- t.Fatalf("items = %d, want header plus two rows", len(res.JSON))+ if len(res.JSON) != 1 {+ t.Fatalf("items = %d, want one segmented HTML table item", len(res.JSON))}- header := res.JSON[0]- if header["ck_type"] != "table_header" {- t.Fatalf("header ck_type = %v, want table_header", header["ck_type"])+ item := res.JSON[0]+ if item["ck_type"] != "table" || item["doc_type_kwd"] != "table" {+ t.Fatalf("item types = ck_type:%v doc_type:%v", item["ck_type"], item["doc_type_kwd"])}- if got, want := header["cells"], []string{"ID", "Status"}; !reflect.DeepEqual(got, want) {- t.Fatalf("header cells = %#v, want %#v", got, want)+ text, _ := item["text"].(string)+ wantText := "<table><caption>Sheet1</caption>\n" ++ "<tr><th>ID</th><th>Status</th></tr>\n" ++ "<tr><td>A-100</td><td>paid</td></tr>\n" ++ "<tr><td>A-101</td><td>pending</td></tr>\n" ++ "</table>\n"+ if text != wantText {+ t.Fatalf("text = %q, want %q", text, wantText)}- row := res.JSON[1]- if row["ck_type"] != "table_row" || row["doc_type_kwd"] != "text" {- t.Fatalf("row types = ck_type:%v doc_type:%v", row["ck_type"], row["doc_type_kwd"])+ if item["sheet"] != "Sheet1" || item["sheet_index"] != 1 {+ t.Fatalf("sheet metadata = %#v", item)}- if got, want := row["cells"], []string{"A-100", "paid"}; !reflect.DeepEqual(got, want) {- t.Fatalf("row cells = %#v, want %#v", got, want)+ positions := item["positions"].([][]float64)+ if len(positions) != 3 {+ t.Fatalf("len(positions) = %d, want one tuple per <tr>", len(positions))}- if row["sheet"] != "Sheet1" || row["sheet_index"] != 1 {- t.Fatalf("row sheet metadata = %#v", row)+ if got := positions[1]; !reflect.DeepEqual(got, []float64{1, 2, 2, 1, 2}) {+ t.Fatalf("row 2 tuple = %v, want [1 2 2 1 2]", got)}- if row["row_start"] != 2 || row["row_end"] != 2 || row["col_start"] != 1 || row["col_end"] != 2 {- t.Fatalf("row coordinates = %#v", row)+ if got := positions[2]; !reflect.DeepEqual(got, []float64{1, 3, 3, 1, 2}) {+ t.Fatalf("row 3 tuple = %v, want [1 3 3 1 2]", got)}}-func TestSortSpreadsheetItemsKeepsHeaderBeforeRowsWhenHeaderIsLater(t *testing.T) {- items := []map[string]any{- {"ck_type": "table_row", "row_start": 1, "col_start": 1},- {"ck_type": "table_row", "row_start": 2, "col_start": 1},- {"ck_type": "table_header", "row_start": 5, "col_start": 1},- }-- sortSpreadsheetItems(items)- if items[0]["ck_type"] != "table_header" {- t.Fatalf("sorted items = %#v, want table_header first", items)- }-}--func TestXLSXParserHTML4ExcelRemainsAtomic(t *testing.T) {+// TestXLSXParserHTML4ExcelIsIgnored pins the retirement: html4excel is still+// accepted at the entry (with a deprecation warning) but selects nothing —+// the wire is identical to the default path.+func TestXLSXParserHTML4ExcelIsIgnored(t *testing.T) {data := newTestXLSX(t, func(f *excelize.File) {mustSetCell(t, f, "Sheet1", "A1", "Question")mustSetCell(t, f, "Sheet1", "B1", "Answer")mustSetCell(t, f, "Sheet1", "A2", "Q1")mustSetCell(t, f, "Sheet1", "B2", "A1")})+ plain, _ := NewXLSXParser("")+ plainRes := plain.ParseWithResult(t.Context(), "qa.xlsx", data)+ if plainRes.Err != nil {+ t.Fatalf("ParseWithResult: %v", plainRes.Err)+ }p, _ := NewXLSXParser("")p.ConfigureFromSetup(map[string]any{"html4excel": true})res := p.ParseWithResult(t.Context(), "qa.xlsx", data)@@ -79,6 +82,9 @@ func TestXLSXParserHTML4ExcelRemainsAtomic(t *testing.T) {if res.JSON[0]["ck_type"] != "table" || !strings.Contains(res.JSON[0]["text"].(string), "<table>") {t.Fatalf("html4excel item = %#v", res.JSON[0])}+ if !reflect.DeepEqual(res.JSON, plainRes.JSON) {+ t.Errorf("html4excel changed the wire: %v vs %v", res.JSON, plainRes.JSON)+ }}func TestXLSXParserDoesNotPrechunkRows(t *testing.T) {@@ -94,12 +100,20 @@ func TestXLSXParserDoesNotPrechunkRows(t *testing.T) {if res.Err != nil {t.Fatalf("ParseWithResult: %v", res.Err)}- if len(res.JSON) != dataRows+1 {- t.Fatalf("items = %d, want one header plus %d data rows", len(res.JSON), dataRows)+ if len(res.JSON) != 1 {+ t.Fatalf("items = %d, want one HTML table item (parser must not pre-split rows)", len(res.JSON))+ }+ positions := res.JSON[0]["positions"].([][]float64)+ if len(positions) != dataRows+1 {+ t.Fatalf("len(positions) = %d, want one tuple per <tr> (header plus %d rows)", len(positions), dataRows)+ }+ last := positions[len(positions)-1]+ if last[1] != float64(dataRows+1) || last[2] != float64(dataRows+1) {+ t.Fatalf("last tuple = %v, want source row %d", last, dataRows+1)}- last := res.JSON[len(res.JSON)-1]- if last["ck_type"] != "table_row" || last["row_start"] != dataRows+1 {- t.Fatalf("last row = %#v, want source row %d", last, dataRows+1)+ text, _ := res.JSON[0]["text"].(string)+ if !strings.Contains(text, "<td>258</td>") {+ t.Fatalf("last data row missing from markup:\n%s", text)}}@@ -119,21 +133,174 @@ func TestXLSXParserPreservesSheetOrderAndIdentity(t *testing.T) {if res.Err != nil {t.Fatalf("ParseWithResult: %v", res.Err)}- if len(res.JSON) != 4 {- t.Fatalf("items = %d, want two headers and two rows", len(res.JSON))+ if len(res.JSON) != 2 {+ t.Fatalf("items = %d, want one HTML table per sheet", len(res.JSON))}- seen := map[string]map[string]any{}- for _, item := range res.JSON {- if item["ck_type"] == "table_header" {- seen[item["sheet"].(string)] = item+ first, second := res.JSON[0], res.JSON[1]+ if first["sheet"] != "Sheet1" || first["sheet_index"] != 1 {+ t.Fatalf("first sheet identity = %#v", first)+ }+ if second["sheet"] != "Orders" || second["sheet_index"] != 2 {+ t.Fatalf("second sheet identity = %#v", second)+ }+ // Table grouping collapses into sheet_index; table_id no longer exists+ // on HTML table items (segmentation is done by the parser itself).+ if _, ok := first["table_id"]; ok {+ t.Errorf("table_id must not be emitted on segmented table items")+ }+}++// TestXLSXParserEmitsHeaderOnlySegment: a sheet with no data rows still+// emits its header as the only searchable representation of the column+// schema — one segment whose markup holds the single <tr> of <th> cells.+func TestXLSXParserEmitsHeaderOnlySegment(t *testing.T) {+ data := newTestXLSX(t, func(f *excelize.File) {+ mustSetCell(t, f, "Sheet1", "A1", "Name")+ mustSetCell(t, f, "Sheet1", "B1", "Amount")+ })+ p, _ := NewXLSXParser("")+ res := p.ParseWithResult(t.Context(), "headers.xlsx", data)+ if res.Err != nil {+ t.Fatalf("ParseWithResult: %v", res.Err)+ }+ if len(res.JSON) != 1 {+ t.Fatalf("items = %d, want one header-only segment", len(res.JSON))+ }+ item := res.JSON[0]+ wantText := "<table><caption>Sheet1</caption>\n" ++ "<tr><th>Name</th><th>Amount</th></tr>\n" ++ "</table>\n"+ if item["text"] != wantText {+ t.Fatalf("text = %q, want %q", item["text"], wantText)+ }+ positions, _ := item["positions"].([][]float64)+ if len(positions) != 1 || !reflect.DeepEqual(positions[0], []float64{1, 1, 1, 1, 2}) {+ t.Fatalf("positions = %v, want the header tuple only", positions)+ }+}++// TestBuildSheetItemsEscapesCellText: cell text carrying markup characters is+// escaped in the wire and reads back byte-identical from the markup, so+// user content can never become markup.+func TestBuildSheetItemsEscapesCellText(t *testing.T) {+ const tricky = `<b>bold</b> & "quoted" > 1`+ items := buildSheetItems([][]string{{"Name"}, {tricky}}, "Sheet1", 1, 1, nil, nil)+ if len(items) != 1 {+ t.Fatalf("items = %d, want one segment", len(items))+ }+ text, _ := items[0]["text"].(string)+ if strings.Contains(text, "<b>") || !strings.Contains(text, "<b>") {+ t.Fatalf("cell text was not escaped: %q", text)+ }+ rows := markupRows(t, text)+ if len(rows) != 2 || rows[1][0] != tricky {+ t.Fatalf("round trip = %#v, want %q back", rows, tricky)+ }+}++// TestXLSXParserImageAnchorsSplitSegments pins the segmentation semantics: an+// image anchored between rows ends the current segment at its anchor, every+// segment repeats the header row, and a data row sharing the anchor's row is+// ordered by column — cells at or left of the anchor column stay in front of+// the image, cells to its right follow it.+func TestXLSXParserImageAnchorsSplitSegments(t *testing.T) {+ const pngBase64 = "iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAQAAAC1HAwCAAAAC0lEQVR42mNk+M8AAAMBAQDJ/pLvAAAAAElFTkSuQmCC"++ build := func(t *testing.T, fillRow4 func(*excelize.File)) []map[string]any {+ t.Helper()+ data := newTestXLSX(t, func(f *excelize.File) {+ mustSetCell(t, f, "Sheet1", "A1", "Name")+ mustSetCell(t, f, "Sheet1", "B1", "Value")+ // Every non-anchor row carries two columns so the header+ // detector never sees row 4 as the only wide row.+ for row, value := range map[int]string{2: "row2", 3: "row3", 5: "row5", 6: "row6"} {+ mustSetCell(t, f, "Sheet1", cellAxis(row, 1), value)+ mustSetCell(t, f, "Sheet1", cellAxis(row, 2), "x")+ }+ fillRow4(f)+ if err := f.AddPictureFromBytes("Sheet1", "A4", &excelize.Picture{+ Extension: ".png",+ File: mustDecodeBase64(t, pngBase64),+ Format: &excelize.GraphicOptions{AltText: "fig"},+ }); err != nil {+ t.Fatalf("AddPictureFromBytes: %v", err)+ }+ })+ p, _ := NewXLSXParser("")+ res := p.ParseWithResult(t.Context(), "anchors.xlsx", data)+ if res.Err != nil {+ t.Fatalf("ParseWithResult: %v", res.Err)}+ return res.JSON}- if first := seen["Sheet1"]; first == nil || first["table_id"] != "sheet-1" || first["sheet_index"] != 1 {- t.Fatalf("Sheet1 identity = %#v", first)++ assertSegments := func(t *testing.T, items []map[string]any, wantFirst, wantSecond []string) {+ t.Helper()+ if len(items) != 3 {+ t.Fatalf("items = %d, want table, image, table", len(items))+ }+ if items[1]["doc_type_kwd"] != "image" || items[1]["row_start"] != 4 {+ t.Fatalf("image item = %#v", items[1])+ }+ for segIdx, want := range [][]string{wantFirst, wantSecond} {+ item := items[segIdx*2]+ text, _ := item["text"].(string)+ rows := markupRows(t, text)+ if len(rows) != len(want)+1 {+ t.Fatalf("segment %d rows = %d, want header plus %d", segIdx, len(rows), len(want))+ }+ if rows[0][0] != "Name" {+ t.Errorf("segment %d lost the replicated header: %#v", segIdx, rows[0])+ }+ for i, value := range want {+ found := false+ for _, cell := range rows[i+1] {+ if cell == value {+ found = true+ }+ }+ if !found {+ t.Errorf("segment %d row %d = %#v, want %q in it", segIdx, i, rows[i+1], value)+ }+ }+ positions, _ := item["positions"].([][]float64)+ if len(positions) != len(rows) {+ t.Errorf("segment %d positions = %d, want one tuple per <tr> (%d)", segIdx, len(positions), len(rows))+ }+ }}- if second := seen["Orders"]; second == nil || second["table_id"] != "sheet-2" || second["sheet_index"] != 2 {- t.Fatalf("Orders identity = %#v", second)++ t.Run("row at the anchor cell stays in front", func(t *testing.T) {+ items := build(t, func(f *excelize.File) {+ mustSetCell(t, f, "Sheet1", "A4", "row4")+ mustSetCell(t, f, "Sheet1", "B4", "x")+ })+ assertSegments(t, items, []string{"row2", "row3", "row4"}, []string{"row5", "row6"})+ })++ t.Run("row right of the anchor cell follows the image", func(t *testing.T) {+ items := build(t, func(f *excelize.File) { mustSetCell(t, f, "Sheet1", "B4", "row4") })+ assertSegments(t, items, []string{"row2", "row3"}, []string{"row4", "row5", "row6"})+ })+}++// markupRows extracts the cell texts of every <tr> in rendered spreadsheet+// markup, unescaped. It is a deliberately minimal test-local reader: these+// tests assert the parser's rendered output, so the markup itself is the+// contract under test.+func markupRows(t *testing.T, markup string) [][]string {+ t.Helper()+ rowRe := regexp.MustCompile(`(?s)<tr>(.*?)</tr>`)+ cellRe := regexp.MustCompile(`(?s)<t[hd]>(.*?)</t[hd]>`)+ var rows [][]string+ for _, row := range rowRe.FindAllStringSubmatch(markup, -1) {+ var cells []string+ for _, cell := range cellRe.FindAllStringSubmatch(row[1], -1) {+ cells = append(cells, html.UnescapeString(cell[1]))+ }+ rows = append(rows, cells)}+ return rows}func spreadsheetText(res ParseResult) string {@@ -145,13 +312,33 @@ func spreadsheetText(res ParseResult) string {return out.String()}-func spreadsheetHeaderItem(res ParseResult) map[string]any {+// spreadsheetHeaderText returns the leading <table> item's header labels+// joined by "; " — the rendered <th> cells of its first <tr> — or "" when no+// table item exists.+func spreadsheetHeaderText(res ParseResult) string {for _, item := range res.JSON {- if item["ck_type"] == "table_header" {- return item+ text, _ := item["text"].(string)+ if !strings.Contains(strings.ToLower(text), "<table") {+ continue+ }+ row := text+ if i := strings.Index(row, "<tr>"); i >= 0 {+ row = row[i+len("<tr>"):]+ }+ if i := strings.Index(row, "</tr>"); i >= 0 {+ row = row[:i]+ }+ var labels []string+ for _, cell := range strings.Split(row, "<th>")[1:] {+ if i := strings.Index(cell, "</th>"); i >= 0 {+ labels = append(labels, cell[:i])+ }+ }+ if len(labels) > 0 {+ return strings.Join(labels, "; ")}}- return nil+ return ""}func TestXLSXImageMIMEType(t *testing.T) {@@ -238,8 +425,9 @@ func mustAddTable(t *testing.T, f *excelize.File, sheet string, table *excelize.}}-// TestXLSXParser_HeaderAndCaption asserts the XLSX parser emits a typed header-// row and data row with the header labels preserved separately from values.+// TestXLSXParser_HeaderAndCaption asserts the XLSX parser renders the typed+// header row as <th> cells and data rows as <td> cells inside one captioned+// table, with header labels never duplicated as values.func TestXLSXParser_HeaderAndCaption(t *testing.T) {data := newTestXLSX(t, func(f *excelize.File) {mustSetCell(t, f, "Sheet1", "A1", "Product")@@ -255,14 +443,14 @@ func TestXLSXParser_HeaderAndCaption(t *testing.T) {t.Fatalf("ParseWithResult: %v", res.Err)}text := spreadsheetText(res)- if !strings.Contains(text, "Product; Price") {- t.Fatalf("want header text, got:\n%s", text)+ if !strings.Contains(text, "<tr><th>Product</th><th>Price</th></tr>") {+ t.Fatalf("want header row, got:\n%s", text)}- if !strings.Contains(text, "Product:Widget; Price:9.99") {- t.Fatalf("want row text, got:\n%s", text)+ if !strings.Contains(text, "<tr><td>Widget</td><td>9.99</td></tr>") {+ t.Fatalf("want data row, got:\n%s", text)}- // The header label must not be duplicated as a value in the row text.- if strings.Contains(text, "Product:Product") {+ // The header label must not be duplicated as a value in any row.+ if strings.Contains(text, "<td>Product</td>") {t.Fatalf("header value leaked into row text:\n%s", text)}}@@ -291,7 +479,7 @@ func TestXLSXParser_MergedHeaderInheritance(t *testing.T) {t.Fatalf("ParseWithResult: %v", res.Err)}text := spreadsheetText(res)- if !strings.Contains(text, "Sales Report; Sales Report; Sales Report") {+ if !strings.Contains(text, "<th>Sales Report</th><th>Sales Report</th><th>Sales Report</th>") {t.Fatalf("merged master text not inherited into header cells, got:\n%s", text)}}@@ -313,14 +501,10 @@ func TestDetectHeaderRow_ListObject(t *testing.T) {if res.Err != nil {t.Fatalf("ParseWithResult: %v", res.Err)}- if !strings.Contains(spreadsheetText(res), "Name; Age") {+ if !strings.Contains(spreadsheetText(res), "<th>Name</th><th>Age</th>") {t.Fatalf("want ListObject header row detected, got:\n%s", spreadsheetText(res))}- header := spreadsheetHeaderItem(res)- if header == nil {- t.Fatal("missing table_header item")- }- if got := header["text"]; got != "Name; Age" {+ if got := spreadsheetHeaderText(res); got != "Name; Age" {t.Fatalf("header text = %v, want Name; Age", got)}}@@ -344,10 +528,10 @@ func TestDetectHeaderRow_Lightweight(t *testing.T) {if res.Err != nil {t.Fatalf("ParseWithResult: %v", res.Err)}- if !strings.Contains(spreadsheetText(res), "Item; Count") {+ if !strings.Contains(spreadsheetText(res), "<th>Item</th><th>Count</th>") {t.Fatalf("want row-2 header detected, got:\n%s", spreadsheetText(res))}- if strings.Contains(spreadsheetText(res), "100; 200") {+ if strings.Contains(spreadsheetText(res), "<th>100</th><th>200</th>") {t.Fatalf("numeric row 1 must not be the header:\n%s", spreadsheetText(res))}}@@ -366,7 +550,7 @@ func TestXLSXParser_CommonCaseNoRegression(t *testing.T) {if res.Err != nil {t.Fatalf("ParseWithResult: %v", res.Err)}- if !strings.Contains(spreadsheetText(res), "col_a; col_b") {+ if !strings.Contains(spreadsheetText(res), "<th>col_a</th><th>col_b</th>") {t.Fatalf("common-case header must be preserved:\n%s", spreadsheetText(res))}}@@ -394,10 +578,10 @@ func TestDetectHeaderRow_BoldSubtotalNotHeader(t *testing.T) {if res.Err != nil {t.Fatalf("ParseWithResult: %v", res.Err)}- if !strings.Contains(spreadsheetText(res), "2023; 2024") {+ if !strings.Contains(spreadsheetText(res), "<th>2023</th><th>2024</th>") {t.Fatalf("numeric row 1 must remain the header:\n%s", spreadsheetText(res))}- if strings.Contains(spreadsheetText(res), "Total; Summary") {+ if strings.Contains(spreadsheetText(res), "<th>Total</th><th>Summary</th>") {t.Fatalf("bold subtotal row must not be promoted to header:\n%s", spreadsheetText(res))}}@@ -427,14 +611,10 @@ func TestDetectHeaderRow_StyledHeaderPastFarMerge(t *testing.T) {if res.Err != nil {t.Fatalf("ParseWithResult: %v", res.Err)}- if !strings.Contains(spreadsheetText(res), "Name; Desc") {+ if !strings.Contains(spreadsheetText(res), "<th>Name</th><th>Desc</th>") {t.Fatalf("narrow bold header past far merge must be detected:\n%s", spreadsheetText(res))}- header := spreadsheetHeaderItem(res)- if header == nil {- t.Fatal("missing table_header item")- }- if got := header["text"]; got != "Name; Desc" {+ if got := spreadsheetHeaderText(res); got != "Name; Desc" {t.Fatalf("header text = %v, want Name; Desc", got)}}@@ -461,14 +641,10 @@ func TestDetectHeaderRow_StyledTextHeaderOverNumeric(t *testing.T) {if res.Err != nil {t.Fatalf("ParseWithResult: %v", res.Err)}- if !strings.Contains(spreadsheetText(res), "Product; Units") {+ if !strings.Contains(spreadsheetText(res), "<th>Product</th><th>Units</th>") {t.Fatalf("styled text header over numeric data must be detected:\n%s", spreadsheetText(res))}- header := spreadsheetHeaderItem(res)- if header == nil {- t.Fatal("missing table_header item")- }- if got := header["text"]; got != "Product; Units" {+ if got := spreadsheetHeaderText(res); got != "Product; Units" {t.Fatalf("header text = %v, want Product; Units", got)}}diff --git a/internal/parser/parser/pdf_parser_common.go b/internal/parser/parser/pdf_parser_common.goindex 4cb485a94b..8534d72f7f 100644--- a/internal/parser/parser/pdf_parser_common.go+++ b/internal/parser/parser/pdf_parser_common.go@@ -784,10 +784,10 @@ func normalizePDFPositions(raw any) [][]any {// normalizePDFPageNumber converts a DeepDoc 0-indexed page number to the// 1-indexed form stored in _pdf_positions / positions. It is the SINGLE// 0→1 conversion point: DeepDoc (pdf_oxide/pdfium) emits 0-indexed pages,-// and every downstream consumer (AddPositions for ES storage,+// and every downstream consumer (addPDFPositions for ES storage,// PositionsFromMatrix for the PDFium render path) expects 1-indexed input.// Adding +1 unconditionally — instead of only for v<=0 — keeps all pages-// consistent; the old heuristic left page>=1 unconverted, which AddPositions+// consistent; the old heuristic left page>=1 unconverted, which addPDFPositions// then double-incremented and PositionsFromMatrix mis-decremented.func normalizePDFPageNumber(raw any) (int, bool) {switch v := raw.(type) {diff --git a/internal/parser/parser/pdf_parser_common_test.go b/internal/parser/parser/pdf_parser_common_test.goindex 46358f7285..584171bc68 100644--- a/internal/parser/parser/pdf_parser_common_test.go+++ b/internal/parser/parser/pdf_parser_common_test.go@@ -156,7 +156,7 @@ func TestPDFParseResultToJSON_ClassifiesFigureCaptionWithImage(t *testing.T) {// TestNormalizePDFPageNumber_UnconditionalIncrement pins the contract that// DeepDoc emits 0-indexed page numbers and normalizePDFPageNumber is the// SINGLE conversion point to 1-indexed. It must add +1 unconditionally —-// not just for v<=0 — so that downstream AddPositions (a passthrough) and+// not just for v<=0 — so that downstream addPDFPositions (a passthrough) and// PositionsFromMatrix (which subtracts 1 for the 0-indexed PDFium engine)// each see a consistent 1-indexed value.func TestNormalizePDFPageNumber_UnconditionalIncrement(t *testing.T) {diff --git a/internal/parser/parser/pdf_parser_tcadp.go b/internal/parser/parser/pdf_parser_tcadp.goindex 4aa517bac9..faafe813e8 100644--- a/internal/parser/parser/pdf_parser_tcadp.go+++ b/internal/parser/parser/pdf_parser_tcadp.go@@ -84,7 +84,7 @@ func tcadpAnyToItems(raw any) []map[string]any {emit := func(text, docType, layout string) []map[string]any {m := map[string]any{"text": text, "doc_type_kwd": docType, "layout": layout}if page > 0 {- // 1-indexed 5-tuple. AddPositions is a passthrough so+ // 1-indexed 5-tuple. addPDFPositions is a passthrough so// the final position_int / page_num_int carry the same// 1-indexed page number the caller passes. Mirrors// Python presentation.py:148-149.diff --git a/internal/parser/parser/pdf_parser_tcadp_test.go b/internal/parser/parser/pdf_parser_tcadp_test.goindex bf517546a7..f3775e72b3 100644--- a/internal/parser/parser/pdf_parser_tcadp_test.go+++ b/internal/parser/parser/pdf_parser_tcadp_test.go@@ -92,7 +92,7 @@ func TestPDFParser_ParseWithResult_TCADPRequiresAPIServer(t *testing.T) {// : per-slide page numbers present in the raw TCADP response// must be attached to each chunk item as "positions" (a 1-indexed 5-tuple// [page, 0, 0, 0, 0]). The shared ingestion pipeline-// (processChunkPositions -> AddPositions, a passthrough) derives+// (processChunkPositions -> addPDFPositions, a passthrough) derives// top_int=[0], position_int=[[page,0,0,0,0]] and page_num_int=[page] from// this field. Elements without a page number (e.g. spreadsheet TCADP)// must NOT receive a positions field.diff --git a/internal/parser/parser/xls_parser.go b/internal/parser/parser/xls_parser.goindex e807ea8b63..7dc42a4269 100644--- a/internal/parser/parser/xls_parser.go+++ b/internal/parser/parser/xls_parser.go@@ -22,22 +22,16 @@ import ()type XLSParser struct {- libType stringParseMethod stringOutputFormat string- HTML4Excel boolTCADPAPIServer stringTCADPAPIKey stringTCADPTableResultType stringTCADPMarkdownImageResponseType string}-func NewXLSParser(libType string) (*XLSParser, error) {- if libType == "" {- libType = "excelize"- }+func NewXLSParser(_ string) (*XLSParser, error) {return &XLSParser{- libType: libType,TCADPTableResultType: "1",TCADPMarkdownImageResponseType: "1",}, nil@@ -57,9 +51,7 @@ func (p *XLSParser) ConfigureFromSetup(setup map[string]any) {if v, ok := setup["output_format"].(string); ok && v != "" {p.OutputFormat = v}- if v, ok := setup["html4excel"].(bool); ok {- p.HTML4Excel = v- }+ deprecatedHTML4Excel(setup, p.String())deprecatedChunkRows(setup, p.String())if v, ok := setup["tcadp_apiserver"].(string); ok && v != "" {p.TCADPAPIServer = v@@ -93,7 +85,7 @@ func (p *XLSParser) ParseWithResult(ctx context.Context, filename string, data [}}- items, warnings, sheetsCount, err := parseXLSXBytes(data, p.HTML4Excel)+ items, warnings, sheetsCount, err := parseXLSXBytes(data)if err != nil {return ParseResult{Err: fmt.Errorf("xls parse: %w", err)}}diff --git a/internal/parser/parser/xls_parser_test.go b/internal/parser/parser/xls_parser_test.goindex c9ac44e4db..6f5d8dbf54 100644--- a/internal/parser/parser/xls_parser_test.go+++ b/internal/parser/parser/xls_parser_test.go@@ -3,6 +3,7 @@ package parserimport ("bytes""context"+ "strings""testing""github.com/xuri/excelize/v2"@@ -46,14 +47,16 @@ func TestXLSParser_SpreadsheetJSONOutput(t *testing.T) {t.Fatalf("res.JSON should have structured table items")}item := res.JSON[0]- if item["ck_type"] != "table_header" {- t.Errorf("header ck_type = %v, want table_header", item["ck_type"])+ if item["ck_type"] != "table" {+ t.Errorf("table item ck_type = %v, want table", item["ck_type"])}- if len(res.JSON) < 2 || res.JSON[1]["text"] != "Header1:Val1; Header2:Val2" {- t.Errorf("row item = %#v", res.JSON)+ text, _ := item["text"].(string)+ if !strings.Contains(text, "<tr><th>Header1</th><th>Header2</th></tr>") ||+ !strings.Contains(text, "<tr><td>Val1</td><td>Val2</td></tr>") {+ t.Errorf("table markup = %q, want captioned header and data row", text)}if item["doc_type_kwd"] != "table" {- t.Errorf("header doc_type_kwd = %v, want 'table'", item["doc_type_kwd"])+ t.Errorf("doc_type_kwd = %v, want 'table'", item["doc_type_kwd"])}// Configure output format to "json"@@ -87,11 +90,11 @@ func TestXLSXParser_JSONOutput(t *testing.T) {if res.OutputFormat != "json" {t.Errorf("res.OutputFormat = %q, want 'json'", res.OutputFormat)}- if len(res.JSON) < 2 {- t.Fatalf("res.JSON = %#v, want header and row", res.JSON)+ if len(res.JSON) != 1 {+ t.Fatalf("res.JSON = %#v, want one segmented HTML table item", res.JSON)}- if res.JSON[0]["ck_type"] != "table_header" || res.JSON[1]["ck_type"] != "table_row" {- t.Errorf("items should be header and row, got %#v", res.JSON)+ if res.JSON[0]["ck_type"] != "table" {+ t.Errorf("item ck_type = %v, want table", res.JSON[0]["ck_type"])}// Even if configured to legacy "html", format is unified to "json"diff --git a/internal/parser/parser/xls_tcadp_test.go b/internal/parser/parser/xls_tcadp_test.goindex f72a517dbb..f623c3ce98 100644--- a/internal/parser/parser/xls_tcadp_test.go+++ b/internal/parser/parser/xls_tcadp_test.go@@ -225,11 +225,11 @@ func TestCSVParser_ParseWithResult_DefaultCSVBehavior(t *testing.T) {if got, want := res.OutputFormat, "json"; got != want {t.Fatalf("OutputFormat = %q, want %q", got, want)}- if len(res.JSON) < 2 {- t.Fatalf("JSON items = %d, want at least header and one data row", len(res.JSON))+ if len(res.JSON) != 1 {+ t.Fatalf("JSON items = %d, want one HTML table item", len(res.JSON))}- if res.JSON[0]["ck_type"] != "table_header" || res.JSON[1]["ck_type"] != "table_row" {- t.Fatalf("JSON items = %#v; want header and row", res.JSON)+ if res.JSON[0]["ck_type"] != "table" {+ t.Fatalf("JSON item = %#v; want a table item", res.JSON[0])}}diff --git a/internal/parser/parser/xlsx_parse_method_test.go b/internal/parser/parser/xlsx_parse_method_test.goindex c5e9ae57ba..9374087ac1 100644--- a/internal/parser/parser/xlsx_parse_method_test.go+++ b/internal/parser/parser/xlsx_parse_method_test.go@@ -160,7 +160,8 @@ func mustDecodeBase64(t *testing.T, encoded string) []byte {}// TestCSVParser_DeepDocParseMethod asserts the CSV parser accepts the-// default "deepdoc" parse_method and emits row IR.+// default "deepdoc" parse_method and emits the segmented HTML table wire+// shape.func TestCSVParser_DeepDocParseMethod(t *testing.T) {p := NewCSVParser()p.ConfigureFromSetup(map[string]any{"parse_method": "deepdoc"})@@ -173,11 +174,11 @@ func TestCSVParser_DeepDocParseMethod(t *testing.T) {if got, want := res.OutputFormat, "json"; got != want {t.Fatalf("OutputFormat = %q, want %q", got, want)}- if len(res.JSON) < 2 {- t.Fatalf("JSON items = %d, want at least header and one data row", len(res.JSON))+ if len(res.JSON) != 1 {+ t.Fatalf("JSON items = %d, want one HTML table item", len(res.JSON))}- if res.JSON[0]["ck_type"] != "table_header" || res.JSON[1]["ck_type"] != "table_row" {- t.Fatalf("JSON items = %#v, want header and row", res.JSON)+ if res.JSON[0]["ck_type"] != "table" {+ t.Fatalf("JSON item = %#v, want a table item", res.JSON[0])}}diff --git a/internal/parser/parser/xlsx_parser.go b/internal/parser/parser/xlsx_parser.goindex aff077bd90..d71ca9c170 100644--- a/internal/parser/parser/xlsx_parser.go+++ b/internal/parser/parser/xlsx_parser.go@@ -26,22 +26,16 @@ import ()type XLSXParser struct {- libType stringParseMethod stringOutputFormat string- HTML4Excel boolTCADPAPIServer stringTCADPAPIKey stringTCADPTableResultType stringTCADPMarkdownImageResponseType string}-func NewXLSXParser(libType string) (*XLSXParser, error) {- if libType == "" {- libType = "excelize"- }+func NewXLSXParser(_ string) (*XLSXParser, error) {return &XLSXParser{- libType: libType,TCADPTableResultType: "1",TCADPMarkdownImageResponseType: "1",}, nil@@ -61,9 +55,7 @@ func (p *XLSXParser) ConfigureFromSetup(setup map[string]any) {if v, ok := setup["output_format"].(string); ok && v != "" {p.OutputFormat = v}- if v, ok := setup["html4excel"].(bool); ok {- p.HTML4Excel = v- }+ deprecatedHTML4Excel(setup, p.String())deprecatedChunkRows(setup, p.String())if v, ok := setup["tcadp_apiserver"].(string); ok && v != "" {p.TCADPAPIServer = v@@ -115,7 +107,7 @@ func (p *XLSXParser) ParseWithResult(ctx context.Context, filename string, data// for spreadsheet processing.}- items, warnings, sheets, err := parseXLSXBytes(data, p.HTML4Excel)+ items, warnings, sheets, err := parseXLSXBytes(data)if err == nil {return xlsxParseResult(filename, items, warnings, sheets)}@@ -127,7 +119,7 @@ func (p *XLSXParser) ParseWithResult(ctx context.Context, filename string, dataif !changed {return ParseResult{Err: fmt.Errorf("xlsx parse: %w", err)}}- items, warnings, sheets, retryErr := parseXLSXBytes(normalized, p.HTML4Excel)+ items, warnings, sheets, retryErr := parseXLSXBytes(normalized)if retryErr != nil {return ParseResult{Err: fmt.Errorf("xlsx parse: %w; retry after normalization: %v", err, retryErr)}}@@ -135,7 +127,7 @@ func (p *XLSXParser) ParseWithResult(ctx context.Context, filename string, datareturn xlsxParseResult(filename, items, warnings, sheets)}-func parseXLSXBytes(data []byte, html4excel bool) ([]map[string]any, []string, int, error) {+func parseXLSXBytes(data []byte) ([]map[string]any, []string, int, error) {f, err := excelize.OpenReader(bytes.NewReader(data))if err != nil {return nil, nil, 0, fmt.Errorf("open XLSX: %w", err)@@ -151,25 +143,16 @@ func parseXLSXBytes(data []byte, html4excel bool) ([]map[string]any, []string, ireturn nil, warnings, len(sheets), err}warnings = append(warnings, sheetWarnings...)- var sheetItems []map[string]any- if html4excel {- if table := recordsToHTMLTableItem(records, sheet, sheetIdx+1, headerRow, dataRows); table != nil {- sheetItems = []map[string]any{table}- }- } else {- sheetItems = recordsToSpreadsheetItems(records, sheet, sheetIdx+1, headerRow, dataRows)- }images, imageWarnings := extractXLSXImages(f, sheet)for _, image := range images {row, _ := numericItemInt(image["row_start"])col, _ := numericItemInt(image["col_start"])image["sheet_index"] = sheetIdx + 1- image["table_id"] = fmt.Sprintf("sheet-%d", sheetIdx+1)image["positions"] = [][]float64{{float64(sheetIdx + 1), float64(row), float64(row), float64(col), float64(col)}}}- sheetItems = append(sheetItems, images...)- sortSpreadsheetItems(sheetItems)- items = append(items, sheetItems...)+ // One wire shape for every spreadsheet sheet: segmented HTML tables+ // with row-aligned positions, images interleaved at their anchors.+ items = append(items, buildSheetItems(records, sheet, sheetIdx+1, headerRow, dataRows, images)...)warnings = append(warnings, imageWarnings...)}return items, warnings, len(sheets), nildiff --git a/internal/service/file/file_content.go b/internal/service/file/file_content.goindex 0abb3a6929..551c968316 100644--- a/internal/service/file/file_content.go+++ b/internal/service/file/file_content.go@@ -5,7 +5,6 @@ import ("encoding/base64""encoding/json""fmt"- "html""ragflow/internal/dao""ragflow/internal/entity""ragflow/internal/parser/parser"@@ -234,9 +233,6 @@ func parseResultText(res parser.ParseResult) (string, error) {return res.HTML, nilcase "json":if len(res.JSON) > 0 {- if rendered, ok := renderSpreadsheetJSON(res.JSON); ok {- return rendered, nil- }parts := make([]string, 0, len(res.JSON))for _, item := range res.JSON {if text, ok := item["text"].(string); ok {@@ -265,164 +261,6 @@ func parseResultText(res parser.ParseResult) (string, error) {}}-func renderSpreadsheetJSON(items []map[string]any) (string, bool) {- parts := make([]string, 0, len(items))- rendered := false- for i := 0; i < len(items); {- if !isSpreadsheetRowItem(items[i]) {- if text, ok := items[i]["text"].(string); ok {- parts = append(parts, text)- } else {- raw, err := json.Marshal(items[i])- if err != nil {- return "", false- }- parts = append(parts, string(raw))- }- i++- continue- }-- start := i- i = spreadsheetTableEnd(items, start)- parts = append(parts, renderSpreadsheetTable(items[start:i]))- rendered = true- }- if !rendered {- return "", false- }- return strings.Join(parts, "\n"), true-}--func isSpreadsheetRowItem(item map[string]any) bool {- kind, _ := item["ck_type"].(string)- return kind == "table_header" || kind == "table_row"-}--func spreadsheetTableKey(item map[string]any) string {- if tableID, _ := item["table_id"].(string); strings.TrimSpace(tableID) != "" {- return "table:" + tableID- }- if sheetIndex, ok := item["sheet_index"].(float64); ok {- return fmt.Sprintf("sheet-index:%g", sheetIndex)- }- if sheetIndex, ok := item["sheet_index"].(int); ok {- return fmt.Sprintf("sheet-index:%d", sheetIndex)- }- if sheet, _ := item["sheet"].(string); strings.TrimSpace(sheet) != "" {- return "sheet:" + sheet- }- return ""-}--func spreadsheetTableEnd(items []map[string]any, start int) int {- key := spreadsheetTableKey(items[start])- if key != "" {- i := start- for i < len(items) && isSpreadsheetRowItem(items[i]) && spreadsheetTableKey(items[i]) == key {- i++- }- return i- }-- // Without a stable table identity, do not merge unrelated row sequences.- // A header can still own the following anonymous rows until the next- // anonymous header; anonymous row-only inputs are kept as separate tables- // because their table boundary cannot be inferred safely.- if kind, _ := items[start]["ck_type"].(string); kind == "table_header" {- i := start + 1- for i < len(items) && isSpreadsheetRowItem(items[i]) {- if nextKind, _ := items[i]["ck_type"].(string); nextKind == "table_header" {- break- }- if spreadsheetTableKey(items[i]) != "" {- break- }- i++- }- return i- }- return start + 1-}--func renderSpreadsheetTable(items []map[string]any) string {- if len(items) == 0 {- return ""- }- var header []string- if kind, _ := items[0]["ck_type"].(string); kind == "table_header" {- header = spreadsheetStringSlice(items[0]["cells"])- if len(header) == 0 {- header = spreadsheetStringSlice(items[0]["text"])- }- } else {- // Row IR carries the logical column names separately from its cells.- // Use those names as a preview header without consuming the first row.- header = spreadsheetStringSlice(items[0]["headers"])- }-- sheet, _ := items[0]["sheet"].(string)- var builder strings.Builder- builder.WriteString("<table>")- if strings.TrimSpace(sheet) != "" {- builder.WriteString("<caption>")- builder.WriteString(html.EscapeString(strings.TrimSpace(sheet)))- builder.WriteString("</caption>")- }- if len(header) > 0 {- builder.WriteString("<tr>")- for _, cell := range header {- builder.WriteString("<th>")- builder.WriteString(html.EscapeString(cell))- builder.WriteString("</th>")- }- builder.WriteString("</tr>")- }- for _, item := range items {- kind, _ := item["ck_type"].(string)- if kind == "table_header" {- continue- }- cells := spreadsheetStringSlice(item["cells"])- if len(cells) == 0 {- cells = spreadsheetStringSlice(item["text"])- }- builder.WriteString("<tr>")- for _, cell := range cells {- builder.WriteString("<td>")- builder.WriteString(html.EscapeString(cell))- builder.WriteString("</td>")- }- builder.WriteString("</tr>")- }- builder.WriteString("</table>")- return builder.String()-}--func spreadsheetStringSlice(value any) []string {- switch values := value.(type) {- case []string:- out := make([]string, len(values))- for i, value := range values {- out[i] = strings.TrimSpace(value)- }- return out- case []any:- out := make([]string, 0, len(values))- for _, value := range values {- out = append(out, strings.TrimSpace(fmt.Sprint(value)))- }- return out- case string:- if strings.TrimSpace(values) == "" {- return nil- }- return []string{strings.TrimSpace(values)}- default:- return nil- }-}-// parseFileContent tries to parse a file's contents using the appropriate parser.// Falls back to returning raw text if no parser is available.func parseFileContent(ctx context.Context, filename string, data []byte) string {diff --git a/internal/service/file/file_content_test.go b/internal/service/file/file_content_test.goindex 5a70e6edf3..b0421c0681 100644--- a/internal/service/file/file_content_test.go+++ b/internal/service/file/file_content_test.go@@ -75,88 +75,22 @@ func TestParseResultText_JSONItemsPreservesOrderAndFallsBackToJSON(t *testing.T)}}-func TestParseResultText_SpreadsheetRowsRenderReadableHTMLTable(t *testing.T) {+// TestParseResultText_SpreadsheetSegmentsAreTheirOwnMarkup: the wire emits+// self-describing HTML table segments, so the readable preview is exactly+// the item text — no parser-side table re-rendering.+func TestParseResultText_SpreadsheetSegmentsAreTheirOwnMarkup(t *testing.T) {+ segment := "<table><caption>Data</caption>\n<tr><th>a</th><th>b</th></tr>\n<tr><td>1</td><td>2</td></tr>\n</table>\n"result, err := parseResultText(parser.ParseResult{OutputFormat: "json",JSON: []map[string]any{- {- "text": "a; b",- "doc_type_kwd": "table",- "ck_type": "table_header",- "table_id": "sheet-1",- "sheet": "Data",- "cells": []string{"a", "b"},- },- {- "text": "a:1; b:2 ——Data",- "doc_type_kwd": "text",- "ck_type": "table_row",- "table_id": "sheet-1",- "sheet": "Data",- "headers": []string{"a", "b"},- "cells": []string{"1", "2"},- },+ {"text": segment, "doc_type_kwd": "table", "ck_type": "table", "sheet": "Data", "sheet_index": 1},},})if err != nil {t.Fatalf("parseResultText: %v", err)}- for _, want := range []string{"<table", "<caption>Data</caption>", "<th>a</th>", "<td>1</td>"} {- if !strings.Contains(result, want) {- t.Errorf("parseResultText = %q, want %q", result, want)- }- }-}--func TestParseResultText_SpreadsheetRowsWithoutHeaderDoNotDuplicateFirstRow(t *testing.T) {- result, err := parseResultText(parser.ParseResult{- OutputFormat: "json",- JSON: []map[string]any{- {- "ck_type": "table_row",- "headers": []string{"name", "value"},- "cells": []string{"alpha", "1"},- },- },- })- if err != nil {- t.Fatalf("parseResultText: %v", err)- }- if !strings.Contains(result, "<th>name</th>") {- t.Fatalf("row headers were not rendered: %q", result)- }- if got := strings.Count(result, "<td>alpha</td>"); got != 1 {- t.Fatalf("first row rendered %d times, want once: %q", got, result)- }-}--func TestParseResultText_SpreadsheetRowsWithoutIdentityStaySeparate(t *testing.T) {- result, err := parseResultText(parser.ParseResult{- OutputFormat: "json",- JSON: []map[string]any{- {- "ck_type": "table_header",- "cells": []string{"first"},- },- {- "ck_type": "table_row",- "cells": []string{"one"},- },- {- "ck_type": "table_header",- "cells": []string{"second"},- },- {- "ck_type": "table_row",- "cells": []string{"two"},- },- },- })- if err != nil {- t.Fatalf("parseResultText: %v", err)- }- if got := strings.Count(result, "<table>"); got != 2 {- t.Fatalf("unknown spreadsheet tables were merged: got %d tables in %q", got, result)+ if result != segment {+ t.Fatalf("parseResultText = %q, want the segment markup whole", result)}}