RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.
Go agent-harnessagent-native-databaseagentic-aiagentic-retrievalagentic-searchaiai-agentscontext-enginecontext-engineeringcontext-managementharness-engineeringknowledge-compilationragretrieval-augmented-generationretrieval-harnesssearch-harness
refactor(ingestion): retire ck_type in favor of layout and doc_type (#21259)
jay7772111ca59d5113e
diff --git a/internal/ingestion/component/chunker/a1a2_regression_test.go b/internal/ingestion/component/chunker/a1a2_regression_test.goindex 110f8dfa60..a6f171697e 100644--- a/internal/ingestion/component/chunker/a1a2_regression_test.go+++ b/internal/ingestion/component/chunker/a1a2_regression_test.go@@ -83,7 +83,7 @@ func TestCropImageChunks_UploadFailureRedactsChunkText(t *testing.T) {eng := mockCropEngine{}pos := jsonPositions(t, []float64{1, 10, 100, 10, 100})secret := "SECRET_DOC_TEXT_9f3a"- chunks := []schema.ChunkDoc{{CKType: "image", Text: secret, PDFPositions: pos}}+ chunks := []schema.ChunkDoc{{DocType: "image", Text: secret, PDFPositions: pos}}cropImageChunks(ctx, eng, chunks)for _, e := range observedLogs.All() {diff --git a/internal/ingestion/component/chunker/block_kind.go b/internal/ingestion/component/chunker/block_kind.gonew file mode 100644index 0000000000..1a32d4ebf5--- /dev/null+++ b/internal/ingestion/component/chunker/block_kind.go@@ -0,0 +1,50 @@+// Copyright 2026 The InfiniFlow Authors. All Rights Reserved.+//+// Licensed under the Apache License, Version 2.0 (the "License");+// you may not use this file except in compliance with the License.+// You may obtain a copy of the License at+//+// http://www.apache.org/licenses/LICENSE-2.0+//+// Unless required by applicable law or agreed to in writing, software+// distributed under the License is distributed on an "AS IS" BASIS,+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+// See the License for the specific language governing permissions and+// limitations under the License.++package chunker++import "ragflow/internal/ingestion/component/schema"++// structuralLayouts are the non-media block kinds a parser asserts on the+// layout channel. They gate heading detection and line splitting; they never+// describe content type (doc_type_kwd owns that) and are cleared before the+// chunker output boundary.+//+// "title" is deliberately absent: it is the PDF models' heuristic label and+// goes through the Title chunker's notTitle guard, while "heading" is an+// asserted title that bypasses that guard.+var structuralLayouts = map[string]bool{+ "heading": true,+ "paragraph": true,+ "list": true,+ "code": true,+ "quote": true,+}++func isStructuralLayout(layout string) bool {+ return structuralLayouts[layout]+}++// clearStructuralLayout drops the block-kind marker after a chunker has+// merged or split the block into plain body text. PDF model layouts+// (title/figure/equation/...) pass through unchanged.+func clearStructuralLayout(ck *schema.ChunkDoc) {+ if isStructuralLayout(ck.Layout) {+ ck.Layout = ""+ }+}++func isHeadingLayout(layout string) bool {+ return layout == "heading"+}diff --git a/internal/ingestion/component/chunker/block_kind_contract_cgo_test.go b/internal/ingestion/component/chunker/block_kind_contract_cgo_test.gonew file mode 100644index 0000000000..66e657d230--- /dev/null+++ b/internal/ingestion/component/chunker/block_kind_contract_cgo_test.go@@ -0,0 +1,46 @@+// Copyright 2026 The InfiniFlow Authors. All Rights Reserved.+//+// Licensed under the Apache License, Version 2.0 (the "License");+// you may not use this file except in compliance with the License.+// You may obtain a copy of the License at+//+// http://www.apache.org/licenses/LICENSE-2.0+//+// Unless required by applicable law or agreed to in writing, software+// distributed under the License is distributed on an "AS IS" BASIS,+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+// See the License for the specific language governing permissions and+// limitations under the License.++//go:build cgo++package chunker++import (+ "testing"++ deepdoctype "ragflow/internal/deepdoc/parser/type"+)++// TestCropTitleChunksNeverEmitsChunkStructure pins that the Title-family crop+// path keeps the output shape identical whether or not a PDF engine is+// available: retired ck_type and structural layout never leave the component.+func TestCropTitleChunksNeverEmitsChunkStructure(t *testing.T) {+ for _, engine := range []deepdoctype.PDFEngine{nil, mockCropEngine{}} {+ in := []map[string]any{{+ "text": "body",+ "doc_type_kwd": "text",+ "ck_type": "heading",+ "layout": "heading",+ }}+ got := cropTitleChunks(t.Context(), engine, in)+ for i, ck := range got {+ if kind, ok := ck["ck_type"]; ok {+ t.Errorf("engine=%T chunk[%d] emits retired ck_type %v: %#v", engine, i, kind, ck)+ }+ if layout, ok := ck["layout"]; ok {+ t.Errorf("engine=%T chunk[%d] leaks structural layout %v: %#v", engine, i, layout, ck)+ }+ }+ }+}diff --git a/internal/ingestion/component/chunker/block_kind_contract_test.go b/internal/ingestion/component/chunker/block_kind_contract_test.gonew file mode 100644index 0000000000..86f7283fa6--- /dev/null+++ b/internal/ingestion/component/chunker/block_kind_contract_test.go@@ -0,0 +1,223 @@+// Copyright 2026 The InfiniFlow Authors. All Rights Reserved.+//+// Licensed under the Apache License, Version 2.0 (the "License");+// you may not use this file except in compliance with the License.+// You may obtain a copy of the License at+//+// http://www.apache.org/licenses/LICENSE-2.0+//+// Unless required by applicable law or agreed to in writing, software+// distributed under the License is distributed on an "AS IS" BASIS,+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+// See the License for the specific language governing permissions and+// limitations under the License.++package chunker++import (+ "testing"++ "ragflow/internal/ingestion/component/schema"+)++// TestChunkerOutputDropsRetiredCKType pins the output contract: ck_type is a+// retired parser key and no chunker may emit it, even when the input payload+// still carries a legacy ck_type.+func TestChunkerOutputDropsRetiredCKType(t *testing.T) {+ cases := []struct {+ name string+ run func(t *testing.T) []map[string]any+ }{+ {+ name: "general",+ run: func(t *testing.T) []map[string]any {+ c, err := NewGeneralChunker(map[string]any{"chunk_token_size": 128})+ if err != nil {+ t.Fatalf("NewGeneralChunker: %v", err)+ }+ out, err := c.Invoke(t.Context(), nil, map[string]any{+ "name": "t.md", "file_type": "md", "output_format": "json",+ "json": []map[string]any{+ {"text": "# Heading", "doc_type_kwd": "text", "layout": "heading", "ck_type": "heading"},+ {"text": "body", "doc_type_kwd": "text", "ck_type": "text"},+ },+ })+ if err != nil {+ t.Fatalf("GeneralChunker.Invoke: %v", err)+ }+ return outputChunks(t, out)+ },+ },+ {+ name: "token",+ run: func(t *testing.T) []map[string]any {+ c, err := NewTokenChunker(map[string]any{"chunk_token_size": 128})+ if err != nil {+ t.Fatalf("NewTokenChunker: %v", err)+ }+ out, err := c.Invoke(t.Context(), nil, map[string]any{+ "name": "t.md", "output_format": "json",+ "json": []map[string]any{+ {"text": "# Heading", "doc_type_kwd": "text", "layout": "heading", "ck_type": "heading"},+ {"text": "body", "doc_type_kwd": "text", "ck_type": "text"},+ },+ })+ if err != nil {+ t.Fatalf("TokenChunker.Invoke: %v", err)+ }+ return outputChunks(t, out)+ },+ },+ {+ name: "one",+ run: func(t *testing.T) []map[string]any {+ c, err := NewOneChunker(nil)+ if err != nil {+ t.Fatalf("NewOneChunker: %v", err)+ }+ out, err := c.Invoke(t.Context(), nil, map[string]any{+ "name": "t.txt", "output_format": "text", "text": "hello",+ })+ if err != nil {+ t.Fatalf("OneChunker.Invoke: %v", err)+ }+ return outputChunks(t, out)+ },+ },+ {+ name: "title",+ run: func(t *testing.T) []map[string]any {+ c, err := NewGroupTitleChunker(map[string]any{"levels": [][]string{{`^# `}}})+ if err != nil {+ t.Fatalf("NewGroupTitleChunker: %v", err)+ }+ out, err := c.Invoke(t.Context(), nil, map[string]any{+ "name": "t.md",+ "chunks": []map[string]any{+ {"text": "# Heading", "doc_type_kwd": "text", "layout": "heading", "ck_type": "heading"},+ {"text": "body", "doc_type_kwd": "text", "ck_type": "text"},+ },+ })+ if err != nil {+ t.Fatalf("GroupTitleChunker.Invoke: %v", err)+ }+ return outputChunks(t, out)+ },+ },+ }++ for _, tc := range cases {+ t.Run(tc.name, func(t *testing.T) {+ chunks := tc.run(t)+ if len(chunks) == 0 {+ t.Fatal("no chunks emitted")+ }+ for i, ck := range chunks {+ if kind, ok := ck["ck_type"]; ok {+ t.Errorf("chunk[%d] emits retired ck_type %v: %#v", i, kind, ck)+ }+ if layout, ok := ck["layout"].(string); ok && isStructuralLayout(layout) {+ t.Errorf("chunk[%d] leaks structural layout %q: %#v", i, layout, ck)+ }+ }+ })+ }+}++// TestChunkOutputBoundaryClearsStructuralLayout pins which layout values are+// chunker-internal. Structural block kinds are consumed before the boundary+// and must not leak; PDF model layouts pass through unchanged.+func TestChunkOutputBoundaryClearsStructuralLayout(t *testing.T) {+ out := chunkOutputs([]schema.ChunkDoc{+ {Text: "heading", Layout: "heading"},+ {Text: "paragraph", Layout: "paragraph"},+ {Text: "code", Layout: "code"},+ {Text: "figure", DocType: "image", Layout: "figure"},+ {Text: "title", Layout: "title"},+ {Text: "equation", Layout: "equation"},+ })+ chunks := outputChunks(t, out)+ for i := 0; i < 3; i++ {+ if layout, ok := chunks[i]["layout"]; ok {+ t.Errorf("chunk[%d] leaked structural layout %v: %#v", i, layout, chunks[i])+ }+ }+ for i, want := range []string{"figure", "title", "equation"} {+ if got := chunks[3+i]["layout"]; got != want {+ t.Errorf("chunk[%d] layout = %v, want %q (PDF model layout must pass through)", 3+i, got, want)+ }+ }+}++// TestTitleReadsLayoutNotRetiredCKType pins that a legacy ck_type alone no+// longer promotes a line to a heading; only layout does.+func TestTitleReadsLayoutNotRetiredCKType(t *testing.T) {+ c, err := NewGroupTitleChunker(map[string]any{"levels": [][]string{{`^# `}}})+ if err != nil {+ t.Fatalf("NewGroupTitleChunker: %v", err)+ }+ out, err := c.Invoke(t.Context(), nil, map[string]any{+ "name": "t.docx",+ "chunks": []map[string]any{+ {"text": "Introduction", "doc_type_kwd": "text", "ck_type": "heading"},+ {"text": "Body.", "doc_type_kwd": "text"},+ {"text": "Conclusion", "doc_type_kwd": "text", "ck_type": "heading"},+ {"text": "Final.", "doc_type_kwd": "text"},+ },+ })+ if err != nil {+ t.Fatalf("Invoke: %v", err)+ }+ if chunks := outputChunks(t, out); len(chunks) != 1 {+ t.Fatalf("legacy ck_type must not create headings: got %d chunks, want 1", len(chunks))+ }+}++// TestMarkdownImageHeadingFoldsIntoPreviousChunk pins the base chunk+// boundary for an image-only heading: the image is a media attachment and+// folds into the preceding chunk.+func TestMarkdownImageHeadingFoldsIntoPreviousChunk(t *testing.T) {+ c, err := NewGeneralChunker(map[string]any{"chunk_token_size": 512})+ if err != nil {+ t.Fatalf("NewGeneralChunker: %v", err)+ }+ out, err := c.Invoke(t.Context(), nil, map[string]any{+ "name": "t.md", "file_type": "md", "output_format": "json",+ "json": []map[string]any{+ {"text": "intro", "doc_type_kwd": "text"},+ // An image-only heading: markdown labels it as an image but keeps+ // the heading layout. Base folded it into the preceding chunk.+ {"text": "##", "doc_type_kwd": "image", "image": "data:image/png;base64,AAAA", "layout": "heading"},+ {"text": "after", "doc_type_kwd": "text"},+ },+ })+ if err != nil {+ t.Fatalf("GeneralChunker.Invoke: %v", err)+ }+ if chunks := outputChunks(t, out); len(chunks) != 1 {+ t.Fatalf("image-only heading must fold into the preceding chunk: got %d chunks, want 1", len(chunks))+ }+}++// TestTitleKeepsHTMLParagraphAtomic pins the line-split gate: a paragraph+// layout keeps a multi-line block as one record, so an inner line that looks+// like a heading cannot create a spurious section.+func TestTitleKeepsHTMLParagraphAtomic(t *testing.T) {+ c, err := NewGroupTitleChunker(map[string]any{"levels": [][]string{{`^# `}}})+ if err != nil {+ t.Fatalf("NewGroupTitleChunker: %v", err)+ }+ out, err := c.Invoke(t.Context(), nil, map[string]any{+ "name": "t.html",+ "chunks": []map[string]any{+ {"text": "intro\n# not a heading", "doc_type_kwd": "text", "layout": "paragraph"},+ {"text": "body", "doc_type_kwd": "text"},+ },+ })+ if err != nil {+ t.Fatalf("Invoke: %v", err)+ }+ if chunks := outputChunks(t, out); len(chunks) != 1 {+ t.Fatalf("paragraph must stay atomic: got %d chunks, want 1", len(chunks))+ }+}diff --git a/internal/ingestion/component/chunker/common.go b/internal/ingestion/component/chunker/common.goindex ea99778323..4ef02a6645 100644--- a/internal/ingestion/component/chunker/common.go+++ b/internal/ingestion/component/chunker/common.go@@ -216,15 +216,10 @@ func itemDocType(it schema.ChunkDoc) string {return "text"}-// isMediaChunk reports whether a chunk is an image or table region. It checks-// CKType first (general/token paths set it) and falls back to DocType-// (group/hierarchy forward parser output that carries only doc_type_kwd), so-// it classifies media regardless of which chunker produced the chunk.+// isMediaChunk reports whether a chunk is an image or table region. The+// content type lives on doc_type_kwd; layout never describes media.func isMediaChunk(ck schema.ChunkDoc) bool {- typ := strings.ToLower(strings.TrimSpace(ck.CKType))- if typ == "" {- typ = strings.ToLower(strings.TrimSpace(ck.DocType))- }+ typ := itemDocType(ck)return typ == "image" || typ == "table"}@@ -310,6 +305,11 @@ func chunkOutputs(chunks []schema.ChunkDoc) map[string]any {materialized := make([]schema.ChunkDoc, len(chunks))for i := range chunks {materialized[i] = materializeMediaContext(chunks[i])+ // Structural layout is chunker-internal: it is consumed by merge,+ // split, and crop, and must not cross the output boundary. PDF model+ // layouts (title/figure/equation/...) are not structural and pass+ // through unchanged.+ clearStructuralLayout(&materialized[i])}return map[string]any{"output_format": "chunks",diff --git a/internal/ingestion/component/chunker/coverage_blind_spots_test.go b/internal/ingestion/component/chunker/coverage_blind_spots_test.goindex 312583b740..5baa972172 100644--- a/internal/ingestion/component/chunker/coverage_blind_spots_test.go+++ b/internal/ingestion/component/chunker/coverage_blind_spots_test.go@@ -25,7 +25,7 @@ import ()func TestSplitTextParserSentencesKeepsSentenceUnits(t *testing.T) {- item := schema.ChunkDoc{Text: "First!Second?第三。Fourth;", DocType: "text", CKType: "text"}+ item := schema.ChunkDoc{Text: "First!Second?第三。Fourth;", DocType: "text"}parts := splitTextParserSentences(item)if got, want := len(parts), 4; got != want {t.Fatalf("sentence parts = %d, want %d: %#v", got, want, parts)@@ -136,8 +136,8 @@ func TestGeneralChunkerMarkdownLongHeadingKeepsFollowingTableAtomic(t *testing.T"file_type": "md","output_format": "json","json": []map[string]any{- {"text": longHeading, "doc_type_kwd": "text", "ck_type": "heading"},- {"text": "<table><tr><td>A</td></tr></table>", "doc_type_kwd": "table", "ck_type": "table"},+ {"text": longHeading, "doc_type_kwd": "text", "layout": "heading"},+ {"text": "<table><tr><td>A</td></tr></table>", "doc_type_kwd": "table"},},})if err != nil {@@ -147,10 +147,10 @@ func TestGeneralChunkerMarkdownLongHeadingKeepsFollowingTableAtomic(t *testing.Tif len(chunks) != 2 {t.Fatalf("chunks = %#v, want separate heading and table chunks", chunks)}- if chunks[0]["ck_type"] != "heading" || chunks[0]["text"] != strings.TrimSpace(longHeading) {+ if chunks[0]["doc_type_kwd"] != "text" || chunks[0]["text"] != strings.TrimSpace(longHeading) {t.Errorf("heading chunk = %#v", chunks[0])}- if chunks[1]["ck_type"] != "table" || chunks[1]["text"] != "<table><tr><td>A</td></tr></table>" {+ if chunks[1]["doc_type_kwd"] != "table" || chunks[1]["text"] != "<table><tr><td>A</td></tr></table>" {t.Errorf("table chunk = %#v", chunks[1])}}diff --git a/internal/ingestion/component/chunker/crop_title_test.go b/internal/ingestion/component/chunker/crop_title_test.goindex a64acd4a86..58256d756f 100644--- a/internal/ingestion/component/chunker/crop_title_test.go+++ b/internal/ingestion/component/chunker/crop_title_test.go@@ -24,11 +24,10 @@ import (// TestCropTitleChunks_CropsPositionedChunk pins residual A: chunks// produced by the Title/Group/Hierarchy chunkers must be cropped on demand// (mirroring the TokenChunker JSON path). The chunk shape mirrors the real-// Group/Hierarchy output: it carries doc_type_kwd + positions but NO ck_type-// (buildChunksFromRecordGroups never sets ck_type). cropTitleChunks must-// derive ck_type from doc_type_kwd so needsCrop (pdfcrop_cgo.go:151) fires.+// Group/Hierarchy output: it carries doc_type_kwd + positions and no+// block-kind field. cropImageChunks derives the crop type from doc_type_kwd.// A text chunk with positions gets a rendered preview; a chunk without-// positions is left untouched; and the derived ck_type must not leak into+// positions is left untouched; and the retired ck_type must not appear in// the returned chunk shape.func TestCropTitleChunks_CropsPositionedChunk(t *testing.T) {ctx := context.Background()@@ -50,7 +49,7 @@ func TestCropTitleChunks_CropsPositionedChunk(t *testing.T) {t.Errorf("chunk without positions should not be cropped, got %v", got[1]["image"])}if _, ok := got[0]["ck_type"]; ok {- t.Errorf("derived ck_type must not leak into output chunk: %v", got[0])+ t.Errorf("retired ck_type must not appear in output chunk: %v", got[0])}}diff --git a/internal/ingestion/component/chunker/custom_delim_test.go b/internal/ingestion/component/chunker/custom_delim_test.goindex 382a5e0c94..06ba1f33ca 100644--- a/internal/ingestion/component/chunker/custom_delim_test.go+++ b/internal/ingestion/component/chunker/custom_delim_test.go@@ -14,8 +14,7 @@ import (//// Plain text/markdown/html inputs must not gain doc_type_kwd merely because a// custom delimiter is configured: Python emits only text for those paths.-// Structured JSON inputs keep their source doc_type_kwd, while ck_type remains-// on Go text chunks for downstream crop dispatch.+// Structured JSON inputs keep their source doc_type_kwd.const backtickNewline = "`\n`"@@ -54,16 +53,16 @@ func assertPlainCustomDelimiterMetadata(t *testing.T, chunks []map[string]any) {if _, ok := chunk["doc_type_kwd"]; ok {t.Errorf("chunk[%d] must not synthesize doc_type_kwd: %v", i, chunk)}- if got := chunk["ck_type"]; got != "text" {- t.Errorf("chunk[%d] ck_type: want %q got %v", i, "text", got)+ if _, ok := chunk["ck_type"]; ok {+ t.Errorf("chunk[%d] must not emit the retired ck_type: %v", i, chunk)}}}-// TestCustomDelimTextChildrenKeepCKType covers primary and child splitting in-// combination: child chunks remain unstructured but retain Go's live CKType,-// and every delimiter (primary "::" and children "--") is kept.-func TestCustomDelimTextChildrenKeepCKType(t *testing.T) {+// TestCustomDelimTextChildrenKeepDelimiters covers primary and child splitting+// in combination: child chunks remain unstructured and every delimiter+// (primary "::" and children "--") is kept.+func TestCustomDelimTextChildrenKeepDelimiters(t *testing.T) {params := map[string]any{"chunk_token_size": float64(128),"delimiters": []string{"`::`"},diff --git a/internal/ingestion/component/chunker/general.go b/internal/ingestion/component/chunker/general.goindex 57c03cc6d7..9a63747458 100644--- a/internal/ingestion/component/chunker/general.go+++ b/internal/ingestion/component/chunker/general.go@@ -266,7 +266,6 @@ func (c *GeneralChunkerComponent) chunkPDF(ctx context.Context, db *gorm.DB, upsmedia := cloneChunkDoc(unit)media.Text = normalizeGeneralNewlines(media.Text)media.DocType = itemDocType(media)- media.CKType = media.DocTypechunks = append(chunks, media)}}@@ -491,7 +490,6 @@ func groupDOCXUnitsByRecordBoundary(units []schema.ChunkDoc, recordPrefixes []stflush()media := cloneChunkDoc(unit)media.DocType = itemDocType(media)- media.CKType = media.DocTypegrouped = append(grouped, media)continue}@@ -500,7 +498,7 @@ func groupDOCXUnitsByRecordBoundary(units []schema.ChunkDoc, recordPrefixes []st}text := cloneChunkDoc(unit)text.DocType = "text"- text.CKType = "text"+ clearStructuralLayout(&text)if docxTextStartsRecordBoundary(text.Text, recordPrefixes) {sawBoundary = trueflush()@@ -742,7 +740,7 @@ type markdownSplit struct {}// splitMarkdownUnits expands only text units while retaining the semantic-// parser type on every piece. Markdown heading detection depends on ck_type,+// parser type on every piece. Markdown heading detection depends on layout,// so the generic splitter's type normalization cannot be used here.func splitMarkdownUnits(units []schema.ChunkDoc, pattern *regexp.Regexp, keepDelimiter bool) markdownSplit {result := markdownSplit{@@ -753,12 +751,15 @@ func splitMarkdownUnits(units []schema.ChunkDoc, pattern *regexp.Regexp, keepDelfor _, unit := range units {unit = cloneChunkDoc(unit)unit.Text = normalizeGeneralNewlines(itemTextOrFallback(unit))- if !keepDelimiter || unit.CKType == "heading" {+ if !keepDelimiter || isHeadingLayout(unit.Layout) {unit.Text = strings.TrimSpace(unit.Text)}unit.DocType = itemDocType(unit)+ // Media units are content attachments, not block-kind boundaries.+ // Base normalized their CKType to DocType here, so an image-only+ // heading folded into the preceding chunk; preserve that behavior.if unit.DocType == "table" || unit.DocType == "image" {- unit.CKType = unit.DocType+ clearStructuralLayout(&unit)}shortHeading := isShortMarkdownHeading(unit)if unit.DocType != "text" || pattern == nil || !pattern.MatchString(unit.Text) {@@ -838,7 +839,7 @@ func mergeMarkdownUnits(parts markdownSplit, target int, overlapPct float64) []sjoinSep = "\n"}if itemDocType(unit) == "table" {- if current >= 0 && merged[current].CKType == "heading" && previousShortHeading {+ if current >= 0 && isHeadingLayout(merged[current].Layout) && previousShortHeading {mergedImage, imagesOK := mergeMarkdownImagesChecked(merged[current].Image, unit.Image)if !imagesOK {merged = append(merged, cloneChunkDoc(unit))@@ -859,7 +860,6 @@ func mergeMarkdownUnits(parts markdownSplit, target int, overlapPct float64) []smergeGeneralMetadata(&table, heading)table.Image = mergedImagetable.DocType = "table"- table.CKType = "table"table.ContextAbove = stripTextLinesFromMediaContextAbove(table.ContextAbove, heading.Text)merged[current] = tablecurrent = -1@@ -879,9 +879,6 @@ func mergeMarkdownUnits(parts markdownSplit, target int, overlapPct float64) []s}standalone := cloneChunkDoc(unit)standalone.DocType = "image"- if standalone.CKType == "" {- standalone.CKType = "image"- }standalone.TKNums = intPtr(generalUnitTokens(standalone))merged = append(merged, standalone)continue@@ -889,9 +886,6 @@ func mergeMarkdownUnits(parts markdownSplit, target int, overlapPct float64) []sunit = cloneChunkDoc(unit)unit.DocType = "text"- if unit.CKType == "" {- unit.CKType = "text"- }unit.TKNums = intPtr(generalUnitTokens(unit))if current < 0 {merged = append(merged, unit)@@ -902,13 +896,13 @@ func mergeMarkdownUnits(parts markdownSplit, target int, overlapPct float64) []sprevious := &merged[current]unitTokens := generalUnitTokens(unit)- if startsBlock && unit.CKType == "heading" && previous.CKType != "heading" {+ if startsBlock && isHeadingLayout(unit.Layout) && !isHeadingLayout(previous.Layout) {merged = append(merged, unit)current = len(merged) - 1currentTokens = unitTokenscontinue}- forceMerge := previous.CKType == "heading" && previousShortHeading+ forceMerge := isHeadingLayout(previous.Layout) && previousShortHeadingmergedImage, imagesOK := mergeMarkdownImagesChecked(previous.Image, unit.Image)if !imagesOK {merged = append(merged, unit)@@ -940,8 +934,8 @@ func mergeMarkdownUnits(parts markdownSplit, target int, overlapPct float64) []smergeGeneralChunk(previous, unit, joinSep)previous.Image = mergedImageprevious.DocType = "text"- if startsBlock || unit.CKType != "heading" {- previous.CKType = "text"+ if startsBlock || !isHeadingLayout(unit.Layout) {+ clearStructuralLayout(previous)}currentTokens = projectedprevious.TKNums = intPtr(currentTokens)@@ -950,7 +944,7 @@ func mergeMarkdownUnits(parts markdownSplit, target int, overlapPct float64) []s}func isShortMarkdownHeading(unit schema.ChunkDoc) bool {- if unit.CKType != "heading" {+ if !isHeadingLayout(unit.Layout) {return false}return tokenizeStr(markdownHeadingContent(unit.Text)) < 50@@ -1190,7 +1184,7 @@ func mergeGeneralUnits(units []schema.ChunkDoc, target int, overlapPct float64,count := generalUnitTokens(unit)unit.DocType = "text"- unit.CKType = "text"+ clearStructuralLayout(&unit)unit.TKNums = intPtr(count)if count > target {merged = append(merged, cloneChunkDoc(unit))@@ -1335,7 +1329,7 @@ func splitGeneralUnitsPreservingRecordPrefix(units []schema.ChunkDoc, pattern *runit = cloneChunkDoc(unit)unit.Text = normalizeGeneralNewlines(itemTextOrFallback(unit))unit.DocType = itemDocType(unit)- unit.CKType = unit.DocType+ clearStructuralLayout(&unit)prefix, rest := peelLeadingRecordPrefix(unit.Text, recordPrefixes)if unit.DocType != "text" || pattern == nil || !pattern.MatchString(rest) {unit.Text = prefix + rest@@ -1387,7 +1381,7 @@ func splitGeneralUnits(units []schema.ChunkDoc, pattern *regexp.Regexp, keepDeliunit = cloneChunkDoc(unit)unit.Text = normalizeGeneralNewlines(itemTextOrFallback(unit))unit.DocType = itemDocType(unit)- unit.CKType = unit.DocType+ clearStructuralLayout(&unit)if unit.DocType != "text" || pattern == nil || !pattern.MatchString(unit.Text) {unit.TKNums = intPtr(tokenizeStr(unit.Text))result = append(result, unit)diff --git a/internal/ingestion/component/chunker/general_block_delimiter_test.go b/internal/ingestion/component/chunker/general_block_delimiter_test.goindex db51ad6ad8..bece09f171 100644--- a/internal/ingestion/component/chunker/general_block_delimiter_test.go+++ b/internal/ingestion/component/chunker/general_block_delimiter_test.go@@ -145,15 +145,15 @@ func TestGeneralChunkerPDFAndMarkdownCustomMediaContextDropsMarker(t *testing.T)func TestGeneralChunkerMarkdownPunctuationPreservesMediaBoundaries(t *testing.T) {for _, tc := range []struct {- name, docType, ckType, first, media string+ name, docType, layout, first, media stringcontextSize intwant []string}{{"heading and table", "table", "heading", "标题。还有标题!", "<table>A</table>", 512,[]string{"标题。还有标题!\n<table>A</table>后文。还有一句。", "后文。还有一句。"}},- {"image attachment", "image", "text", "前文。还有一句。", "图注。", 0,+ {"image attachment", "image", "", "前文。还有一句。", "图注。", 0,[]string{"前文。还有一句。\n图注。\n后文。还有一句。"}},- {"standalone image", "image", "text", "前文。还有一句。", "图注。", 512,+ {"standalone image", "image", "", "前文。还有一句。", "图注。", 512,[]string{"前文。还有一句。", "前文。还有一句。图注。后文。还有一句。", "后文。还有一句。"}},} {t.Run(tc.name, func(t *testing.T) {@@ -164,10 +164,14 @@ func TestGeneralChunkerMarkdownPunctuationPreservesMediaBoundaries(t *testing.T)if err != nil {t.Fatal(err)}+ first := map[string]any{"text": tc.first, "doc_type_kwd": "text"}+ if tc.layout != "" {+ first["layout"] = tc.layout+ }out, err := c.Invoke(t.Context(), nil, map[string]any{"name": "document.md", "file_type": "md", "output_format": "json","json": []map[string]any{- {"text": tc.first, "doc_type_kwd": "text", "ck_type": tc.ckType},+ first,{"text": tc.media, "doc_type_kwd": tc.docType, "img_id": "media-id"},{"text": "后文。还有一句。", "doc_type_kwd": "text"},},diff --git a/internal/ingestion/component/chunker/general_context_regression_test.go b/internal/ingestion/component/chunker/general_context_regression_test.goindex e33dc23f0f..f12a4d0ed8 100644--- a/internal/ingestion/component/chunker/general_context_regression_test.go+++ b/internal/ingestion/component/chunker/general_context_regression_test.go@@ -60,7 +60,7 @@ func TestGeneralChunkerMarkdownFoldedHeadingDropsPartialContext(t *testing.T) {out, err := c.Invoke(t.Context(), nil, map[string]any{"name": "test.md", "file_type": "md", "output_format": "json","json": []map[string]any{- {"text": "# 标题。还有标题!", "doc_type_kwd": "text", "ck_type": "heading"},+ {"text": "# 标题。还有标题!", "doc_type_kwd": "text", "layout": "heading"},{"text": "<table>A</table>", "doc_type_kwd": "table"},},})@@ -91,7 +91,7 @@ func TestGeneralChunkerMarkdownLongHeadingUsesSourceClassification(t *testing.T)out, err := c.Invoke(t.Context(), nil, map[string]any{"name": "test.md", "file_type": "md", "output_format": "json","json": []map[string]any{- {"text": heading, "doc_type_kwd": "text", "ck_type": "heading"},+ {"text": heading, "doc_type_kwd": "text", "layout": "heading"},{"text": "body", "doc_type_kwd": "text"},},})diff --git a/internal/ingestion/component/chunker/general_merge_test.go b/internal/ingestion/component/chunker/general_merge_test.goindex 6717a209fb..20e9924574 100644--- a/internal/ingestion/component/chunker/general_merge_test.go+++ b/internal/ingestion/component/chunker/general_merge_test.go@@ -28,9 +28,9 @@ import (func TestMergeGeneralUnitsAllowsOneBoundaryOverflow(t *testing.T) {units := []schema.ChunkDoc{- {Text: "alpha", DocType: "text", CKType: "text", TKNums: intPtr(2)},- {Text: "beta", DocType: "text", CKType: "text", TKNums: intPtr(2)},- {Text: "gamma", DocType: "text", CKType: "text", TKNums: intPtr(2)},+ {Text: "alpha", DocType: "text", TKNums: intPtr(2)},+ {Text: "beta", DocType: "text", TKNums: intPtr(2)},+ {Text: "gamma", DocType: "text", TKNums: intPtr(2)},}got := mergeGeneralUnits(units, 3, 0, "\n")@@ -58,8 +58,8 @@ func TestSameSpreadsheetTableUsesPositionSheet(t *testing.T) {func TestMergeGeneralUnitsMergesWhenCurrentEqualsCap(t *testing.T) {units := []schema.ChunkDoc{- {Text: "at cap", DocType: "text", CKType: "text", TKNums: intPtr(3)},- {Text: "overflow", DocType: "text", CKType: "text", TKNums: intPtr(1)},+ {Text: "at cap", DocType: "text", TKNums: intPtr(3)},+ {Text: "overflow", DocType: "text", TKNums: intPtr(1)},}got := mergeGeneralUnits(units, 3, 0, "\n")@@ -70,9 +70,9 @@ func TestMergeGeneralUnitsMergesWhenCurrentEqualsCap(t *testing.T) {func TestMergeGeneralUnitsKeepsOversizedAtomWhole(t *testing.T) {units := []schema.ChunkDoc{- {Text: "before", DocType: "text", CKType: "text", TKNums: intPtr(2)},- {Text: "oversized atom stays whole", DocType: "text", CKType: "text", TKNums: intPtr(8)},- {Text: "after", DocType: "text", CKType: "text", TKNums: intPtr(1)},+ {Text: "before", DocType: "text", TKNums: intPtr(2)},+ {Text: "oversized atom stays whole", DocType: "text", TKNums: intPtr(8)},+ {Text: "after", DocType: "text", TKNums: intPtr(1)},}got := mergeGeneralUnits(units, 3, 0, "\n")@@ -83,9 +83,9 @@ func TestMergeGeneralUnitsKeepsOversizedAtomWhole(t *testing.T) {func TestMergeGeneralUnitsAppliesUnconditionalCharacterOverlap(t *testing.T) {units := []schema.ChunkDoc{- {Text: "abcdefghij@@1\t0\t1\t2\t3##", DocType: "text", CKType: "text", TKNums: intPtr(2)},- {Text: "klmnopqrst", DocType: "text", CKType: "text", TKNums: intPtr(2)},- {Text: "uvwxyzABCD", DocType: "text", CKType: "text", TKNums: intPtr(2)},+ {Text: "abcdefghij@@1\t0\t1\t2\t3##", DocType: "text", TKNums: intPtr(2)},+ {Text: "klmnopqrst", DocType: "text", TKNums: intPtr(2)},+ {Text: "uvwxyzABCD", DocType: "text", TKNums: intPtr(2)},}got := mergeGeneralUnits(units, 2, 50, "\n")@@ -106,9 +106,9 @@ func TestApplyGeneralOverlapDoesNotAccumulatePositions(t *testing.T) {return json.RawMessage(fmt.Sprintf("[[%d,0,10,0,5]]", page))}chunks := []schema.ChunkDoc{- {Text: "alpha beta", DocType: "text", CKType: "text", PDFPositions: position(1), Positions: position(1)},- {Text: "gamma delta", DocType: "text", CKType: "text", PDFPositions: position(2), Positions: position(2)},- {Text: "epsilon zeta", DocType: "text", CKType: "text", PDFPositions: position(3), Positions: position(3)},+ {Text: "alpha beta", DocType: "text", PDFPositions: position(1), Positions: position(1)},+ {Text: "gamma delta", DocType: "text", PDFPositions: position(2), Positions: position(2)},+ {Text: "epsilon zeta", DocType: "text", PDFPositions: position(3), Positions: position(3)},}got := applyGeneralOverlap(chunks, 50, "\n")@@ -125,10 +125,10 @@ func TestApplyGeneralOverlapDoesNotAccumulatePositions(t *testing.T) {func TestMergeMarkdownUnitsCarriesCharacterOverlapIntoNextBudget(t *testing.T) {units := []schema.ChunkDoc{- {Text: "alpha beta", DocType: "text", CKType: "text"},- {Text: "gamma delta", DocType: "text", CKType: "text"},- {Text: "epsilon zeta", DocType: "text", CKType: "text"},- {Text: "eta theta", DocType: "text", CKType: "text"},+ {Text: "alpha beta", DocType: "text"},+ {Text: "gamma delta", DocType: "text"},+ {Text: "epsilon zeta", DocType: "text"},+ {Text: "eta theta", DocType: "text"},}for i := range units {units[i].TKNums = intPtr(tokenizeStr(units[i].Text))@@ -153,7 +153,6 @@ func TestMergeMarkdownUnitsOverlapKeepsCurrentChunkMetadata(t *testing.T) {{Text: "alpha beta",DocType: "text",- CKType: "text",Image: "previous-image",ImgID: "previous-image-id",PageNumber: &previousPage,@@ -163,7 +162,6 @@ func TestMergeMarkdownUnitsOverlapKeepsCurrentChunkMetadata(t *testing.T) {{Text: "gamma delta",DocType: "text",- CKType: "text",PageNumber: ¤tPage,Positions: json.RawMessage(`[[0,0,10,20,30]]`),Extra: map[string]json.RawMessage{"current": json.RawMessage(`true`)},@@ -203,11 +201,10 @@ func TestMergeMarkdownUnitsOverlapKeepsCurrentChunkMetadata(t *testing.T) {func TestMergeMarkdownUnitsStripsFoldedHeadingFromTableContextAbove(t *testing.T) {units := []schema.ChunkDoc{- {Text: "## Section", DocType: "text", CKType: "heading", TKNums: intPtr(1)},+ {Text: "## Section", DocType: "text", Layout: "heading", TKNums: intPtr(1)},{Text: "<table></table>",DocType: "table",- CKType: "table",ContextAbove: "## Section",TKNums: intPtr(1),},@@ -226,12 +223,11 @@ func TestMergeMarkdownUnitsStripsFoldedHeadingFromTableContextAbove(t *testing.Tfunc TestMergeMarkdownUnitsStripsOnlyFoldedHeadingFromTableContextAbove(t *testing.T) {units := []schema.ChunkDoc{- {Text: "intro", DocType: "text", CKType: "text", TKNums: intPtr(1)},- {Text: "## Section", DocType: "text", CKType: "heading", TKNums: intPtr(1)},+ {Text: "intro", DocType: "text", TKNums: intPtr(1)},+ {Text: "## Section", DocType: "text", Layout: "heading", TKNums: intPtr(1)},{Text: "<table></table>",DocType: "table",- CKType: "table",ContextAbove: "intro\n## Section",TKNums: intPtr(1),},@@ -247,32 +243,31 @@ func TestMergeMarkdownUnitsStripsOnlyFoldedHeadingFromTableContextAbove(t *testifunc TestMergeMarkdownUnitsKeepsImageWithMediaContextStandalone(t *testing.T) {units := []schema.ChunkDoc{- {Text: "before", DocType: "text", CKType: "text", TKNums: intPtr(1)},+ {Text: "before", DocType: "text", TKNums: intPtr(1)},{Text: "figure",DocType: "image",- CKType: "image",ContextAbove: "before",ContextBelow: "after",Image: "data:image/png;base64,AAAA",TKNums: intPtr(1),},- {Text: "after", DocType: "text", CKType: "text", TKNums: intPtr(1)},+ {Text: "after", DocType: "text", TKNums: intPtr(1)},}got := mergeMarkdownUnits(markdownSplit{chunks: units, blockStarts: []bool{true, true, true}, shortHeadings: []bool{false, false, false}}, 10, 0)if len(got) != 3 {t.Fatalf("chunks = %d, want text, image, text", len(got))}- if got[1].CKType != "image" || got[1].ContextAbove != "before" || got[1].ContextBelow != "after" {+ if itemDocType(got[1]) != "image" || got[1].ContextAbove != "before" || got[1].ContextBelow != "after" {t.Fatalf("image chunk = %#v, want preserved media context", got[1])}}func TestMergeMarkdownUnitsStartsNewChunkForIncomingHeading(t *testing.T) {units := []schema.ChunkDoc{- {Text: "Background details", DocType: "text", CKType: "text", TKNums: intPtr(1)},- {Text: "## Architecture", DocType: "text", CKType: "heading", TKNums: intPtr(1)},- {Text: "The architecture is split into parser and chunker.", DocType: "text", CKType: "text", TKNums: intPtr(1)},+ {Text: "Background details", DocType: "text", TKNums: intPtr(1)},+ {Text: "## Architecture", DocType: "text", Layout: "heading", TKNums: intPtr(1)},+ {Text: "The architecture is split into parser and chunker.", DocType: "text", TKNums: intPtr(1)},}got := mergeMarkdownUnits(markdownSplit{chunks: units, blockStarts: []bool{true, true, true}, shortHeadings: []bool{false, true, false}}, 3, 0)@@ -282,14 +277,14 @@ func TestMergeMarkdownUnitsStartsNewChunkForIncomingHeading(t *testing.T) {}) {t.Fatalf("markdown chunks = %q, want body and heading/body chunks", texts)}- if got[1].CKType != "text" {- t.Fatalf("heading/body chunk CKType = %q, want text after heading absorption", got[1].CKType)+ if got[1].Layout != "" {+ t.Fatalf("heading/body chunk Layout = %q, want cleared after heading absorption", got[1].Layout)}}func TestShortMarkdownHeadingBudgetIgnoresATXMarker(t *testing.T) {content := strings.TrimSpace(strings.Repeat("word ", 49))- unit := schema.ChunkDoc{CKType: "heading", Text: "## " + content}+ unit := schema.ChunkDoc{Layout: "heading", Text: "## " + content}if tokenizeStr(content) >= 50 {t.Fatalf("test heading content unexpectedly reached threshold: %d", tokenizeStr(content))}@@ -304,9 +299,9 @@ func TestShortMarkdownHeadingBudgetIgnoresATXMarker(t *testing.T) {func TestMergeGeneralUnitsOverlapDoesNotCopyPreviousMediaMetadata(t *testing.T) {previousPage := 4units := []schema.ChunkDoc{- {Text: "first", DocType: "text", CKType: "text", TKNums: intPtr(2), Image: "previous-image", ImgID: "previous-image-id", PageNumber: &previousPage, Extra: map[string]json.RawMessage{"source": json.RawMessage(`"previous"`)}},- {Text: "second", DocType: "text", CKType: "text", TKNums: intPtr(2)},- {Text: "third", DocType: "text", CKType: "text", TKNums: intPtr(2)},+ {Text: "first", DocType: "text", TKNums: intPtr(2), Image: "previous-image", ImgID: "previous-image-id", PageNumber: &previousPage, Extra: map[string]json.RawMessage{"source": json.RawMessage(`"previous"`)}},+ {Text: "second", DocType: "text", TKNums: intPtr(2)},+ {Text: "third", DocType: "text", TKNums: intPtr(2)},}got := mergeGeneralUnits(units, 4, 50, "\n")@@ -337,16 +332,16 @@ func TestTakeContextSentencesPreservesBoundarySelection(t *testing.T) {func TestMergeGeneralUnitsKeepsMediaAtomicAndBreaksTextRun(t *testing.T) {units := []schema.ChunkDoc{- {Text: "before", DocType: "text", CKType: "text", TKNums: intPtr(1)},- {Text: "A|B", DocType: "table", CKType: "table", TKNums: intPtr(2)},- {Text: "after", DocType: "text", CKType: "text", TKNums: intPtr(1)},+ {Text: "before", DocType: "text", TKNums: intPtr(1)},+ {Text: "A|B", DocType: "table", TKNums: intPtr(2)},+ {Text: "after", DocType: "text", TKNums: intPtr(1)},}got := mergeGeneralUnits(units, 10, 0, "\n")if texts := generalChunkTexts(got); !reflect.DeepEqual(texts, []string{"before", "A|B", "after"}) {t.Fatalf("texts = %q", texts)}- if got[1].DocType != "table" || got[1].CKType != "table" {+ if itemDocType(got[1]) != "table" {t.Errorf("table type changed: %#v", got[1])}}@@ -355,7 +350,6 @@ func TestMergeGeneralUnitsCombinesPositionsAndMetadata(t *testing.T) {first := schema.ChunkDoc{Text: "alpha",DocType: "text",- CKType: "text",TKNums: intPtr(1),Image: "image-a",Positions: json.RawMessage(`[[0,0,10,0,10]]`),@@ -366,7 +360,6 @@ func TestMergeGeneralUnitsCombinesPositionsAndMetadata(t *testing.T) {second := schema.ChunkDoc{Text: "beta",DocType: "text",- CKType: "text",TKNums: intPtr(1),Positions: json.RawMessage(`[[0,0,10,20,30]]`),PDFPositions: json.RawMessage(`[[0,0,10,20,30]]`),diff --git a/internal/ingestion/component/chunker/general_positions_slice_test.go b/internal/ingestion/component/chunker/general_positions_slice_test.goindex e1f1c6b87e..d0c3ca21e9 100644--- a/internal/ingestion/component/chunker/general_positions_slice_test.go+++ b/internal/ingestion/component/chunker/general_positions_slice_test.go@@ -49,7 +49,6 @@ func TestSplitGeneralChildrenSlicesPositionsAbut(t *testing.T) {chunk := schema.ChunkDoc{Text: "aaa;bbb",DocType: "text",- CKType: "text",PDFPositions: json.RawMessage(`[[1,10,20,0,100]]`),Positions: json.RawMessage(`[[1,10,20,0,100]]`),}diff --git a/internal/ingestion/component/chunker/general_test.go b/internal/ingestion/component/chunker/general_test.goindex c337efafb0..093e99f5cf 100644--- a/internal/ingestion/component/chunker/general_test.go+++ b/internal/ingestion/component/chunker/general_test.go@@ -80,7 +80,6 @@ func spreadsheetSegmentItem(sheet string, header []string, rows [][]string, shee// so text-bound assertions (e.g. TokenChunker boundary tests) stay valid."text": tableutil.RenderTableHTML(td),"doc_type_kwd": "table",- "ck_type": "table","sheet_index": sheetIndex,"positions": matrix,}@@ -319,8 +318,8 @@ func TestGeneralChunkerMarkdownShortHeadingForcesNextUnit(t *testing.T) {"file_type": "md","output_format": "json","json": []map[string]any{- {"text": "Title", "doc_type_kwd": "text", "ck_type": "heading"},- {"text": "body", "doc_type_kwd": "text", "ck_type": "text"},+ {"text": "Title", "doc_type_kwd": "text", "layout": "heading"},+ {"text": "body", "doc_type_kwd": "text"},},})if err != nil {@@ -346,8 +345,8 @@ func TestGeneralChunkerMarkdownLongHeadingUsesNormalCap(t *testing.T) {"file_type": "md","output_format": "json","json": []map[string]any{- {"text": longHeading, "doc_type_kwd": "text", "ck_type": "heading"},- {"text": "body", "doc_type_kwd": "text", "ck_type": "text"},+ {"text": longHeading, "doc_type_kwd": "text", "layout": "heading"},+ {"text": "body", "doc_type_kwd": "text"},},})if err != nil {@@ -375,7 +374,7 @@ func TestGeneralChunkerMarkdownImageMergesWithTextAndPreservesPayload(t *testing"file_type": "md","output_format": "json","json": []map[string]any{- {"text": "Title", "doc_type_kwd": "text", "ck_type": "heading"},+ {"text": "Title", "doc_type_kwd": "text", "layout": "heading"},{"text": "figure caption", "doc_type_kwd": "image", "image": "data:image/png;base64,AAAA"},},})@@ -516,9 +515,9 @@ func TestGeneralChunkerMarkdownShortHeadingKeepsFollowingTableAtomic(t *testing."file_type": "md","output_format": "json","json": []map[string]any{- {"text": "Title", "doc_type_kwd": "text", "ck_type": "heading"},- {"text": "<table><tr><td>A</td></tr></table>", "doc_type_kwd": "table", "ck_type": "table"},- {"text": "body", "doc_type_kwd": "text", "ck_type": "text"},+ {"text": "Title", "doc_type_kwd": "text", "layout": "heading"},+ {"text": "<table><tr><td>A</td></tr></table>", "doc_type_kwd": "table"},+ {"text": "body", "doc_type_kwd": "text"},},})if err != nil {@@ -534,9 +533,6 @@ func TestGeneralChunkerMarkdownShortHeadingKeepsFollowingTableAtomic(t *testing.if got := chunks[0]["doc_type_kwd"]; got != "table" {t.Errorf("table doc_type_kwd = %v, want table", got)}- if got := chunks[0]["ck_type"]; got != "table" {- t.Errorf("table ck_type = %v, want table", got)- }}func TestGeneralChunkerDOCXMediaDoesNotBreakTextMerge(t *testing.T) {@@ -639,7 +635,7 @@ func TestGeneralChunkerMarkdownTableDoesNotDuplicateShortHeadingInContext(t *tes"file_type": "md","output_format": "json","json": []map[string]any{- {"text": "## Section", "doc_type_kwd": "text", "ck_type": "heading"},+ {"text": "## Section", "doc_type_kwd": "text", "layout": "heading"},{"text": "<table><tr><td>A</td></tr></table>", "doc_type_kwd": "table"},},})@@ -678,7 +674,7 @@ func TestGeneralChunkerMarkdownTableKeepsEarlierContextWhenHeadingFolds(t *testi"output_format": "json","json": []map[string]any{{"text": "intro", "doc_type_kwd": "text"},- {"text": "## Section", "doc_type_kwd": "text", "ck_type": "heading"},+ {"text": "## Section", "doc_type_kwd": "text", "layout": "heading"},{"text": "<table><tr><td>A</td></tr></table>", "doc_type_kwd": "table"},},})@@ -803,11 +799,11 @@ func TestGeneralChunkerDOCXAttachesMediaContextBeforeTextMerge(t *testing.T) {func TestGeneralMediaContextSeparatesAdjacentSourceUnits(t *testing.T) {units := []schema.ChunkDoc{- {Text: "before one", DocType: "text", CKType: "text", TKNums: intPtr(1)},- {Text: "before two", DocType: "text", CKType: "text", TKNums: intPtr(1)},- {Text: "", DocType: "table", CKType: "table"},- {Text: "after one", DocType: "text", CKType: "text", TKNums: intPtr(1)},- {Text: "after two", DocType: "text", CKType: "text", TKNums: intPtr(1)},+ {Text: "before one", DocType: "text", TKNums: intPtr(1)},+ {Text: "before two", DocType: "text", TKNums: intPtr(1)},+ {Text: "", DocType: "table"},+ {Text: "after one", DocType: "text", TKNums: intPtr(1)},+ {Text: "after two", DocType: "text", TKNums: intPtr(1)},}if got, want := collectGeneralMediaContext(units, 2, 10, true, contextSentencePattern), "before one\nbefore two"; got != want {@@ -855,8 +851,8 @@ func TestGeneralChunkerDOCXCustomDelimiterGroupsMultiParagraphRecords(t *testingfunc TestGroupDOCXUnitsByRecordBoundaryInactiveWithoutBoundary(t *testing.T) {units := []schema.ChunkDoc{- {Text: "plain paragraph one", DocType: "text", CKType: "text"},- {Text: "plain paragraph two", DocType: "text", CKType: "text"},+ {Text: "plain paragraph one", DocType: "text"},+ {Text: "plain paragraph two", DocType: "text"},}got, grouped := groupDOCXUnitsByRecordBoundary(units, []string{"问:"})if grouped {@@ -1013,9 +1009,9 @@ func TestSortPDFUnitsKeepsUnpositionedMediaInInputOrder(t *testing.T) {return json.RawMessage(fmt.Sprintf("[[1,0,10,%g,%g]]", top, top+5))}units := []schema.ChunkDoc{- {Text: "before", DocType: "text", CKType: "text", Positions: position(10)},- {Text: "figure", DocType: "image", CKType: "image"},- {Text: "after", DocType: "text", CKType: "text", Positions: position(30)},+ {Text: "before", DocType: "text", Positions: position(10)},+ {Text: "figure", DocType: "image"},+ {Text: "after", DocType: "text", Positions: position(30)},}got := sortPDFUnits(units)@@ -1044,7 +1040,7 @@ func TestGeneralChunkerSpreadsheetHeaderOnlyPreservesHeaderChunk(t *testing.T) {t.Fatalf("chunks = %#v, want one header-only chunk", chunks)}text, _ := chunks[0]["text"].(string)- if chunks[0]["ck_type"] != "table" || text == "" || !strings.Contains(text, "Name") || !strings.Contains(text, "Amount") {+ if chunks[0]["doc_type_kwd"] != "table" || text == "" || !strings.Contains(text, "Name") || !strings.Contains(text, "Amount") {t.Fatalf("header-only chunk = %#v", chunks[0])}}@@ -1063,7 +1059,7 @@ func TestGeneralChunkerSpreadsheetAttachesImageContext(t *testing.T) {"output_format": "json","json": []map[string]any{spreadsheetSegmentItem("Sheet1", []string{"Name"}, [][]string{{"Revenue"}}, 1, 2),- {"text": "B2", "doc_type_kwd": "image", "ck_type": "image", "image": "figure", "sheet_index": 1},+ {"text": "B2", "doc_type_kwd": "image", "image": "figure", "sheet_index": 1},spreadsheetSegmentItem("Sheet1", []string{"Name"}, [][]string{{"Growth"}}, 1, 3),},})@@ -1074,7 +1070,7 @@ func TestGeneralChunkerSpreadsheetAttachesImageContext(t *testing.T) {if len(chunks) != 3 {t.Fatalf("chunks = %#v, want segment, image, segment", chunks)}- if chunks[1]["ck_type"] != "image" {+ if chunks[1]["doc_type_kwd"] != "image" {t.Fatalf("image chunk = %#v", chunks[1])}assertMaterializedMediaContext(t, chunks[1], "RevenueB2Growth")@@ -1091,7 +1087,7 @@ func TestGeneralChunkerSpreadsheetImageContextStopsAtSheetBoundary(t *testing.T)"output_format": "json","json": []map[string]any{spreadsheetSegmentItem("Sheet1", []string{"Name"}, [][]string{{"Sheet one"}}, 1, 2),- {"text": "B2", "doc_type_kwd": "image", "ck_type": "image", "image": "figure", "sheet_index": 1},+ {"text": "B2", "doc_type_kwd": "image", "image": "figure", "sheet_index": 1},spreadsheetSegmentItem("Sheet2", []string{"Name"}, [][]string{{"Sheet two"}}, 2, 2),},})@@ -1243,9 +1239,8 @@ func TestGeneralChunkerMediaContextReachesChunkIDAndIndexContent(t *testing.T) {}const wantText = "before<table><tr><td>A</td></tr></table>after"- // The table chunk is selected by the STORED type column: ck_type is chunker- // bookkeeping and the index boundary strips it (it has no chunk-store- // column, and a strict engine rejects the whole insert over one).+ // The table chunk is selected by the STORED type column: doc_type_kwd is+ // the content-type contract and the index boundary keeps it.var table map[string]anyfor _, ck := range chunks {if ck["doc_type_kwd"] == "table" {@@ -1347,7 +1342,6 @@ func TestGeneralChunkerNonSpreadsheetTableNotMediaContext(t *testing.T) {table := map[string]any{"text": "<table><caption>Doc</caption>\n<tr><th>Name</th></tr>\n<tr><td>Revenue</td></tr>\n</table>\n","doc_type_kwd": "table",- "ck_type": "table",}out, err := component.Invoke(t.Context(), nil, map[string]any{"name": "document.docx",@@ -1355,7 +1349,7 @@ func TestGeneralChunkerNonSpreadsheetTableNotMediaContext(t *testing.T) {"output_format": "json","json": []map[string]any{table,- {"text": "B2", "doc_type_kwd": "image", "ck_type": "image", "image": "figure"},+ {"text": "B2", "doc_type_kwd": "image", "image": "figure"},table,},})@@ -1366,7 +1360,7 @@ func TestGeneralChunkerNonSpreadsheetTableNotMediaContext(t *testing.T) {if len(chunks) != 3 {t.Fatalf("chunks = %#v, want table, image, table", chunks)}- if chunks[1]["ck_type"] != "image" {+ if chunks[1]["doc_type_kwd"] != "image" {t.Fatalf("image chunk = %#v", chunks[1])}if got, _ := chunks[1]["text"].(string); got != "B2" {diff --git a/internal/ingestion/component/chunker/golden_parity_test.go b/internal/ingestion/component/chunker/golden_parity_test.goindex a88851675e..2b982fdf2b 100644--- a/internal/ingestion/component/chunker/golden_parity_test.go+++ b/internal/ingestion/component/chunker/golden_parity_test.go@@ -246,7 +246,7 @@ func invokeChunker(t *testing.T, tc parityCase) []map[string]any {//// Deriving both sets per case rather than hardcoding an ignore list matters:// the Go-only field set is not constant across components (TokenChunker emits-// ck_type/tk_nums/id, TitleChunker emits only id), so a fixed list would both+// tk_nums/id, TitleChunker emits only id), so a fixed list would both// over- and under-match.func compareChunks(t *testing.T, want, got []map[string]any, allowedExtra map[string]string) {t.Helper()@@ -392,9 +392,9 @@ func extraFieldsToStrip(rules []diffRule) map[string]bool {// used for go_snapshot files. Go's json encoder sorts map keys, so two runs// with equal content produce byte-identical output.//-// The Go-only decorative fields declared in known_diffs.json (id, ck_type,+// The Go-only decorative fields declared in known_diffs.json (id,// tk_nums) are stripped first: `id` is non-deterministic (derived from a-// per-run doc id), and `ck_type`/`tk_nums` are Go-internal metadata already+// per-run doc id), and `tk_nums` are Go-internal metadata already// covered by the extra_fields allow-list. Snapshotting them would make the// ratchet non-reproducible across runs, defeating its purpose.func serializeChunks(caseID string, chunks []map[string]any, strip map[string]bool) string {diff --git a/internal/ingestion/component/chunker/group.go b/internal/ingestion/component/chunker/group.goindex e217ab227e..dec608e3e0 100644--- a/internal/ingestion/component/chunker/group.go+++ b/internal/ingestion/component/chunker/group.go@@ -548,7 +548,6 @@ func recordsFromStructured(items []schema.ChunkDoc) []lineRecord {imgID: imgID,image: image,layout: it.Layout,- ckType: it.CKType,pdfPositions: it.PDFPositions,positions: it.Positions,preserveOrder: it.PreserveOrder,@@ -562,7 +561,7 @@ func recordsFromStructured(items []schema.ChunkDoc) []lineRecord {// so heading regexes can see headings after the first line. Structured// media/table items and positioned text remain atomic, and the image// payload stays out of per-line records so a blob is never duplicated.- plainText := itemDocType(it) == "text" && (it.CKType == "" || it.CKType == "text")+ plainText := itemDocType(it) == "text" && !isStructuralLayout(it.Layout)unpositioned := len(it.PDFPositions) == 0 && len(it.Positions) == 0if plainText && unpositioned && strings.Contains(text, "\n") {for _, line := range strings.Split(normalizeGeneralNewlines(text), "\n") {diff --git a/internal/ingestion/component/chunker/hierarchy_test.go b/internal/ingestion/component/chunker/hierarchy_test.goindex 5ed21ccc7a..c421eb1f82 100644--- a/internal/ingestion/component/chunker/hierarchy_test.go+++ b/internal/ingestion/component/chunker/hierarchy_test.go@@ -444,12 +444,12 @@ func TestHierarchyTitleChunker_ShortSingleCJKLineFilter(t *testing.T) {}}-// TestHierarchyTitleChunker_CKTypeHeadingFallback verifies that-// records with ck_type "heading" (from office_oxide DOCX parsing)+// TestHierarchyTitleChunker_LayoutHeadingFallback verifies that+// records with layout "heading" (from office_oxide DOCX parsing)// are treated as heading nodes in the tree, even when their text// doesn't match any regex pattern. Without the fix, such records// are treated as body text — all content merges into one chunk.-func TestHierarchyTitleChunker_CKTypeHeadingFallback(t *testing.T) {+func TestHierarchyTitleChunker_LayoutHeadingFallback(t *testing.T) {c, err := NewHierarchyTitleChunker(map[string]any{"hierarchy": 2,"levels": [][]string{{`^# `}, {`^## `}},@@ -458,11 +458,11 @@ func TestHierarchyTitleChunker_CKTypeHeadingFallback(t *testing.T) {t.Fatalf("NewHierarchyTitleChunker: %v", err)}items := []map[string]any{- {"text": "Introduction", "doc_type_kwd": "text", "ck_type": "heading"},+ {"text": "Introduction", "doc_type_kwd": "text", "layout": "heading"},{"text": "Intro body.", "doc_type_kwd": "text"},- {"text": "Background", "doc_type_kwd": "text", "ck_type": "heading"},+ {"text": "Background", "doc_type_kwd": "text", "layout": "heading"},{"text": "Background body.", "doc_type_kwd": "text"},- {"text": "Conclusion", "doc_type_kwd": "text", "ck_type": "heading"},+ {"text": "Conclusion", "doc_type_kwd": "text", "layout": "heading"},{"text": "Final words.", "doc_type_kwd": "text"},}out, err := c.Invoke(t.Context(), nil, map[string]any{@@ -479,7 +479,7 @@ func TestHierarchyTitleChunker_CKTypeHeadingFallback(t *testing.T) {// Without the fix: 1 chunk (all text merged as body under root).// With the fix: 3 chunks (one per heading + its body).if len(chunks) != 3 {- t.Fatalf("len(chunks) = %d, want 3 (3 ck_type=headings should produce 3 chunks)", len(chunks))+ t.Fatalf("len(chunks) = %d, want 3 (3 layout=headings should produce 3 chunks)", len(chunks))}for _, ck := range chunks {text, _ := ck["text"].(string)diff --git a/internal/ingestion/component/chunker/live_parity_test.go b/internal/ingestion/component/chunker/live_parity_test.goindex 7785ee4153..6e5858ffcc 100644--- a/internal/ingestion/component/chunker/live_parity_test.go+++ b/internal/ingestion/component/chunker/live_parity_test.go@@ -35,7 +35,7 @@ package chunker// silently-collapsed baseline into a hard failure instead of a false pass.//// Comparison logic is shared with the golden harness: chunkDiffs reports every-// difference, allowedExtraFields tolerates the Go-only id/ck_type/tk_nums+// difference, allowedExtraFields tolerates the Go-only id/tk_nums// fields, and known_diffs.json is consulted so documented divergences are// reported (not failed) rather than surprising the reader.diff --git a/internal/ingestion/component/chunker/one.go b/internal/ingestion/component/chunker/one.goindex 53ae5fd1fe..52a685ae49 100644--- a/internal/ingestion/component/chunker/one.go+++ b/internal/ingestion/component/chunker/one.go@@ -115,7 +115,6 @@ func emitOne(text, docType string) map[string]any {// scope here.Text: removeTag(text),DocType: docType,- CKType: docType,}})}@@ -145,7 +144,6 @@ func emitOneFromItems(items, chunks []schema.ChunkDoc) map[string]any {// Strip parser coordinate tags (see emitOne).Text: removeTag(text),DocType: docType,- CKType: docType,Image: it.Image,Positions: it.Positions,PDFPositions: it.PDFPositions,@@ -170,7 +168,7 @@ func emitOneFromItems(items, chunks []schema.ChunkDoc) map[string]any {if strings.TrimSpace(merged) == "" && img == "" {return emptyOutputs()}- out := schema.ChunkDoc{Text: merged, DocType: "text", CKType: "text"}+ out := schema.ChunkDoc{Text: merged, DocType: "text"}// Multi-item merge produces a single text-only chunk mirroring Python// one.py:166-168. Per-item Positions/PDFPositions are intentionally// not carried — merging coordinates from different source items woulddiff --git a/internal/ingestion/component/chunker/one_tag_test.go b/internal/ingestion/component/chunker/one_tag_test.goindex 8f61bf62d4..123ea8fad9 100644--- a/internal/ingestion/component/chunker/one_tag_test.go+++ b/internal/ingestion/component/chunker/one_tag_test.go@@ -28,8 +28,8 @@ import (// coordinate tags into the single emitted chunk.func TestOneChunkerStripsCoordTagsJSON(t *testing.T) {items := []schema.ChunkDoc{- {Text: "Sentence one@@1\t2\t3\t4##", CKType: "text"},- {Text: "Sentence two@@1\t2\t3\t4##", CKType: "text"},+ {Text: "Sentence one@@1\t2\t3\t4##"},+ {Text: "Sentence two@@1\t2\t3\t4##"},}out := emitOneFromItems(items, nil)chunks, _ := out["chunks"].([]map[string]any)diff --git a/internal/ingestion/component/chunker/pdfcrop_cgo.go b/internal/ingestion/component/chunker/pdfcrop_cgo.goindex 59c2557014..2e0b448177 100644--- a/internal/ingestion/component/chunker/pdfcrop_cgo.go+++ b/internal/ingestion/component/chunker/pdfcrop_cgo.go@@ -278,15 +278,9 @@ func cropImageChunks(ctx context.Context, engine deepdoctype.PDFEngine, chunks [// region (Python restore_pdf_text_previews). A pre-existing Image is never// re-cropped — cropImageChunks honors that separately.//-// CKType is the chunker-layer refinement (heading/text/image/table/-// image/table). General and token chunkers always set it before cropping, so-// they are handled by the CKType branch above. Group and hierarchy chunkers-// forward the parser's output verbatim, which carries only the coarser-// doc_type_kwd (no ck_type). For those, fall back to DocType so image/table/-// text regions are still cropped on demand — otherwise every figure/table/-// text preview in group/hierarchy would be silently skipped. When CKType is-// set (general/token) it takes priority, so headings (CKType "heading") stay-// excluded from preview cropping.+// The content type on doc_type_kwd alone decides the crop kind. Headings are+// text chunks and get the same text preview as any other body text, matching+// the historical behavior.func needsCrop(ck schema.ChunkDoc) bool {switch cropChunkType(ck) {case "image", "table", "text":@@ -296,9 +290,10 @@ func needsCrop(ck schema.ChunkDoc) bool {}}+// cropChunkType returns the content type that selects the crop. Callers must+// pass normalized chunks: General/Token clear structural layout before+// cropping, so passing raw parser items would lose their block-kind+// distinction here.func cropChunkType(ck schema.ChunkDoc) string {- if ck.CKType != "" {- return ck.CKType- }- return ck.DocType+ return itemDocType(ck)}diff --git a/internal/ingestion/component/chunker/pdfcrop_cgo_test.go b/internal/ingestion/component/chunker/pdfcrop_cgo_test.goindex 15447b1512..897e8628ef 100644--- a/internal/ingestion/component/chunker/pdfcrop_cgo_test.go+++ b/internal/ingestion/component/chunker/pdfcrop_cgo_test.go@@ -72,29 +72,28 @@ func TestNeedsCrop(t *testing.T) {ck schema.ChunkDocwant bool}{- // --- CKType set (general / token paths): unchanged behavior ---- {"image with positions", schema.ChunkDoc{CKType: "image", PDFPositions: jsonPositions(t, []float64{1, 10, 100, 10, 100})}, true},- {"table with positions", schema.ChunkDoc{CKType: "table", Positions: jsonPositions(t, []float64{1, 10, 100, 10, 100})}, true},- {"text with positions", schema.ChunkDoc{CKType: "text", PDFPositions: jsonPositions(t, []float64{1, 10, 100, 10, 100})}, true},- {"image without positions", schema.ChunkDoc{CKType: "image"}, false},- {"heading excluded via CKType", schema.ChunkDoc{CKType: "heading", PDFPositions: jsonPositions(t, []float64{1, 10, 100, 10, 100})}, false},- {"unknown type via CKType", schema.ChunkDoc{CKType: "equation", PDFPositions: jsonPositions(t, []float64{1, 10, 100, 10, 100})}, false},-- // --- CKType empty: fall back to DocType (group / hierarchy paths,- // whose parser output carries no ck_type). This is the fix for the- // bug where figure/image/table in group/hierarchy were never cropped.+ // Content type comes from doc_type_kwd.+ {"image with positions", schema.ChunkDoc{DocType: "image", PDFPositions: jsonPositions(t, []float64{1, 10, 100, 10, 100})}, true},+ {"table with positions", schema.ChunkDoc{DocType: "table", Positions: jsonPositions(t, []float64{1, 10, 100, 10, 100})}, true},+ {"text with positions", schema.ChunkDoc{DocType: "text", PDFPositions: jsonPositions(t, []float64{1, 10, 100, 10, 100})}, true},+ {"image without positions", schema.ChunkDoc{DocType: "image"}, false},+ {"image with heading layout still crops", schema.ChunkDoc{DocType: "image", Layout: "heading", PDFPositions: jsonPositions(t, []float64{1, 10, 100, 10, 100})}, true},+ {"heading text previews like text", schema.ChunkDoc{Layout: "heading", DocType: "text", PDFPositions: jsonPositions(t, []float64{1, 10, 100, 10, 100})}, true},+ // A non-media model layout (equation) is not structural, so it falls+ // back to its content type and is previewed as text.+ {"equation layout falls back to text", schema.ChunkDoc{Layout: "equation", DocType: "text", PDFPositions: jsonPositions(t, []float64{1, 10, 100, 10, 100})}, true},++ // Group/Hierarchy forward parser output whose content type is already+ // on doc_type_kwd; no block-kind derivation is involved.{"doc image fallback", schema.ChunkDoc{DocType: "image", PDFPositions: jsonPositions(t, []float64{1, 10, 100, 10, 100})}, true},{"doc table fallback", schema.ChunkDoc{DocType: "table", Positions: jsonPositions(t, []float64{1, 10, 100, 10, 100})}, true},{"doc text fallback", schema.ChunkDoc{DocType: "text", PDFPositions: jsonPositions(t, []float64{1, 10, 100, 10, 100})}, true},- // Heading-tagged blocks carry doc_type_kwd "text" but- // must NOT be previewed when CKType is empty — only a plain text body- // chunk legitimately has CKType empty AND should be previewed. Since- // group/hierarchy cannot distinguish heading from body at this layer,- // aligning with Python (which previews every text region) we still crop- // doc_type_kwd=="text"; the heading-collision guard lives in the- // general/token paths where CKType is set.+ // Headings are text chunks and are previewed too, matching Python's+ // restore_pdf_text_previews.{"doc image without positions", schema.ChunkDoc{DocType: "image"}, false},- {"doc unknown type", schema.ChunkDoc{DocType: "equation", PDFPositions: jsonPositions(t, []float64{1, 10, 100, 10, 100})}, false},+ // Unknown doc_type values normalize to text (itemDocType), matching+ // every other content-type dispatch.+ {"unknown doc type normalizes to text", schema.ChunkDoc{DocType: "equation", PDFPositions: jsonPositions(t, []float64{1, 10, 100, 10, 100})}, true},}for _, tc := range cases {if got := needsCrop(tc.ck); got != tc.want {@@ -110,17 +109,17 @@ func TestCropImageChunks_CropsImageTableAndText(t *testing.T) {pos := jsonPositions(t, []float64{1, 10, 100, 10, 100})chunks := []schema.ChunkDoc{- {CKType: "image", Text: "img", PDFPositions: pos},- {CKType: "table", Text: "tbl", PDFPositions: pos},- {CKType: "text", PDFPositions: pos}, // restored preview (Chunker-1.3)- {CKType: "image", Image: "data:image/png;base64,preexisting"}, // preserved+ {DocType: "image", Text: "img", PDFPositions: pos},+ {DocType: "table", Text: "tbl", PDFPositions: pos},+ {PDFPositions: pos}, // restored preview (Chunker-1.3)+ {DocType: "image", Image: "data:image/png;base64,preexisting"}, // preserved}out := cropImageChunks(ctx, eng, chunks)if len(out) != len(chunks) {t.Fatalf("len(out) = %d, want %d", len(out), len(chunks))}for i, ck := range out {- switch ck.CKType {+ switch itemDocType(ck) {case "text":// Chunker-1.3: text chunks with PDF positions get a rendered// preview, mirroring Python restore_pdf_text_previews.@@ -148,10 +147,9 @@ func TestCropImageChunks_CropsImageTableAndText(t *testing.T) {// img_id upload is owned by imageUploadDecorator (image_upload.go) and is// not asserted here.// TestCropImageChunks_DocTypeFallbackCropsImageTableText covers the-// group/hierarchy regression fix: those chunkers forward parser output that-// carries doc_type_kwd but no ck_type. needsCrop must fall back to DocType so-// image/table/text regions are still cropped on demand (the bug was that they-// were silently skipped because ck_type was empty).+// group/hierarchy path: those chunkers forward parser output whose content+// type is already on doc_type_kwd. needsCrop reads DocType so image/table/text+// regions are cropped on demand.func TestCropImageChunks_DocTypeFallbackCropsImageTableText(t *testing.T) {ctx := context.Background()eng := assertZeroPageEngine{}@@ -160,11 +158,11 @@ func TestCropImageChunks_DocTypeFallbackCropsImageTableText(t *testing.T) {chunks := []schema.ChunkDoc{{DocType: "image", Text: "img", PDFPositions: pos},{DocType: "table", Text: "tbl", PDFPositions: pos},- // A plain text body chunk from group/hierarchy has CKType empty and- // doc_type_kwd "text"; the DocType fallback must crop its preview,- // mirroring Python restore_pdf_text_previews. (Real group/hierarchy- // headings are also doc_type_kwd "text" with empty CKType and are- // intentionally previewed too — full alignment with Python.)+ // A plain text body chunk from group/hierarchy has no block-kind+ // marker and doc_type_kwd "text"; DocType drives its preview,+ // mirroring Python restore_pdf_text_previews. Real group/hierarchy+ // headings are also doc_type_kwd "text" and are intentionally+ // previewed too — full alignment with Python.{DocType: "text", PDFPositions: pos},{DocType: "image", Image: "data:image/png;base64,preexisting"}, // preserved}@@ -193,31 +191,12 @@ func TestCropImageChunks_DocTypeFallbackCropsImageTableText(t *testing.T) {}}-// TestCropImageChunks_HeadingExcludedByCKType verifies that the general/token-// path still excludes headings: a chunk with CKType "heading" is never-// previewed even when it carries positions — the CKType branch takes priority-// over the DocType fallback.-func TestCropImageChunks_HeadingExcludedByCKType(t *testing.T) {- ctx := context.Background()- pos := jsonPositions(t, []float64{1, 10, 100, 10, 100})- chunks := []schema.ChunkDoc{- {CKType: "heading", DocType: "text", PDFPositions: pos},- }- out := cropImageChunks(ctx, mockCropEngine{}, chunks)- if len(out) != 1 {- t.Fatalf("len(out) = %d, want 1", len(out))- }- if out[0].Image != "" {- t.Errorf("heading chunk: image = %q, want no preview (CKType takes priority)", out[0].Image)- }-}-func TestRestorePDFTextPreview(t *testing.T) {ctx := context.Background()pos := jsonPositions(t, []float64{1, 10, 100, 10, 100})- withPos := schema.ChunkDoc{CKType: "text", PDFPositions: pos}- withoutPos := schema.ChunkDoc{CKType: "text", Text: "plain text, no coordinates"}+ withPos := schema.ChunkDoc{PDFPositions: pos}+ withoutPos := schema.ChunkDoc{Text: "plain text, no coordinates"}out := cropImageChunks(ctx, mockCropEngine{}, []schema.ChunkDoc{withPos, withoutPos})if len(out) != 2 {@@ -235,8 +214,8 @@ func TestCropImageChunks_NilEnginePassesThrough(t *testing.T) {ctx := context.Background()pos := jsonPositions(t, []float64{1, 10, 100, 10, 100})chunks := []schema.ChunkDoc{- {CKType: "image", PDFPositions: pos},- {CKType: "table", PDFPositions: pos},+ {DocType: "image", PDFPositions: pos},+ {DocType: "table", PDFPositions: pos},}out := cropImageChunks(ctx, nil, chunks)if len(out) != 2 {@@ -254,7 +233,7 @@ func TestCropImageChunks_RenderFailureSkipsChunk(t *testing.T) {// mockCropEngine renders a real image, so a non-empty crop is expected.pos := jsonPositions(t, []float64{1, 10, 100, 10, 100})chunks := []schema.ChunkDoc{- {CKType: "image", Text: "cap", PDFPositions: pos},+ {DocType: "image", Text: "cap", PDFPositions: pos},}out := cropImageChunks(ctx, mockCropEngine{}, chunks)if len(out) != 1 {@@ -316,9 +295,9 @@ func TestCropImageChunks_StreamingUpload(t *testing.T) {eng := mockCropEngine{}pos := jsonPositions(t, []float64{1, 10, 100, 10, 100})chunks := []schema.ChunkDoc{- {CKType: "image", Text: "img-chunk", PDFPositions: pos},- {CKType: "table", Text: "tbl-chunk", PDFPositions: pos},- {CKType: "text", Text: "txt-chunk", PDFPositions: pos},+ {DocType: "image", Text: "img-chunk", PDFPositions: pos},+ {DocType: "table", Text: "tbl-chunk", PDFPositions: pos},+ {Text: "txt-chunk", PDFPositions: pos},}out := cropImageChunks(ctx, eng, chunks)if len(out) != len(chunks) {@@ -327,10 +306,10 @@ func TestCropImageChunks_StreamingUpload(t *testing.T) {for i, ck := range out {if ck.ImgID == "" {- t.Errorf("chunk %d (%s): ImgID empty, want uploaded id", i, ck.CKType)+ t.Errorf("chunk %d (%s): ImgID empty, want uploaded id", i, itemDocType(ck))}if ck.Image != "" {- t.Errorf("chunk %d (%s): Image = %q, want cleared after upload", i, ck.CKType, ck.Image)+ t.Errorf("chunk %d (%s): Image = %q, want cleared after upload", i, itemDocType(ck), ck.Image)}}if len(rec.calls) != len(chunks) {@@ -376,7 +355,7 @@ func TestCropImageChunks_NoUploadWhenKBAbsent(t *testing.T) {eng := mockCropEngine{}pos := jsonPositions(t, []float64{1, 10, 100, 10, 100})- chunks := []schema.ChunkDoc{{CKType: "image", Text: "img", PDFPositions: pos}}+ chunks := []schema.ChunkDoc{{DocType: "image", Text: "img", PDFPositions: pos}}out := cropImageChunks(ctx, eng, chunks)if len(rec.calls) != 0 {@@ -404,7 +383,7 @@ func TestCropImageChunks_UploadFailureFallsThroughToBatchPass(t *testing.T) {eng := mockCropEngine{}pos := jsonPositions(t, []float64{1, 10, 100, 10, 100})- chunks := []schema.ChunkDoc{{CKType: "image", Text: "img-chunk", PDFPositions: pos}}+ chunks := []schema.ChunkDoc{{DocType: "image", Text: "img-chunk", PDFPositions: pos}}out := cropImageChunks(ctx, eng, chunks)if out[0].ImgID != "" {@@ -436,7 +415,7 @@ func TestCropImageChunks_StreamingUploadUsesFinalizedText(t *testing.T) {// ContextAbove is folded into the body by materializeMediaContext, so the// finalized text differs from the raw Text.chunks := []schema.ChunkDoc{- {CKType: "image", Text: "body", ContextAbove: "ABOVE ", PDFPositions: pos},+ {DocType: "image", Text: "body", ContextAbove: "ABOVE ", PDFPositions: pos},}out := cropImageChunks(ctx, eng, chunks)@@ -486,11 +465,11 @@ func TestCropImageChunks_SkipsEmptyTextMedia(t *testing.T) {eng := mockCropEngine{}pos := jsonPositions(t, []float64{1, 10, 100, 10, 100})emptyMedia := []schema.ChunkDoc{- {CKType: "image", PDFPositions: pos}, // no text- {CKType: "table", PDFPositions: pos}, // no text- {CKType: "image", Text: "@@1\t2##", PDFPositions: pos}, // position tag only, no body- {CKType: "table", ContextAbove: "", ContextBelow: "", PDFPositions: pos}, // explicit empty- {DocType: "image", PDFPositions: pos}, // doc-type fallback, no text+ {DocType: "image", PDFPositions: pos}, // no text+ {DocType: "table", PDFPositions: pos}, // no text+ {DocType: "image", Text: "@@1\t2##", PDFPositions: pos}, // position tag only, no body+ {DocType: "table", ContextAbove: "", ContextBelow: "", PDFPositions: pos}, // explicit empty+ {DocType: "image", PDFPositions: pos}, // doc-type fallback, no text}out := cropImageChunks(ctx, eng, emptyMedia)if len(out) != len(emptyMedia) {@@ -510,7 +489,7 @@ func TestCropImageChunks_SkipsEmptyTextMedia(t *testing.T) {// Sanity: a captioned media chunk is still cropped and uploaded.captioned := []schema.ChunkDoc{- {CKType: "image", Text: "a real figure caption", PDFPositions: pos},+ {DocType: "image", Text: "a real figure caption", PDFPositions: pos},}out2 := cropImageChunks(ctx, eng, captioned)if len(out2) != 1 {@@ -555,7 +534,7 @@ func TestCropImageChunks_StreamingUploadUsesCanonicalID(t *testing.T) {// Text carries a parser position tag (@@x\t y##) AND a caption, no media// context — materializeMediaContext short-circuits, so the tag-stripped// canonical text is "fig caption".- in := schema.ChunkDoc{CKType: "image", Text: "fig caption@@1\t2##", PDFPositions: pos}+ in := schema.ChunkDoc{DocType: "image", Text: "fig caption@@1\t2##", PDFPositions: pos}out := cropImageChunks(ctx, eng, []schema.ChunkDoc{in})if len(out) != 1 {diff --git a/internal/ingestion/component/chunker/pdfcrop_concurrency_test.go b/internal/ingestion/component/chunker/pdfcrop_concurrency_test.goindex b98a735ad5..f29afda359 100644--- a/internal/ingestion/component/chunker/pdfcrop_concurrency_test.go+++ b/internal/ingestion/component/chunker/pdfcrop_concurrency_test.go@@ -22,7 +22,7 @@ func TestCropImageChunks_ConcurrentCropsAll(t *testing.T) {const n = 64chunks := make([]schema.ChunkDoc, n)for i := 0; i < n; i++ {- chunks[i] = schema.ChunkDoc{CKType: "image", Text: "cap", PDFPositions: pos}+ chunks[i] = schema.ChunkDoc{DocType: "image", Text: "cap", PDFPositions: pos}}out := cropImageChunks(ctx, eng, chunks)if len(out) != n {@@ -48,7 +48,7 @@ func TestCropImageChunks_ConcurrentPageCacheReuse(t *testing.T) {const n = 200chunks := make([]schema.ChunkDoc, n)for i := 0; i < n; i++ {- chunks[i] = schema.ChunkDoc{CKType: "table", Text: "cap", Positions: pos}+ chunks[i] = schema.ChunkDoc{DocType: "table", Text: "cap", Positions: pos}}out := cropImageChunks(ctx, eng, chunks)for i, ck := range out {diff --git a/internal/ingestion/component/chunker/pdfcrop_progress_cgo_test.go b/internal/ingestion/component/chunker/pdfcrop_progress_cgo_test.goindex 93d2bd2b58..e72d4529c1 100644--- a/internal/ingestion/component/chunker/pdfcrop_progress_cgo_test.go+++ b/internal/ingestion/component/chunker/pdfcrop_progress_cgo_test.go@@ -44,17 +44,14 @@ func TestCropImageChunks_ReportsFractions(t *testing.T) {{Text: "content 1",Positions: jsonPositions(t, []float64{1, 0, 100, 0, 100}),- CKType: "text",},{Text: "content 2",Positions: jsonPositions(t, []float64{1, 100, 200, 100, 200}),- CKType: "text",},{Text: "content 3",Positions: jsonPositions(t, []float64{1, 200, 300, 200, 300}),- CKType: "text",},}@@ -91,8 +88,7 @@ func TestCropImageChunks_ZeroTasksReportsNothing(t *testing.T) {chunks := []schema.ChunkDoc{{- Text: "no positions",- CKType: "text",+ Text: "no positions",},}@@ -116,12 +112,10 @@ func TestCropImageChunks_CancelledContextDoesNotReport(t *testing.T) {{Text: "content 1",Positions: jsonPositions(t, []float64{1, 0, 100, 0, 100}),- CKType: "text",},{Text: "content 2",Positions: jsonPositions(t, []float64{1, 100, 200, 100, 200}),- CKType: "text",},}diff --git a/internal/ingestion/component/chunker/pdfcrop_text_test.go b/internal/ingestion/component/chunker/pdfcrop_text_test.goindex d460422613..805362ba3c 100644--- a/internal/ingestion/component/chunker/pdfcrop_text_test.go+++ b/internal/ingestion/component/chunker/pdfcrop_text_test.go@@ -42,10 +42,10 @@ func TestCropImageChunksPreservesCompleteTextLines(t *testing.T) {ck schema.ChunkDocwantRows int}{- {"text", schema.ChunkDoc{Text: "two lines", CKType: "text", PDFPositions: pos}, 48},+ {"text", schema.ChunkDoc{Text: "two lines", PDFPositions: pos}, 48},{"text fallback", schema.ChunkDoc{Text: "two lines", DocType: "text", Positions: pos}, 48},- {"image", schema.ChunkDoc{Text: "figure", CKType: "image", PDFPositions: pos}, 24},- {"table", schema.ChunkDoc{Text: "table", CKType: "table", PDFPositions: pos}, 24},+ {"image", schema.ChunkDoc{Text: "figure", DocType: "image", PDFPositions: pos}, 24},+ {"table", schema.ChunkDoc{Text: "table", DocType: "table", PDFPositions: pos}, 24},} {t.Run(tc.name, func(t *testing.T) {out := cropImageChunks(t.Context(), textCropEngine{page: page}, []schema.ChunkDoc{tc.ck})@@ -83,7 +83,7 @@ func TestCropImageChunksKeepsSeparateMediaBoxes(t *testing.T) {pos := jsonPositions(t,[]float64{1, 10, 60, 112.0 / 3, 162.0 / 3},[]float64{1, 10, 60, 162.0 / 3, 212.0 / 3})- out := cropImageChunks(t.Context(), textCropEngine{page: page}, []schema.ChunkDoc{{Text: typ, CKType: typ, PDFPositions: pos}})+ out := cropImageChunks(t.Context(), textCropEngine{page: page}, []schema.ChunkDoc{{Text: typ, DocType: typ, PDFPositions: pos}})img, err := util.DecodeBase64PNG(strings.TrimPrefix(out[0].Image, "data:image/png;base64,"))if err != nil {t.Fatal(err)diff --git a/internal/ingestion/component/chunker/qa.go b/internal/ingestion/component/chunker/qa.goindex fb09b86f3b..e10c1efb6a 100644--- a/internal/ingestion/component/chunker/qa.go+++ b/internal/ingestion/component/chunker/qa.go@@ -423,10 +423,9 @@ func extractQAJSON(items []schema.ChunkDoc, fileType string) []qaPair {// block that merely opens with "<table" text (no row) stays on the// prose path instead of silently pairing nothing. (Python's qa.py// drops plain-text table payloads such as PDF tables entirely; the- // walker keeps their pairs.) Pre-upgrade row-IR payloads- // (ck_type: table_row/table_header with cells) hold no markup and- // fall through to the text extractor, so documents from before this- // wire must be re-parsed rather than re-chunked.+ // walker keeps their pairs.) Pre-upgrade row-IR payloads hold no+ // markup and fall through to the text extractor, so documents from+ // before this wire must be re-parsed rather than re-chunked.var rows [][]stringif td != nil {rows = td.CellTexts()diff --git a/internal/ingestion/component/chunker/qa_test.go b/internal/ingestion/component/chunker/qa_test.goindex 9a1e8785a4..9fdc5e3fb4 100644--- a/internal/ingestion/component/chunker/qa_test.go+++ b/internal/ingestion/component/chunker/qa_test.go@@ -197,7 +197,6 @@ func TestQAChunker_JSONCSVNameUsesStrictRowShapeWithoutFileType(t *testing.T) {"json": []map[string]any{{"text": "<table><tr><td>question</td><td></td><td>unexpected</td></tr></table>","doc_type_kwd": "table",- "ck_type": "table",}},}out, err := comp.Invoke(t.Context(), nil, inputs)diff --git a/internal/ingestion/component/chunker/table.go b/internal/ingestion/component/chunker/table.goindex cf455aba8e..3bfae3bc1e 100644--- a/internal/ingestion/component/chunker/table.go+++ b/internal/ingestion/component/chunker/table.go@@ -114,7 +114,7 @@ func (c *TableChunkerComponent) invoke(_ context.Context, inputs map[string]any)// tableItems returns the per-row records, preferring JSONResult and// falling back to Chunks. Each table row becomes exactly one chunk; every// other payload record passes through as one chunk — including pre-upgrade-// row-IR records (ck_type: table_row/table_header with cells), which hold no+// pre-upgrade row-IR records, which hold no// markup and therefore keep no per-row positions; documents from before this// wire must be re-parsed rather than re-chunked.func tableItems(items, chunks []schema.ChunkDoc) []schema.ChunkDoc {diff --git a/internal/ingestion/component/chunker/table_test.go b/internal/ingestion/component/chunker/table_test.goindex 86f7b39c3c..82e5189b9f 100644--- a/internal/ingestion/component/chunker/table_test.go+++ b/internal/ingestion/component/chunker/table_test.go@@ -45,7 +45,7 @@ func tableChunksOf(t *testing.T, inputs map[string]any) []map[string]any {// tableItem builds a chunker JSON input item carrying the structured TableData// contract a post-migration producer emits (derived from markup), plus any-// extra fields (positions, sheet_index, ck_type, ...). It deliberately does+// extra fields (positions, sheet_index, ...). It deliberately does// NOT set "text": the chunker derives a plain text body from TableData.func tableItem(t *testing.T, html string, extra ...map[string]any) map[string]any {t.Helper()@@ -125,7 +125,7 @@ func TestTableChunkerPreservesHeaderOnlyTable(t *testing.T) {if len(chunks) != 1 {t.Fatalf("got %d chunks, want the header-only table preserved", len(chunks))}- if chunks[0]["ck_type"] != "table" {+ if chunks[0]["doc_type_kwd"] != "table" {t.Fatalf("chunk = %#v, want a table chunk", chunks[0])}if got, ok := chunks[0]["text"].(string); !ok || got == "" {@@ -154,7 +154,7 @@ func TestTableChunkerPreservesHeaderOnlySheetAlongsideDataSheet(t *testing.T) {!strings.Contains(row0, "<td>paid</td>") || !strings.Contains(row0, "<th>ID</th>") {t.Fatalf("chunk0 = %#v", chunks[0])}- if chunks[1]["ck_type"] != "table" || chunks[1]["table"] != nil {+ if chunks[1]["doc_type_kwd"] != "table" || chunks[1]["table"] != nil {t.Fatalf("chunk1 = %#v, want the header-only segment whole with TableData cleared", chunks[1])}}@@ -174,7 +174,6 @@ func TestTableChunker_ExpandsHTMLRowsWithAlignedPositions(t *testing.T) {"output_format": "json","json": []map[string]any{tableItem(t, table, map[string]any{- "ck_type": "table","sheet_index": 1,"positions": [][]float64{{1, 1, 1, 1, 2}, {1, 2, 2, 1, 2}, {1, 3, 3, 1, 2}},}),@@ -238,7 +237,6 @@ func TestTableChunker_NonSpreadsheetPositionsNotAttached(t *testing.T) {"output_format": "json","json": []map[string]any{tableItem(t, table, map[string]any{- "ck_type": "table","positions": [][]float64{{1, 1, 1, 1, 1}, {1, 2, 2, 1, 1}, {1, 3, 3, 1, 1}},}),},@@ -267,7 +265,7 @@ func TestTableChunker_HeaderOnlyHTMLTableKeepsWholePayload(t *testing.T) {"name": "template.xlsx","output_format": "json","json": []map[string]any{- tableItem(t, table, map[string]any{"ck_type": "table"}),+ tableItem(t, table, map[string]any{}),},})if len(chunks) != 1 {diff --git a/internal/ingestion/component/chunker/table_wire_invariant_test.go b/internal/ingestion/component/chunker/table_wire_invariant_test.goindex 7502bf0557..c7d08ddc5d 100644--- a/internal/ingestion/component/chunker/table_wire_invariant_test.go+++ b/internal/ingestion/component/chunker/table_wire_invariant_test.go@@ -55,9 +55,8 @@ func TestTableWireHasNoRowIRChunkTypes(t *testing.T) {continue}for i, chunk := range chunks {- kind, _ := chunk["ck_type"].(string)- if kind == "table_header" || kind == "table_row" {- t.Errorf("%s chunk[%d] ck_type = %q: the row IR must not come back", name, i, kind)+ if kind, ok := chunk["ck_type"]; ok {+ t.Errorf("%s chunk[%d] emits retired ck_type %v: the row IR must not come back", name, i, kind)}}tested++diff --git a/internal/ingestion/component/chunker/testdata/parity/README.md b/internal/ingestion/component/chunker/testdata/parity/README.mdindex 64b6501818..c2597b05d7 100644--- a/internal/ingestion/component/chunker/testdata/parity/README.md+++ b/internal/ingestion/component/chunker/testdata/parity/README.md@@ -98,7 +98,7 @@ Field comparison is deliberately asymmetric:field is justified rather than assumed harmless.The allowed set is derived per case rather than hardcoded, because it is not-constant across components: `TokenChunker` emits `ck_type`/`tk_nums`/`id` while+constant across components: `TokenChunker` emits `tk_nums`/`id` while`TitleChunker` emits only `id`.## No CGO requireddiff --git a/internal/ingestion/component/chunker/testdata/parity/known_diffs.json b/internal/ingestion/component/chunker/testdata/parity/known_diffs.jsonindex c0c3ff696c..fb904a0048 100644--- a/internal/ingestion/component/chunker/testdata/parity/known_diffs.json+++ b/internal/ingestion/component/chunker/testdata/parity/known_diffs.json@@ -15,9 +15,9 @@"tag": "go_intentional","kind": "extra_fields","applies_to": ["token__*"],- "fields": ["ck_type", "tk_nums"],+ "fields": ["tk_nums"],"permanent": true,- "reason": "Python builds both fields internally (token_chunker.py:141,144) but _finalize_json_chunks (token_chunker.py:256-259) whitelists only text/doc_type_kwd/_pdf_positions/mom/img_id, dropping them. Go keeps them because they are still live downstream: needsCrop (pdfcrop_cgo.go:151) reads ck_type, and tokenizer.go:615 forwards tk_nums. The divergence follows from Go cropping after the chunker returns, whereas Python crops inside _invoke (token_chunker.py:370) before finalizing."+ "reason": "Python builds tk_nums internally (token_chunker.py:144) but _finalize_json_chunks (token_chunker.py:256-259) whitelists only text/doc_type_kwd/_pdf_positions/mom/img_id, dropping it. Go keeps it because tokenizer.go forwards tk_nums to the Tokenizer."},{"id": "token-hardcap-chunk-count",diff --git a/internal/ingestion/component/chunker/title.go b/internal/ingestion/component/chunker/title.goindex 4ff195dbc4..282df0f280 100644--- a/internal/ingestion/component/chunker/title.go+++ b/internal/ingestion/component/chunker/title.go@@ -337,7 +337,9 @@ func resolveTitleLevels(records []lineRecord, p *titleChunkerParam) []int {out[i] = lvlcontinue}- if rec.ckType == "heading" {+ // Only the asserted "heading" marker bypasses the notTitle guard;+ // model heuristics use "title" and go through matchLayoutLevel below.+ if isHeadingLayout(rec.layout) {out[i] = fallbackLevelcontinue}@@ -640,7 +642,6 @@ type lineRecord struct {imgID *stringimage stringlayout string- ckType stringpdfPositions json.RawMessagepositions json.RawMessagepreserveOrder booldiff --git a/internal/ingestion/component/chunker/token.go b/internal/ingestion/component/chunker/token.goindex 68cf9c0e4e..6351a056f4 100644--- a/internal/ingestion/component/chunker/token.go+++ b/internal/ingestion/component/chunker/token.go@@ -286,41 +286,39 @@ func stripChunkerRuntimeTimestamps(inputs map[string]any) map[string]any {// cropTitleChunks crops image/table/text previews for chunks produced by// the Title/Group/Hierarchy chunkers, mirroring the TokenChunker JSON path-// (cropImageChunks at token.go:513). A nil engine — or an-// empty chunk list — leaves chunks unchanged (best-effort, matching the-// on-demand PDF crop contract used by the TokenChunker path).+// (cropImageChunks at token.go:513). A nil engine (or an undecodable payload)+// skips cropping, but the returned maps are still normalized so retired+// ck_type and chunker-internal structural layout never leave the component.func cropTitleChunks(ctx context.Context, engine deepdoctype.PDFEngine, chunks []map[string]any) []map[string]any {- if engine == nil || len(chunks) == 0 {+ if len(chunks) == 0 {+ return chunks+ }+ if engine == nil {+ normalizeTitleChunkMaps(chunks)return chunks}docs, _, err := schema.ChunkDocsFromAny(chunks)if err != nil || len(docs) == 0 {+ normalizeTitleChunkMaps(chunks)return chunks}- // The Title/Group/Hierarchy chunkers emit doc_type_kwd but not the- // ck_type field that cropImageChunks' needsCrop consults- // (pdfcrop_cgo.go:151). Derive ck_type from doc_type_kwd so the crop- // decision matches the TokenChunker path. The derived ck_type is- // stripped from the returned maps so the downstream chunk shape is- // unchanged (setting ck_type in the real output would also change- // how a downstream TokenChunker merges these chunks — a separate- // concern, out of scope here).- for i := range docs {- if docs[i].CKType == "" {- switch docs[i].DocType {- case "image", "table":- docs[i].CKType = docs[i].DocType- default:- docs[i].CKType = "text"- }- }- }cropped := cropImageChunks(ctx, engine, docs)- out := schema.ChunkDocsToMaps(cropped)- for _, m := range out {- delete(m, "ck_type")+ for i := range cropped {+ clearStructuralLayout(&cropped[i])+ }+ return schema.ChunkDocsToMaps(cropped)+}++// normalizeTitleChunkMaps keeps the Title-family output shape independent of+// whether a PDF engine is available: retired ck_type and chunker-internal+// structural layout never leave the component.+func normalizeTitleChunkMaps(chunks []map[string]any) {+ for _, chunk := range chunks {+ delete(chunk, "ck_type")+ if layout, _ := chunk["layout"].(string); isStructuralLayout(layout) {+ delete(chunk, "layout")+ }}- return out}// invokeTextPayload handles plain-text input (output_format in@@ -370,10 +368,9 @@ func (c *TokenChunkerComponent) chunkPerSegment(text string, delimPattern, child}textDocs := make([]schema.ChunkDoc, 0, len(cleaned))for _, s := range cleaned {- // Python's unstructured text path emits only text here. Keep CKType for- // Go's downstream crop dispatch, but do not synthesize structured-input- // metadata such as doc_type_kwd.- textDocs = append(textDocs, schema.ChunkDoc{Text: s, CKType: "text"})+ // Python's unstructured text path emits only text here. Do not+ // synthesize structured-input metadata such as doc_type_kwd.+ textDocs = append(textDocs, schema.ChunkDoc{Text: s})}// A backtick children delimiter is also custom and is dropped; a bare// children delimiter (e.g. "。") is retained losslessly like the main@@ -484,7 +481,7 @@ func (c *TokenChunkerComponent) mergeByTokenSize(text string, delimPattern, chilif sub == "" {continue}- units = append(units, schema.ChunkDoc{Text: sub, TKNums: intPtr(tokenizeStr(sub)), CKType: "text"})+ units = append(units, schema.ChunkDoc{Text: sub, TKNums: intPtr(tokenizeStr(sub))})}} else {sections := []string{text}@@ -496,7 +493,7 @@ func (c *TokenChunkerComponent) mergeByTokenSize(text string, delimPattern, chilt := "\n" + sectk := tokenizeStr(t)if tk <= target {- units = append(units, schema.ChunkDoc{Text: t, TKNums: intPtr(tk), CKType: "text"})+ units = append(units, schema.ChunkDoc{Text: t, TKNums: intPtr(tk)})continue}// Oversized section: split on production sentence delimiters into@@ -515,10 +512,10 @@ func (c *TokenChunkerComponent) mergeByTokenSize(text string, delimPattern, chil}hadPart = trueseg := "\n" + part- units = append(units, schema.ChunkDoc{Text: seg, TKNums: intPtr(tokenizeStr(seg)), CKType: "text"})+ units = append(units, schema.ChunkDoc{Text: seg, TKNums: intPtr(tokenizeStr(seg))})}if !hadPart {- units = append(units, schema.ChunkDoc{Text: t, TKNums: intPtr(tokenizeStr(t)), CKType: "text"})+ units = append(units, schema.ChunkDoc{Text: t, TKNums: intPtr(tokenizeStr(t))})}}}@@ -730,17 +727,17 @@ func separateItemBoundaries(perItem [][]schema.ChunkDoc) {continue}firstIndex, lastIndex := 0, len(units)-1- for firstIndex <= lastIndex && units[firstIndex].CKType == "text" && units[firstIndex].Text == "" {+ for firstIndex <= lastIndex && itemDocType(units[firstIndex]) == "text" && units[firstIndex].Text == "" {firstIndex++}- for lastIndex >= firstIndex && units[lastIndex].CKType == "text" && units[lastIndex].Text == "" {+ for lastIndex >= firstIndex && itemDocType(units[lastIndex]) == "text" && units[lastIndex].Text == "" {lastIndex--}if firstIndex > lastIndex {continue}first := &units[firstIndex]- if previous != nil && previous.CKType == "text" && first.CKType == "text" &&+ if previous != nil && itemDocType(*previous) == "text" && itemDocType(*first) == "text" &&strings.TrimRightFunc(previous.Text, unicode.IsSpace) == previous.Text &&strings.TrimLeftFunc(first.Text, unicode.IsSpace) == first.Text {first.Text = "\n" + first.Text@@ -803,7 +800,7 @@ func chunkFromItem(it schema.ChunkDoc, delimPattern *regexp.Regexp, keepDelim bo// buildChunkMap constructs the python-compatible chunk payload.//-// The chunker output carries the basic text+doc_type_kwd+ck_type+// The chunker output carries the basic text+doc_type_kwd// fields plus the per-chunk meta fields the python// rag/flow/chunker/token_chunker.py emits://@@ -824,7 +821,6 @@ func buildChunkDoc(it schema.ChunkDoc, ckType, text, ctxAbove, ctxBelow string)out := schema.ChunkDoc{Text: text,DocType: ckType,- CKType: ckType,TKNums: intPtr(tokenizeStr(text)),Mom: it.Mom,ImgID: it.ImgID,@@ -837,6 +833,9 @@ func buildChunkDoc(it schema.ChunkDoc, ckType, text, ctxAbove, ctxBelow string)SheetIndex: it.SheetIndex,Extra: cloneExtra(it.Extra),}+ // buildChunkDoc rewrites every unit to its content type, so the parser's+ // block-kind assertion no longer applies to the emitted chunk.+ clearStructuralLayout(&out)if ctxAbove != "" {out.ContextAbove = ctxAbove}@@ -884,12 +883,12 @@ func attachMediaContext(chunks []schema.ChunkDoc, tableCtx, imageCtx int) []schereturn chunks}for i, ck := range chunks {- ckType := ck.CKType- if ckType != "table" && ckType != "image" {+ docType := itemDocType(ck)+ if docType != "table" && docType != "image" {continue}ctx := imageCtx- if ckType == "table" {+ if docType == "table" {ctx = tableCtx}if ctx <= 0 {@@ -913,7 +912,7 @@ func collectContext(chunks []schema.ChunkDoc, i, ctxTokens int, above bool) striif above {pos = i - 1for pos >= 0 && remain > 0 {- if chunks[pos].CKType == "text" {+ if itemDocType(chunks[pos]) == "text" {tk := intValue(chunks[pos].TKNums)txt := chunks[pos].Textif tk >= remain {@@ -929,7 +928,7 @@ func collectContext(chunks []schema.ChunkDoc, i, ctxTokens int, above bool) stri} else {pos = i + 1for pos < len(chunks) && remain > 0 {- if chunks[pos].CKType == "text" {+ if itemDocType(chunks[pos]) == "text" {tk := intValue(chunks[pos].TKNums)txt := chunks[pos].Textif tk >= remain {@@ -1088,7 +1087,7 @@ func mergeUnits(units []schema.ChunkDoc, target int, overlapPct float64, joinSepprevIdx := -1for i := range units {ck := units[i]- if ck.CKType != "text" {+ if itemDocType(ck) != "text" {merged = append(merged, cloneChunkDoc(ck))mergedItems = append(mergedItems, nil)prevIdx = -1@@ -1224,7 +1223,7 @@ func expandOversizedUnits(units []schema.ChunkDoc, target int) []schema.ChunkDoc}out := make([]schema.ChunkDoc, 0, len(units))for _, ck := range units {- if ck.CKType != "text" {+ if itemDocType(ck) != "text" {out = append(out, ck)continue}@@ -1286,7 +1285,7 @@ func splitOversizedText(ck schema.ChunkDoc, target int) []schema.ChunkDoc {wsPending += partcontinue}- p := schema.ChunkDoc{Text: wsPending + part, DocType: ck.DocType, CKType: "text"}+ p := schema.ChunkDoc{Text: wsPending + part, DocType: ck.DocType}wsPending = ""if n := tokenizeStr(p.Text); n > target {pieces = append(pieces, hardSplitPiece(p.Text, ck.DocType, target)...)@@ -1312,7 +1311,7 @@ func splitOversizedText(ck schema.ChunkDoc, target int) []schema.ChunkDoc {if n := tokenizeStr(wsPending); n > target {pieces = append(pieces, hardSplitPiece(wsPending, ck.DocType, target)...)} else {- pieces = append(pieces, schema.ChunkDoc{Text: wsPending, DocType: ck.DocType, TKNums: intPtr(n), CKType: "text"})+ pieces = append(pieces, schema.ChunkDoc{Text: wsPending, DocType: ck.DocType, TKNums: intPtr(n)})}}}@@ -1345,7 +1344,7 @@ func splitOversizedText(ck schema.ChunkDoc, target int) []schema.ChunkDoc {inherited := cloneChunkDoc(ck)inherited.Text = p.Textinherited.TKNums = p.TKNums- inherited.CKType = "text"+ clearStructuralLayout(&inherited)inherited.PDFPositions = pdfSliced[i]inherited.Positions = posSliced[i]pieces[i] = inherited@@ -1404,7 +1403,7 @@ func hardSplitPiece(text string, docType string, target int) []schema.ChunkDoc {if target <= 0 || len(tokens) <= target {// target <= 0 is unreachable from validated configs; the byte loop// would emit the whole text as one piece, so mirror that here.- return []schema.ChunkDoc{{Text: text, DocType: docType, TKNums: intPtr(tokenizeStr(text)), CKType: "text"}}+ return []schema.ChunkDoc{{Text: text, DocType: docType, TKNums: intPtr(tokenizeStr(text))}}}// offsets[i] = byte length of the first i tokens decoded. Token ids repeat@@ -1485,7 +1484,7 @@ func hardSplitPiece(text string, docType string, target int) []schema.ChunkDoc {q = p + 1 // never stall (defensive; unreachable above)}piece := text[offsets[p]:offsets[q]]- out = append(out, schema.ChunkDoc{Text: piece, DocType: docType, TKNums: intPtr(tokenizeStr(piece)), CKType: "text"})+ out = append(out, schema.ChunkDoc{Text: piece, DocType: docType, TKNums: intPtr(tokenizeStr(piece))})p = q}return out@@ -1527,17 +1526,17 @@ func hardSplitPieceByteLoop(text string, docType string, target int) []schema.Ch// Safety: never stall. An empty cut or a whole-text cut means no// progress is possible; emit the remaining text as the last piece.if rest != "" {- out = append(out, schema.ChunkDoc{Text: rest, DocType: docType, TKNums: intPtr(tokenizeStr(rest)), CKType: "text"})+ out = append(out, schema.ChunkDoc{Text: rest, DocType: docType, TKNums: intPtr(tokenizeStr(rest))})}rest = ""break}head = rest[:cut]- out = append(out, schema.ChunkDoc{Text: head, DocType: docType, TKNums: intPtr(tokenizeStr(head)), CKType: "text"})+ out = append(out, schema.ChunkDoc{Text: head, DocType: docType, TKNums: intPtr(tokenizeStr(head))})rest = rest[cut:]}if rest != "" {- out = append(out, schema.ChunkDoc{Text: rest, DocType: docType, TKNums: intPtr(tokenizeStr(rest)), CKType: "text"})+ out = append(out, schema.ChunkDoc{Text: rest, DocType: docType, TKNums: intPtr(tokenizeStr(rest))})}return out}@@ -1820,7 +1819,7 @@ func applyChildrenDelimText(docs []schema.ChunkDoc, pattern *regexp.Regexp, keeptext = leading + textleading = ""}- child := schema.ChunkDoc{CKType: d.CKType, Mom: mom}+ child := schema.ChunkDoc{DocType: d.DocType, Layout: d.Layout, Mom: mom}setChunkText(&child, text)out = append(out, child)}diff --git a/internal/ingestion/component/chunker/token_batch1_test.go b/internal/ingestion/component/chunker/token_batch1_test.goindex 32e3946853..ce9ca63e5b 100644--- a/internal/ingestion/component/chunker/token_batch1_test.go+++ b/internal/ingestion/component/chunker/token_batch1_test.go@@ -88,9 +88,9 @@ func TestMergeByTokenSizeFromJSON_OverlapStripsTags(t *testing.T) {}items := [][]schema.ChunkDoc{{- {Text: aText, DocType: "text", CKType: "text", TKNums: intPtr(aN)},- {Text: bText, DocType: "text", CKType: "text", TKNums: intPtr(bN)},- {Text: cText, DocType: "text", CKType: "text", TKNums: intPtr(cN)},+ {Text: aText, DocType: "text", TKNums: intPtr(aN)},+ {Text: bText, DocType: "text", TKNums: intPtr(bN)},+ {Text: cText, DocType: "text", TKNums: intPtr(cN)},},}got := mergeByTokenSizeFromJSON(items, budget, 30.0)@@ -150,11 +150,11 @@ func TestMergeByTokenSizeFromJSON_NonTextBoundaryResetsMergeRun(t *testing.T) {items := [][]schema.ChunkDoc{{- {Text: t1, DocType: "text", CKType: "text", TKNums: intPtr(t1N)},- {Text: t2, DocType: "text", CKType: "text", TKNums: intPtr(t2N)},- {Text: "[image]", DocType: "image", CKType: "image", TKNums: intPtr(1)},- {Text: t3, DocType: "text", CKType: "text", TKNums: intPtr(t3N)},- {Text: t4, DocType: "text", CKType: "text", TKNums: intPtr(t4N)},+ {Text: t1, DocType: "text", TKNums: intPtr(t1N)},+ {Text: t2, DocType: "text", TKNums: intPtr(t2N)},+ {Text: "[image]", DocType: "image", TKNums: intPtr(1)},+ {Text: t3, DocType: "text", TKNums: intPtr(t3N)},+ {Text: t4, DocType: "text", TKNums: intPtr(t4N)},},}got := mergeByTokenSizeFromJSON(items, budget, 0.0)@@ -163,7 +163,7 @@ func TestMergeByTokenSizeFromJSON_NonTextBoundaryResetsMergeRun(t *testing.T) {if len(merged) != 3 {var texts []stringfor _, c := range merged {- texts = append(texts, c.CKType+":"+c.Text)+ texts = append(texts, itemDocType(c)+":"+c.Text)}t.Fatalf("want 3 chunks (merged + non-text + merged), got %d: %v", len(merged), texts)}@@ -174,8 +174,8 @@ func TestMergeByTokenSizeFromJSON_NonTextBoundaryResetsMergeRun(t *testing.T) {if merged[0].Text != t1+t2 {t.Errorf("chunk[0] should be the merged T1+T2 chunk: got %q", merged[0].Text)}- if merged[1].CKType != "image" {- t.Errorf("chunk[1] should be the non-text image chunk, got CKType=%q text=%q", merged[1].CKType, merged[1].Text)+ if itemDocType(merged[1]) != "image" {+ t.Errorf("chunk[1] should be the non-text image chunk, got doc_type=%q text=%q", itemDocType(merged[1]), merged[1].Text)}wantMerged := t3 + t4if merged[2].Text != wantMerged {@@ -207,9 +207,9 @@ func TestMergeByTokenSizeFromJSON_UnderCapNoOverflow(t *testing.T) {}items := [][]schema.ChunkDoc{{- {Text: aText, DocType: "text", CKType: "text", TKNums: intPtr(aN)},- {Text: bText, DocType: "text", CKType: "text", TKNums: intPtr(bN)},- {Text: cText, DocType: "text", CKType: "text", TKNums: intPtr(cN)},+ {Text: aText, DocType: "text", TKNums: intPtr(aN)},+ {Text: bText, DocType: "text", TKNums: intPtr(bN)},+ {Text: cText, DocType: "text", TKNums: intPtr(cN)},},}got := mergeByTokenSizeFromJSON(items, budget, 0.0)@@ -247,8 +247,8 @@ func TestMergeByTokenSizeFromJSON_UnderCapNoOverflow(t *testing.T) {func clampOverlapFixture() [][]schema.ChunkDoc {return [][]schema.ChunkDoc{{- {Text: strings.Repeat("word ", 20), DocType: "text", CKType: "text", TKNums: intPtr(130)},- {Text: "body", DocType: "text", CKType: "text", TKNums: intPtr(5)},+ {Text: strings.Repeat("word ", 20), DocType: "text", TKNums: intPtr(130)},+ {Text: "body", DocType: "text", TKNums: intPtr(5)},},}}@@ -289,8 +289,8 @@ func TestMergeByTokenSizeFromJSON_ClampsOverlappedPct(t *testing.T) {func TestMergeByTokenSizeFromJSON_EmptyPrevKeepsChunk(t *testing.T) {items := [][]schema.ChunkDoc{{- {Text: "", DocType: "text", CKType: "text", TKNums: intPtr(5)},- {Text: "keepme", DocType: "text", CKType: "text", TKNums: intPtr(5)},+ {Text: "", DocType: "text", TKNums: intPtr(5)},+ {Text: "keepme", DocType: "text", TKNums: intPtr(5)},},}got := mergeByTokenSizeFromJSON(items, 128, 0)diff --git a/internal/ingestion/component/chunker/token_delim_lossless_boundary_test.go b/internal/ingestion/component/chunker/token_delim_lossless_boundary_test.goindex 653097af19..3e8082d20e 100644--- a/internal/ingestion/component/chunker/token_delim_lossless_boundary_test.go+++ b/internal/ingestion/component/chunker/token_delim_lossless_boundary_test.go@@ -64,7 +64,7 @@ func TestDelimiterLossless_BoundaryReconstruction(t *testing.T) {// adjacent child rather than dropped, so the child texts still reconstruct "A\n\nB".func TestSplitByChildrenKeepsWhitespaceOnlyPiece(t *testing.T) {pat := compileDelimPattern([]string{"\n"})- doc := schema.ChunkDoc{Text: "A\n\nB", DocType: "text", CKType: "text"}+ doc := schema.ChunkDoc{Text: "A\n\nB", DocType: "text"}out := splitByChildren([]schema.ChunkDoc{doc}, pat, true)var got strings.Builderfor _, c := range out {@@ -80,7 +80,7 @@ func TestSplitByChildrenKeepsWhitespaceOnlyPiece(t *testing.T) {// text-children split helper.func TestApplyChildrenDelimTextKeepsWhitespaceOnlyPiece(t *testing.T) {pat := compileDelimPattern([]string{"\n"})- doc := schema.ChunkDoc{Text: "A\n\nB", CKType: "text"}+ doc := schema.ChunkDoc{Text: "A\n\nB"}out := applyChildrenDelimText([]schema.ChunkDoc{doc}, pat, true)var got strings.Builderfor _, c := range out {diff --git a/internal/ingestion/component/chunker/token_delim_lossless_followup_test.go b/internal/ingestion/component/chunker/token_delim_lossless_followup_test.goindex 086dfbd81e..ca8582c891 100644--- a/internal/ingestion/component/chunker/token_delim_lossless_followup_test.go+++ b/internal/ingestion/component/chunker/token_delim_lossless_followup_test.go@@ -41,8 +41,8 @@ import (func TestApplyChildrenDelimText_NoCrossParentFold(t *testing.T) {pattern := regexp.MustCompile("\n")docs := []schema.ChunkDoc{- {Text: "\nA", CKType: "text"},- {Text: "\nA", CKType: "text"},+ {Text: "\nA"},+ {Text: "\nA"},}out := applyChildrenDelimText(docs, pattern, true)if len(out) != 2 {@@ -73,7 +73,7 @@ func TestChunkFromItem_RetainsWhitespacePieces(t *testing.T) {}for _, c := range cases {t.Run(c.name, func(t *testing.T) {- it := schema.ChunkDoc{Text: c.text, CKType: "text"}+ it := schema.ChunkDoc{Text: c.text}out := chunkFromItem(it, pattern, true)got := make([]string, 0, len(out))for _, d := range out {@@ -92,7 +92,7 @@ func TestChunkFromItem_RetainsWhitespacePieces(t *testing.T) {func TestChunkFromItem_LosslessWhitespaceReconstruct(t *testing.T) {pattern := regexp.MustCompile("\n")src := "alpha\n\nbeta\n\ngamma"- it := schema.ChunkDoc{Text: src, CKType: "text"}+ it := schema.ChunkDoc{Text: src}out := chunkFromItem(it, pattern, true)var recon stringfor _, d := range out {diff --git a/internal/ingestion/component/chunker/token_hardcap_test.go b/internal/ingestion/component/chunker/token_hardcap_test.goindex 30361c327d..2e0493c35c 100644--- a/internal/ingestion/component/chunker/token_hardcap_test.go+++ b/internal/ingestion/component/chunker/token_hardcap_test.go@@ -44,10 +44,10 @@ import (func TestMergeUnits_UnderCapRunningSum(t *testing.T) {const target = 50units := []schema.ChunkDoc{- {Text: "u1", TKNums: intPtr(25), CKType: "text"},- {Text: "u2", TKNums: intPtr(25), CKType: "text"},- {Text: "u3", TKNums: intPtr(25), CKType: "text"},- {Text: "u4", TKNums: intPtr(25), CKType: "text"},+ {Text: "u1", TKNums: intPtr(25)},+ {Text: "u2", TKNums: intPtr(25)},+ {Text: "u3", TKNums: intPtr(25)},+ {Text: "u4", TKNums: intPtr(25)},}got := mergeUnits(units, target, 0, "")if len(got) != 2 {@@ -64,15 +64,15 @@ func TestMergeUnits_UnderCapRunningSum(t *testing.T) {// still merges (<= target), prev+incoming > target starts a fresh chunk.func TestMergeUnits_BoundaryExactFit(t *testing.T) {fit := mergeUnits([]schema.ChunkDoc{- {Text: "a", TKNums: intPtr(25), CKType: "text"},- {Text: "b", TKNums: intPtr(25), CKType: "text"},+ {Text: "a", TKNums: intPtr(25)},+ {Text: "b", TKNums: intPtr(25)},}, 50, 0, "")if len(fit) != 1 {t.Fatalf("exact-fit join must merge, got %d chunks", len(fit))}over := mergeUnits([]schema.ChunkDoc{- {Text: "a", TKNums: intPtr(30), CKType: "text"},- {Text: "b", TKNums: intPtr(30), CKType: "text"},+ {Text: "a", TKNums: intPtr(30)},+ {Text: "b", TKNums: intPtr(30)},}, 50, 0, "")if len(over) != 2 {t.Fatalf("overflowing join must start a fresh chunk, got %d chunks", len(over))@@ -86,7 +86,7 @@ func TestMergeUnits_ExpandsOversizedUnit(t *testing.T) {const target = 30body := strings.Repeat("word ", 100) // ~100 tokens, no sentence delimitergot := mergeUnits([]schema.ChunkDoc{- {Text: body, CKType: "text", TKNums: intPtr(tokenizeStr(body))},+ {Text: body, TKNums: intPtr(tokenizeStr(body))},}, target, 0, "")if len(got) < 2 {t.Fatalf("oversized unit must be split, got %d chunk(s)", len(got))@@ -114,7 +114,7 @@ func TestMergeUnits_ExpandsOversizedSentence(t *testing.T) {t.Fatal("fixture: tokenize returned 0 for sentence")}got := mergeUnits([]schema.ChunkDoc{- {Text: body, CKType: "text", TKNums: intPtr(tokenizeStr(body))},+ {Text: body, TKNums: intPtr(tokenizeStr(body))},}, target, 0, "")if len(got) < 2 {t.Fatalf("oversized unit must be sentence-split, got %d chunk(s)", len(got))@@ -139,7 +139,7 @@ func TestMergeUnits_SubChunksKeepPositions(t *testing.T) {body := strings.Repeat("word ", 100)pos := []byte(`[{"page":1}]`)got := mergeUnits([]schema.ChunkDoc{- {Text: body, CKType: "text", TKNums: intPtr(tokenizeStr(body)), PDFPositions: pos},+ {Text: body, TKNums: intPtr(tokenizeStr(body)), PDFPositions: pos},}, target, 0, "")if len(got) < 2 {t.Fatalf("want >=2 chunks, got %d", len(got))@@ -154,7 +154,7 @@ func TestMergeUnits_SubChunksKeepPositions(t *testing.T) {// TestMergeUnits_AtomicNonText verifies non-text chunks (tables/images) are// never split even when their token count exceeds the target.func TestMergeUnits_AtomicNonText(t *testing.T) {- table := schema.ChunkDoc{Text: strings.Repeat("t", 500), CKType: "table", TKNums: intPtr(500)}+ table := schema.ChunkDoc{Text: strings.Repeat("t", 500), DocType: "table", TKNums: intPtr(500)}got := mergeUnits([]schema.ChunkDoc{table}, 50, 0, "")if len(got) != 1 || got[0].Text != table.Text {t.Fatalf("table chunk must pass through unchanged, got %#v", got)@@ -168,7 +168,7 @@ func TestMergeUnits_ExpandedPiecesKeepDocType(t *testing.T) {const target = 30body := strings.Repeat("word ", 100)got := mergeUnits([]schema.ChunkDoc{- {Text: body, DocType: "text", CKType: "text", TKNums: intPtr(tokenizeStr(body))},+ {Text: body, DocType: "text", TKNums: intPtr(tokenizeStr(body))},}, target, 0, "")if len(got) < 2 {t.Fatalf("oversized unit must be split, got %d chunk(s)", len(got))@@ -188,7 +188,7 @@ func TestMergeUnits_HardSplitNeverSplitsCoordTag(t *testing.T) {const tag = "@@1\t2\t3\t4##"body := strings.Repeat("word ", 30) + tag + strings.Repeat("pad ", 30)got := mergeUnits([]schema.ChunkDoc{- {Text: body, CKType: "text", TKNums: intPtr(tokenizeStr(body))},+ {Text: body, TKNums: intPtr(tokenizeStr(body))},}, target, 0, "")if len(got) < 3 {t.Fatalf("want multiple pieces, got %d", len(got))@@ -213,8 +213,8 @@ func TestMergeUnits_HardSplitNeverSplitsCoordTag(t *testing.T) {func TestMergeUnits_OverlapStrictCap(t *testing.T) {const target = 40units := []schema.ChunkDoc{- {Text: strings.Repeat("word ", 30), CKType: "text"}, // ~30 tokens- {Text: strings.Repeat("pad ", 38), CKType: "text"}, // ~38 tokens, near target+ {Text: strings.Repeat("word ", 30)}, // ~30 tokens+ {Text: strings.Repeat("pad ", 38)}, // ~38 tokens, near target}got := mergeUnits(units, target, 20, "\n")if len(got) < 2 {@@ -238,7 +238,7 @@ func TestMergeUnits_ExpansionPreservesWhitespaceFragments(t *testing.T) {const target = 30body := strings.Repeat("word ", 60) + "\n\n" + strings.Repeat("pad ", 60)got := mergeUnits([]schema.ChunkDoc{- {Text: body, CKType: "text", TKNums: intPtr(tokenizeStr(body))},+ {Text: body, TKNums: intPtr(tokenizeStr(body))},}, target, 0, "")if len(got) < 2 {t.Fatalf("oversized unit must be split, got %d chunk(s)", len(got))@@ -262,7 +262,6 @@ func TestMergeUnits_ExpandedPiecesCarryMetadata(t *testing.T) {unit := schema.ChunkDoc{Text: body,DocType: "text",- CKType: "text",TKNums: intPtr(tokenizeStr(body)),PDFPositions: []byte(`[{"page":1}]`),Mom: "parent-id",@@ -299,7 +298,7 @@ func TestMergeByTokenSizeFromJSON_HardCapStrict(t *testing.T) {var items []schema.ChunkDocfor i := 0; i < 12; i++ {text := strings.Repeat("w ", 12)- items = append(items, schema.ChunkDoc{Text: text, DocType: "text", CKType: "text", TKNums: intPtr(tokenizeStr(text))})+ items = append(items, schema.ChunkDoc{Text: text, DocType: "text", TKNums: intPtr(tokenizeStr(text))})}got := mergeByTokenSizeFromJSON([][]schema.ChunkDoc{items}, budget, 0)if len(got[0]) < 2 {@@ -325,8 +324,8 @@ func TestMergeUnits_JoinSepAddsTokens(t *testing.T) {t.Skipf("fixture: joinSep newline adds no token for these texts (joined=%d target=%d)", joinedTK, target)}got := mergeUnits([]schema.ChunkDoc{- {Text: a, TKNums: intPtr(na), CKType: "text"},- {Text: b, TKNums: intPtr(nb), CKType: "text"},+ {Text: a, TKNums: intPtr(na)},+ {Text: b, TKNums: intPtr(nb)},}, target, 0, "\n")if len(got) != 2 {t.Fatalf("joined text exceeds target via joinSep: want 2 chunks, got %d", len(got))@@ -349,7 +348,7 @@ func TestExpandOversizedUnits_AllWhitespaceOverCap(t *testing.T) {if n := tokenizeStr(ws); n <= target {t.Skipf("fixture: whitespace tokenizes to %d <= target %d", n, target)}- got := expandOversizedUnits([]schema.ChunkDoc{{Text: ws, CKType: "text"}}, target)+ got := expandOversizedUnits([]schema.ChunkDoc{{Text: ws}}, target)if len(got) < 2 {t.Fatalf("all-whitespace unit over cap must be hard-split, got %d chunk(s)", len(got))}diff --git a/internal/ingestion/component/chunker/token_json_overlap_test.go b/internal/ingestion/component/chunker/token_json_overlap_test.goindex bbe517092d..075e896a1b 100644--- a/internal/ingestion/component/chunker/token_json_overlap_test.go+++ b/internal/ingestion/component/chunker/token_json_overlap_test.go@@ -59,9 +59,9 @@ func TestMergeByTokenSizeFromJSON_TKNumsConsistency(t *testing.T) {}items := [][]schema.ChunkDoc{{- {Text: aText, DocType: "text", CKType: "text", TKNums: intPtr(aN)},- {Text: bText, DocType: "text", CKType: "text", TKNums: intPtr(bN)},- {Text: cText, DocType: "text", CKType: "text", TKNums: intPtr(cN)},+ {Text: aText, DocType: "text", TKNums: intPtr(aN)},+ {Text: bText, DocType: "text", TKNums: intPtr(bN)},+ {Text: cText, DocType: "text", TKNums: intPtr(cN)},},}got := mergeByTokenSizeFromJSON(items, budget, overlapPct)diff --git a/internal/ingestion/component/chunker/token_merge_alloc_test.go b/internal/ingestion/component/chunker/token_merge_alloc_test.goindex 2ac85421fa..cfe146b665 100644--- a/internal/ingestion/component/chunker/token_merge_alloc_test.go+++ b/internal/ingestion/component/chunker/token_merge_alloc_test.go@@ -126,7 +126,6 @@ func benchMergeUnitsJSON(b *testing.B, n int) {for i := 0; i < n; i++ {units[i] = schema.ChunkDoc{Text: strings.Repeat("word ", 5),- CKType: "text",TKNums: intPtr(5),PDFPositions: append(json.RawMessage(nil), pos...),}diff --git a/internal/ingestion/component/chunker/token_merge_parity_test.go b/internal/ingestion/component/chunker/token_merge_parity_test.goindex 6ccf0bb681..ab38e0d4a2 100644--- a/internal/ingestion/component/chunker/token_merge_parity_test.go+++ b/internal/ingestion/component/chunker/token_merge_parity_test.go@@ -91,7 +91,7 @@ func goMergeGroupsOracle(paragraphs []string, cap int, overlapPct float64) []strif sub == "" {continue}- units = append(units, schema.ChunkDoc{Text: sub, TKNums: intPtr(tokenizeStr(sub)), CKType: "text"})+ units = append(units, schema.ChunkDoc{Text: sub, TKNums: intPtr(tokenizeStr(sub))})}merged := mergeUnits(units, cap, overlapPct, "")out := make([]string, 0, len(merged))@@ -239,7 +239,7 @@ func goMergeWithOverlapOracle(paragraphs []string, chunkTokenSize int, overlappeif sub == "" {continue}- units = append(units, schema.ChunkDoc{Text: sub, TKNums: intPtr(tokenizeStr(sub)), CKType: "text"})+ units = append(units, schema.ChunkDoc{Text: sub, TKNums: intPtr(tokenizeStr(sub))})}merged := mergeUnits(units, chunkTokenSize, overlappedPercent, "")out := make([]string, 0, len(merged))diff --git a/internal/ingestion/component/chunker/token_merge_units_test.go b/internal/ingestion/component/chunker/token_merge_units_test.goindex 632956deda..0cf94e33b3 100644--- a/internal/ingestion/component/chunker/token_merge_units_test.go+++ b/internal/ingestion/component/chunker/token_merge_units_test.go@@ -216,7 +216,10 @@ func TestMergeUnitsMatchesHardCapOracle(t *testing.T) {t.Run(c.name, func(t *testing.T) {units := make([]schema.ChunkDoc, len(c.texts))for i, tx := range c.texts {- units[i] = schema.ChunkDoc{Text: tx, TKNums: intPtr(c.tkNums[i]), CKType: c.kinds[i]}+ units[i] = schema.ChunkDoc{Text: tx, TKNums: intPtr(c.tkNums[i])}+ if c.kinds[i] == "image" || c.kinds[i] == "table" {+ units[i].DocType = c.kinds[i]+ }}want := hardCapMergeUnitsOracle(c.texts, c.tkNums, c.kinds, c.target, c.overlap, c.joinSep)got := mergeUnits(units, c.target, c.overlap, c.joinSep)@@ -255,8 +258,8 @@ func TestMergeUnitsOverlapPrefixIsTagFree(t *testing.T) {for _, c := range cases {t.Run(c.name, func(t *testing.T) {units := []schema.ChunkDoc{- {Text: c.prev, TKNums: intPtr(10), CKType: "text"},- {Text: c.cur, TKNums: intPtr(4), CKType: "text"},+ {Text: c.prev, TKNums: intPtr(10)},+ {Text: c.cur, TKNums: intPtr(4)},}got := mergeUnits(units, c.target, c.overlap, "\n")if len(got) < 2 {diff --git a/internal/ingestion/component/chunker/token_pdfpos_test.go b/internal/ingestion/component/chunker/token_pdfpos_test.goindex 9bde5d471b..f6818c6133 100644--- a/internal/ingestion/component/chunker/token_pdfpos_test.go+++ b/internal/ingestion/component/chunker/token_pdfpos_test.go@@ -37,8 +37,8 @@ func TestMergeByTokenSizeFromJSON_ExtendsPDFPositions(t *testing.T) {posB := json.RawMessage(`[[2,15,25,35,45]]`)items := [][]schema.ChunkDoc{{- {Text: "alpha", DocType: "text", CKType: "text", TKNums: intPtr(5), PDFPositions: posA},- {Text: "beta", DocType: "text", CKType: "text", TKNums: intPtr(5), PDFPositions: posB},+ {Text: "alpha", DocType: "text", TKNums: intPtr(5), PDFPositions: posA},+ {Text: "beta", DocType: "text", TKNums: intPtr(5), PDFPositions: posB},},}got := mergeByTokenSizeFromJSON(items, 128, 0)@@ -62,8 +62,8 @@ func TestMergeByTokenSizeFromJSON_ExtendsPositions(t *testing.T) {posB := json.RawMessage(`[[4,5,6]]`)items := [][]schema.ChunkDoc{{- {Text: "a", DocType: "text", CKType: "text", TKNums: intPtr(5), Positions: posA},- {Text: "b", DocType: "text", CKType: "text", TKNums: intPtr(5), Positions: posB},+ {Text: "a", DocType: "text", TKNums: intPtr(5), Positions: posA},+ {Text: "b", DocType: "text", TKNums: intPtr(5), Positions: posB},},}got := mergeByTokenSizeFromJSON(items, 128, 0)@@ -102,8 +102,8 @@ func TestMergeByTokenSizeFromJSON_PositionsDecodeToMatrix(t *testing.T) {posB := json.RawMessage(`[[6,7,8,9,10]]`)items := [][]schema.ChunkDoc{{- {Text: "a", DocType: "text", CKType: "text", TKNums: intPtr(5), Positions: posA},- {Text: "b", DocType: "text", CKType: "text", TKNums: intPtr(5), Positions: posB},+ {Text: "a", DocType: "text", TKNums: intPtr(5), Positions: posA},+ {Text: "b", DocType: "text", TKNums: intPtr(5), Positions: posB},},}got := mergeByTokenSizeFromJSON(items, 128, 0)@@ -144,9 +144,9 @@ func TestMergeByTokenSizeFromJSON_OverlapPrefixCarriesPrevPositions(t *testing.T// first starts a fresh chunk carrying the WHOLE previous chunk as overlap.items := [][]schema.ChunkDoc{{- {Text: "alpha", DocType: "text", CKType: "text", TKNums: intPtr(5), PDFPositions: posA},- {Text: "beta", DocType: "text", CKType: "text", TKNums: intPtr(5), PDFPositions: posB},- {Text: "gamma", DocType: "text", CKType: "text", TKNums: intPtr(5), PDFPositions: posC},+ {Text: "alpha", DocType: "text", TKNums: intPtr(5), PDFPositions: posA},+ {Text: "beta", DocType: "text", TKNums: intPtr(5), PDFPositions: posB},+ {Text: "gamma", DocType: "text", TKNums: intPtr(5), PDFPositions: posC},},}got := mergeByTokenSizeFromJSON(items, 20, 100)@@ -208,12 +208,12 @@ func TestMergeByTokenSizeFromJSON_PartialOverlapPrefixCarriesOnlyTailPositions(t// actual-count check agrees with the declared TKNums.)items := [][]schema.ChunkDoc{{- {Text: "a", DocType: "text", CKType: "text", TKNums: intPtr(1), PDFPositions: posA},- {Text: "b", DocType: "text", CKType: "text", TKNums: intPtr(1), PDFPositions: posB},- {Text: "c", DocType: "text", CKType: "text", TKNums: intPtr(1), PDFPositions: posC},- {Text: "d", DocType: "text", CKType: "text", TKNums: intPtr(1), PDFPositions: posD},- {Text: "e", DocType: "text", CKType: "text", TKNums: intPtr(1), PDFPositions: posE},- {Text: "f", DocType: "text", CKType: "text", TKNums: intPtr(1), PDFPositions: posF},+ {Text: "a", DocType: "text", TKNums: intPtr(1), PDFPositions: posA},+ {Text: "b", DocType: "text", TKNums: intPtr(1), PDFPositions: posB},+ {Text: "c", DocType: "text", TKNums: intPtr(1), PDFPositions: posC},+ {Text: "d", DocType: "text", TKNums: intPtr(1), PDFPositions: posD},+ {Text: "e", DocType: "text", TKNums: intPtr(1), PDFPositions: posE},+ {Text: "f", DocType: "text", TKNums: intPtr(1), PDFPositions: posF},},}got := mergeByTokenSizeFromJSON(items, 5, 20)@@ -254,7 +254,6 @@ func TestChunkFromItem_SlicesPositionsAcrossDelimiterPieces(t *testing.T) {it := schema.ChunkDoc{Text: "AAAA\nBBBB",DocType: "text",- CKType: "text",PDFPositions: json.RawMessage(`[[1,0,200,0,40]]`),}got := chunkFromItem(it, compileDelimPattern([]string{"`\n`"}), false)@@ -294,7 +293,6 @@ func TestSplitOversizedText_SlicedPositionsTrackPieceTextShare(t *testing.T) {return schema.ChunkDoc{Text: text,DocType: "text",- CKType: "text",TKNums: intPtr(tokenizeStr(text)),PDFPositions: json.RawMessage(`[[1,10,200,50,80]]`),}@@ -358,7 +356,6 @@ func TestChunkFromItem_SlicesPositionsIgnoresPositionTags(t *testing.T) {it := schema.ChunkDoc{Text: "AAAA@@1\t0\t200\t0\t40##\nCCCC",DocType: "text",- CKType: "text",PDFPositions: json.RawMessage(`[[1,0,200,0,40]]`),}got := chunkFromItem(it, compileDelimPattern([]string{"`\n`"}), false)@@ -392,7 +389,6 @@ func TestSplitOversizedText_SlicesPositionsIgnoreTags(t *testing.T) {ck := schema.ChunkDoc{Text: body,DocType: "text",- CKType: "text",TKNums: intPtr(tokenizeStr(body)),PDFPositions: json.RawMessage(`[[1,0,200,0,40]]`),}@@ -461,7 +457,6 @@ func TestMergeByTokenSizeFromJSON_ChainedOverlapNoHeadAccumulation(t *testing.T)row = append(row, schema.ChunkDoc{Text: string(rune('a' + i)),DocType: "text",- CKType: "text",TKNums: intPtr(1),PDFPositions: pos(i + 1),})@@ -601,7 +596,6 @@ func TestMergeByTokenSizeFromJSON_ChainedOverlapPerChunkConvergence(t *testing.Trow = append(row, schema.ChunkDoc{Text: string(rune('a' + i)),DocType: "text",- CKType: "text",TKNums: intPtr(1),PDFPositions: pos(i + 1),})diff --git a/internal/ingestion/component/chunker/token_strict_cap_test.go b/internal/ingestion/component/chunker/token_strict_cap_test.goindex 106b458155..de20ef6af2 100644--- a/internal/ingestion/component/chunker/token_strict_cap_test.go+++ b/internal/ingestion/component/chunker/token_strict_cap_test.go@@ -37,7 +37,7 @@ func TestMergeByTokenSizeFromJSON_HardCapNoOvershoot(t *testing.T) {for i := 0; i < 8; i++ {text := strings.TrimSpace(strings.Repeat("w ", 25))sections = append(sections, schema.ChunkDoc{- Text: text, DocType: "text", CKType: "text", TKNums: intPtr(tokenizeStr(text)),+ Text: text, DocType: "text", TKNums: intPtr(tokenizeStr(text)),})}got := mergeByTokenSizeFromJSON([][]schema.ChunkDoc{sections}, budget, 0)@@ -60,7 +60,7 @@ func TestMergeByTokenSizeFromJSON_OverlapStrictCap(t *testing.T) {for i := 0; i < 20; i++ {text := strings.TrimSpace(strings.Repeat("w ", 10))sections = append(sections, schema.ChunkDoc{- Text: text, DocType: "text", CKType: "text", TKNums: intPtr(tokenizeStr(text)),+ Text: text, DocType: "text", TKNums: intPtr(tokenizeStr(text)),})}got := mergeByTokenSizeFromJSON([][]schema.ChunkDoc{sections}, budget, 20)@@ -78,7 +78,7 @@ func TestMergeByTokenSizeFromJSON_OversizedUnitSplit(t *testing.T) {const budget = 30long := strings.TrimSpace(strings.Repeat("word ", 100))items := [][]schema.ChunkDoc{{- {Text: long, DocType: "text", CKType: "text", TKNums: intPtr(tokenizeStr(long))},+ {Text: long, DocType: "text", TKNums: intPtr(tokenizeStr(long))},}}got := mergeByTokenSizeFromJSON(items, budget, 0)merged := got[0]@@ -235,8 +235,8 @@ func TestInvokeJSONPayload_HardCapEndToEnd(t *testing.T) {// standing alone.func TestMergeUnits_UnderCapBoundaryDecision(t *testing.T) {units := []schema.ChunkDoc{- {Text: "prev text", TKNums: intPtr(10), CKType: "text"},- {Text: "incoming text", TKNums: intPtr(10), CKType: "text"},+ {Text: "prev text", TKNums: intPtr(10)},+ {Text: "incoming text", TKNums: intPtr(10)},}// prev+incoming == target: merge into one chunk.got := mergeUnits(units, 20, 0, "\n")@@ -255,8 +255,8 @@ func TestMergeUnits_UnderCapBoundaryDecision(t *testing.T) {// An oversized incoming unit is expanded (split) instead of standing whole.big := []schema.ChunkDoc{- {Text: "prev text", TKNums: intPtr(10), CKType: "text"},- {Text: strings.Repeat("word ", 100), TKNums: intPtr(100), CKType: "text"},+ {Text: "prev text", TKNums: intPtr(10)},+ {Text: strings.Repeat("word ", 100), TKNums: intPtr(100)},}bigMerged := mergeUnits(big, 30, 0, "\n")if len(bigMerged) < 2 {diff --git a/internal/ingestion/component/chunker/token_test.go b/internal/ingestion/component/chunker/token_test.goindex 27e21e1238..a370a29de3 100644--- a/internal/ingestion/component/chunker/token_test.go+++ b/internal/ingestion/component/chunker/token_test.go@@ -97,8 +97,8 @@ func TestTokenChunkerPreservesSpreadsheetSegmentBoundaries(t *testing.T) {t.Fatalf("segment chunks = %#v", chunks)}for i, chunk := range chunks {- if chunk["ck_type"] != "table" {- t.Errorf("chunk[%d] ck_type = %v, want table", i, chunk["ck_type"])+ if chunk["doc_type_kwd"] != "table" {+ t.Errorf("chunk[%d] doc_type_kwd = %v, want table", i, chunk["doc_type_kwd"])}}}@@ -405,9 +405,8 @@ func TestTokenChunker_InvokeJSONPayload_KeepsNonTextStandalone(t *testing.T) {if got != wantTypes[i] {t.Errorf("chunk %d: doc_type_kwd = %q, want %q (full chunk: %+v)", i, got, wantTypes[i], ch)}- ckType, _ := ch["ck_type"].(string)- if ckType != wantTypes[i] {- t.Errorf("chunk %d: ck_type = %q, want %q (derived from doc_type_kwd)", i, ckType, wantTypes[i])+ if _, ok := ch["ck_type"]; ok {+ t.Errorf("chunk %d must not emit the retired ck_type: %+v", i, ch)}}// The two text segments on either side of the table/image must remain@@ -473,7 +472,7 @@ func TestTokenChunkerMediaContextSpansUpstreamItems(t *testing.T) {chunks := outputChunks(t, out)var media map[string]anyfor _, ck := range chunks {- if ck["ck_type"] == tc.mediaType {+ if ck["doc_type_kwd"] == tc.mediaType {media = ckbreak}@@ -548,7 +547,7 @@ func TestTokenChunkerDelimiterWindowUsesChildTokenCounts(t *testing.T) {}var media map[string]anyfor _, ck := range outputChunks(t, out) {- if ck["ck_type"] == "table" {+ if ck["doc_type_kwd"] == "table" {media = ck}}@@ -569,7 +568,6 @@ func TestMaterializeMediaContextKeepsTokenCountInSync(t *testing.T) {ck := schema.ChunkDoc{Text: body,DocType: "table",- CKType: "table",TKNums: intPtr(tokenizeStr(body)),ContextAbove: "above",ContextBelow: "below",@@ -611,7 +609,7 @@ func TestTokenChunkerWindowTruncatesOnSentenceBoundary(t *testing.T) {}var media map[string]anyfor _, ck := range outputChunks(t, out) {- if ck["ck_type"] == "table" {+ if ck["doc_type_kwd"] == "table" {media = ck}}@@ -712,8 +710,8 @@ func TestTokenChunker_SpreadsheetTablePreservesSelectionRange(t *testing.T) {if len(chunks) != 1 {t.Fatalf("chunks = %d, want one unsplit table item", len(chunks))}- if got := chunks[0]["ck_type"]; got != "table" {- t.Errorf("ck_type = %v, want table", got)+ if got := chunks[0]["doc_type_kwd"]; got != "table" {+ t.Errorf("doc_type_kwd = %v, want table", got)}if got := chunks[0]["positions"]; !reflect.DeepEqual(got, wantPositions) {t.Errorf("positions = %#v, want unchanged %#v", got, wantPositions)@@ -1103,7 +1101,6 @@ func TestSplitByChildrenRecomputesTokenCounts(t *testing.T) {children := splitByChildren([]schema.ChunkDoc{{Text: parentText,DocType: "text",- CKType: "text",TKNums: intPtr(tokenizeStr(parentText)),}}, regexp.MustCompile(`\. `), true)@@ -1125,7 +1122,6 @@ func TestApplyChildrenDelimText_RecomputesTokenCounts(t *testing.T) {out := applyChildrenDelimText([]schema.ChunkDoc{{Text: parentText,DocType: "text",- CKType: "text",TKNums: intPtr(tokenizeStr(parentText)),}}, regexp.MustCompile(`\. `), true)diff --git a/internal/ingestion/component/knowledge_compiler/component_test.go b/internal/ingestion/component/knowledge_compiler/component_test.goindex bc27a695a4..724fb2b0ad 100644--- a/internal/ingestion/component/knowledge_compiler/component_test.go+++ b/internal/ingestion/component/knowledge_compiler/component_test.go@@ -1415,7 +1415,7 @@ func TestOverlayTemplateConfigPlanPrecedence(t *testing.T) {// TestKnowledgeCompiler_BuildInputsAcceptsMapSliceChunks locks the chunk-carrier// contract: the upstream chunker hands chunks over as []map[string]any (see the-// chunk map shape {"id","text","ck_type","doc_type_kwd","tk_nums"} observed from+// chunk map shape {"id","text","doc_type_kwd","tk_nums"} observed from// the running pipeline), not as []any of map. buildInputs must accept both// shapes; a strict []any assertion alone would silently drop every chunk and// leave the knowledge compiler with empty input (compiling nothing yet still@@ -1423,8 +1423,8 @@ func TestOverlayTemplateConfigPlanPrecedence(t *testing.T) {func TestKnowledgeCompiler_BuildInputsAcceptsMapSliceChunks(t *testing.T) {in, err := buildInputs(map[string]any{"chunks": []map[string]any{- {"id": "c1", "text": "《三国演义》", "ck_type": "text"},- {"id": "c2", "text": "滚滚长江东逝水", "ck_type": "text"},+ {"id": "c1", "text": "《三国演义》"},+ {"id": "c2", "text": "滚滚长江东逝水"},},}, common.Param{})if err != nil {diff --git a/internal/ingestion/component/parser_dispatch_test.go b/internal/ingestion/component/parser_dispatch_test.goindex 8cbfbf7e41..203ee4e489 100644--- a/internal/ingestion/component/parser_dispatch_test.go+++ b/internal/ingestion/component/parser_dispatch_test.go@@ -133,8 +133,8 @@ func TestBuildParserOutputsNormalizesTextFormatsToJSON(t *testing.T) {if len(items) != 2 {t.Fatalf("markdown json item count = %d, want 2: %#v", len(items), items)}- if got := items[0]["ck_type"]; got != "heading" {- t.Errorf("markdown item[0].ck_type = %v, want heading", got)+ if got := items[0]["layout"]; got != "heading" {+ t.Errorf("markdown item[0].layout = %v, want heading", got)}if got := items[0]["text"]; got != "# Title" {t.Errorf("markdown item[0].text = %v, want # Title", got)diff --git a/internal/ingestion/component/pdf_vision_dispatch.go b/internal/ingestion/component/pdf_vision_dispatch.goindex a607da0a2a..3ddb05df39 100644--- a/internal/ingestion/component/pdf_vision_dispatch.go+++ b/internal/ingestion/component/pdf_vision_dispatch.go@@ -883,8 +883,19 @@ func mineruContentItemToJSON(item map[string]any, images map[string]*zip.File) mtext := mineruItemText(item, typ)imageURI := ""docType := "text"- layout := typ+ // External MinerU labels are mapped explicitly. Unknown labels must not+ // acquire parser block-kind semantics implicitly; list/code are real+ // structural types and stay structural.+ layout := "text"switch typ {+ case "title":+ layout = "title"+ case "list":+ layout = "list"+ case "code":+ layout = "code"+ case "equation":+ layout = "equation"case "image", "chart":docType = "image"layout = "figure"diff --git a/internal/ingestion/component/pdf_vision_dispatch_test.go b/internal/ingestion/component/pdf_vision_dispatch_test.goindex faf45f7ad2..df652a8839 100644--- a/internal/ingestion/component/pdf_vision_dispatch_test.go+++ b/internal/ingestion/component/pdf_vision_dispatch_test.go@@ -217,6 +217,36 @@ func TestDispatchPaddleOCRPDFRejectsMissingBlocks(t *testing.T) {}}+func TestMinerUContentItemToJSONExplicitLayoutMapping(t *testing.T) {+ for _, tc := range []struct {+ name string+ item map[string]any+ wantLayout string+ wantDocType string+ }{+ {"list", map[string]any{"type": "list", "list_items": []any{"a", "b"}}, "list", "text"},+ {"code", map[string]any{"type": "code", "code_body": "x"}, "code", "text"},+ {"title", map[string]any{"type": "title", "text": "T"}, "title", "text"},+ {"equation", map[string]any{"type": "equation", "text": "x=1"}, "equation", "text"},+ {"unknown", map[string]any{"type": "footnote", "text": "x"}, "text", "text"},+ {"image", map[string]any{"type": "image", "image_caption": []any{"cap"}}, "figure", "image"},+ {"table", map[string]any{"type": "table", "table_body": "<table></table>"}, "table", "table"},+ } {+ t.Run(tc.name, func(t *testing.T) {+ got := mineruContentItemToJSON(tc.item, nil)+ if got == nil {+ t.Fatal("item was dropped")+ }+ if got["layout"] != tc.wantLayout {+ t.Errorf("layout = %v, want %q", got["layout"], tc.wantLayout)+ }+ if got["doc_type_kwd"] != tc.wantDocType {+ t.Errorf("doc_type_kwd = %v, want %q", got["doc_type_kwd"], tc.wantDocType)+ }+ })+ }+}+func TestMinerUExtractJSONItemsKeepsImageAndPositions(t *testing.T) {var buf bytes.Bufferzw := zip.NewWriter(&buf)diff --git a/internal/ingestion/component/schema/chunk_docitem_test.go b/internal/ingestion/component/schema/chunk_docitem_test.goindex fcb37dded4..a678bc720b 100644--- a/internal/ingestion/component/schema/chunk_docitem_test.go+++ b/internal/ingestion/component/schema/chunk_docitem_test.go@@ -40,13 +40,13 @@ func TestChunkDocsFromDocItem_ParallelsMapPath(t *testing.T) {{name: "text with extra passthrough",items: []parser.DocItem{- parser.TextItem{Text: "hello", CKType: "heading", Extra: map[string]any{"layout": "1", "page_number": 3}},+ parser.TextItem{Text: "hello", Extra: map[string]any{"layout": "1", "page_number": 3}},},},{name: "image with extra passthrough",items: []parser.DocItem{- parser.ImageItem{Text: "alt", Image: "data:image/png;base64,xxx", CKType: "image", Extra: map[string]any{"page_number": 2}},+ parser.ImageItem{Text: "alt", Image: "data:image/png;base64,xxx", Extra: map[string]any{"page_number": 2}},},},{@@ -66,7 +66,7 @@ func TestChunkDocsFromDocItem_ParallelsMapPath(t *testing.T) {items: []parser.DocItem{parser.TextItem{Text: "a"},parser.RawItem{M: map[string]any{"text": "b", "doc_type_kwd": "text"}},- parser.ImageItem{Image: "i", CKType: "image"},+ parser.ImageItem{Image: "i"},},},// A canonical key present in Extra must NOT shadow the typed field when@@ -83,7 +83,7 @@ func TestChunkDocsFromDocItem_ParallelsMapPath(t *testing.T) {{name: "empty image text must not be shadowed by extra text",items: []parser.DocItem{- parser.ImageItem{Text: "", Image: "i", CKType: "image", Extra: map[string]any{"text": "stale"}},+ parser.ImageItem{Text: "", Image: "i", Extra: map[string]any{"text": "stale"}},},},}@@ -149,7 +149,7 @@ func TestChunkDocsFromDocItem_SkipsNilElement(t *testing.T) {items := []parser.DocItem{parser.TextItem{Text: "a"},nil,- parser.ImageItem{Image: "i", CKType: "image"},+ parser.ImageItem{Image: "i"},}got, found, err := ChunkDocsFromDocItem(items)if err != nil {diff --git a/internal/ingestion/component/schema/chunk_types.go b/internal/ingestion/component/schema/chunk_types.goindex dce014701a..cf2de2738b 100644--- a/internal/ingestion/component/schema/chunk_types.go+++ b/internal/ingestion/component/schema/chunk_types.go@@ -106,7 +106,6 @@ func (m ChunkerFileMeta) MarshalJSON() ([]byte, error) {type ChunkDoc struct {Text string `json:"text,omitempty"`DocType string `json:"doc_type_kwd,omitempty"`- CKType string `json:"ck_type,omitempty"`TKNums *int `json:"tk_nums,omitempty"`Mom string `json:"mom,omitempty"`ImgID string `json:"img_id,omitempty"`@@ -156,6 +155,8 @@ func (d *ChunkDoc) UnmarshalJSON(data []byte) error {// tagger emits tag_feas and the vocabulary comes from a tag source file,// so no chunk carries tag_kwd. Deleted rather than left in raw so it// cannot fall through to Extra and be re-emitted by MarshalJSON.+ // ck_type is a retired parser key: it has no field, but must stay in this+ // list so an old payload cannot leak it back through Extra.for _, key := range []string{"text", "doc_type_kwd", "mom", "img_id","ck_type", "tk_nums", "layout", "layout_type", "layoutno", "image",@@ -184,6 +185,10 @@ func (d ChunkDoc) MarshalJSON() ([]byte, error) {return nil, err}for k, v := range d.Extra {+ // ck_type is retired; a stray Extra key must not resurrect it.+ if k == "ck_type" {+ continue+ }if _, exists := raw[k]; !exists {raw[k] = v}@@ -294,6 +299,9 @@ func (d ChunkDoc) ToMap() map[string]any {return map[string]any{}}for k, raw := range d.Extra {+ if k == "ck_type" {+ continue+ }out[k] = decodeExtraValue(raw)}// Every pre-index chunk has one canonical text field. Keep the key evendiff --git a/internal/ingestion/component/schema/schema_test.go b/internal/ingestion/component/schema/schema_test.goindex dd53b20ced..862f438d81 100644--- a/internal/ingestion/component/schema/schema_test.go+++ b/internal/ingestion/component/schema/schema_test.go@@ -241,7 +241,6 @@ func TestChunkDocSpreadsheetFieldsRoundTrip(t *testing.T) {original := ChunkDoc{Text: "<table><tr><th>ID</th></tr></table>",DocType: "table",- CKType: "table",Sheet: "Orders",SheetIndex: &sheetIndex,}@@ -278,6 +277,32 @@ func TestChunkDocLegacyRowIRKeysPassThrough(t *testing.T) {t.Errorf("legacy key %q lost in round trip: %#v", key, out)}}+ if _, ok := out["ck_type"]; ok {+ t.Errorf("retired ck_type must not survive the round trip: %#v", out)+ }+}++// A programmatic Extra entry must not resurrect the retired ck_type key on+// either projection (MarshalJSON or ToMap).+func TestChunkDocRetiredCKTypeCannotBeResurrectedViaExtra(t *testing.T) {+ d := ChunkDoc{+ Text: "body",+ Extra: map[string]json.RawMessage{"ck_type": json.RawMessage(`"heading"`)},+ }+ if _, ok := d.ToMap()["ck_type"]; ok {+ t.Fatalf("ToMap resurrected ck_type: %#v", d.ToMap())+ }+ data, err := json.Marshal(d)+ if err != nil {+ t.Fatalf("marshal: %v", err)+ }+ var raw map[string]any+ if err := json.Unmarshal(data, &raw); err != nil {+ t.Fatalf("unmarshal: %v", err)+ }+ if _, ok := raw["ck_type"]; ok {+ t.Fatalf("MarshalJSON resurrected ck_type: %s", data)+ }}func TestChunkDocVisualParentMetadataRoundTrip(t *testing.T) {diff --git a/internal/ingestion/component/tokenizer_unit_test.go b/internal/ingestion/component/tokenizer_unit_test.goindex 2fcbd4b473..2db6eeb52b 100644--- a/internal/ingestion/component/tokenizer_unit_test.go+++ b/internal/ingestion/component/tokenizer_unit_test.go@@ -436,7 +436,7 @@ func TestValidateTokenizerOutputs_SymbolOnlyContentLtksIsEmptyFails(t *testing.Tfunc TestChunkDocsToMaps_PreservesPDFPositions(t *testing.T) {pos := json.RawMessage(`[[1,10,20,30,40],[2,15,25,35,45]]`)chunks := []schema.ChunkDoc{- {Text: "PDF paragraph", DocType: "text", CKType: "text",+ {Text: "PDF paragraph", DocType: "text",Positions: pos, PDFPositions: pos},}maps := schema.ChunkDocsToMaps(chunks)diff --git a/internal/ingestion/task/indexdoc/process.go b/internal/ingestion/task/indexdoc/process.goindex 78684dedbb..1415fe7476 100644--- a/internal/ingestion/task/indexdoc/process.go+++ b/internal/ingestion/task/indexdoc/process.go@@ -144,16 +144,19 @@ func cleanupConsumedChunkFields(ck map[string]any) {}// pipelineOnlyFields are the parser/chunker BOOKKEEPING keys: each one is-// consumed inside the pipeline (ck_type drives chunk merging and the image-crop-// decision, tk_nums carries the chunker's token count into the Tokenizer,-// layout*/image/context_* describe the media block, and page_number/table_id/-// sheet/headers/cells describe the table or spreadsheet block) and NONE of them-// is a chunk-store column.+// consumed inside the pipeline (layout drives block-kind decisions, tk_nums+// carries the chunker's token count into the Tokenizer, layout*/image/context_*+// describe the media block, and page_number/table_id/sheet/headers/cells+// describe the table or spreadsheet block) and NONE of them is a chunk-store+// column.//// The Python index doc carries none of them — its chunk builder emits only the// persist fields — but Go's chunker hands them on, and the write boundary is// where they stop: Elasticsearch and Vastbase would add a field for each key,// and Infinity would drop each key at insert with a warning.+//+// ck_type is retired but stays here as a tombstone: a legacy producer can still+// hand it on, and no engine has a column for it.var pipelineOnlyFields = []string{"ck_type", "tk_nums", "layout", "layout_type", "layoutno", "image","context_above", "context_below", "page_number", "_preserve_order",diff --git a/internal/ingestion/task/indexdoc/process_test.go b/internal/ingestion/task/indexdoc/process_test.goindex ddedcb0bf2..31635a7cf8 100644--- a/internal/ingestion/task/indexdoc/process_test.go+++ b/internal/ingestion/task/indexdoc/process_test.go@@ -664,9 +664,9 @@ func TestProcessChunkPositions_ChunkerRoundTrip(t *testing.T) {// normalizes it to [][]float64 at the chunker boundary too, so the parser// container type never reaches the indexer.pdfItem := parser.ImageItem{- Text: "fig",- Image: "",- CKType: "image",+ Text: "fig",+ Image: "",+Positions: parser.Positions{parser.AbsoluteBox{PageNumber: 1, Left: 100, Right: 50, Top: 200, Bottom: 150},},diff --git a/internal/parser/parser/csv_parser_test.go b/internal/parser/parser/csv_parser_test.goindex 24fa1be83f..33fd994dec 100644--- a/internal/parser/parser/csv_parser_test.go+++ b/internal/parser/parser/csv_parser_test.go@@ -27,8 +27,8 @@ func TestCSVParser_EmitsSpreadsheetRows(t *testing.T) {t.Fatalf("len(res.JSON) = %d, want one HTML table item", len(res.JSON))}item := res.JSON[0].ToMap()- if item["ck_type"] != "table" || item["doc_type_kwd"] != "table" {- t.Fatalf("item types = %v/%v, want table/table", item["ck_type"], item["doc_type_kwd"])+ if item["doc_type_kwd"] != "table" {+ t.Fatalf("item doc_type_kwd = %v, want table", item["doc_type_kwd"])}wantText := "<table><caption>Data</caption>\n" +"<tr><th>Name</th><th>Age</th><th>City</th></tr>\n" +@@ -91,7 +91,7 @@ func TestCSVParserHTML4ExcelIsIgnored(t *testing.T) {if res.Err != nil {t.Fatalf("ParseWithResult failed: %v", res.Err)}- if len(res.JSON) != 1 || res.JSON[0].ToMap()["ck_type"] != "table" {+ if len(res.JSON) != 1 || res.JSON[0].ToMap()["doc_type_kwd"] != "table" {t.Fatalf("items = %#v, want one table item", res.JSON)}if !reflect.DeepEqual(res.JSON, plainRes.JSON) {@@ -117,8 +117,8 @@ func TestCSVParser_ConfigureOutputFormatJSON(t *testing.T) {if len(res.JSON) != 1 {t.Fatalf("len(res.JSON) = %d, want one HTML table item", len(res.JSON))}- if res.JSON[0].ToMap()["ck_type"] != "table" {- t.Fatalf("ck_type = %v, want table", res.JSON[0].ToMap()["ck_type"])+ if res.JSON[0].ToMap()["doc_type_kwd"] != "table" {+ t.Fatalf("doc_type_kwd = %v, want table", res.JSON[0].ToMap()["doc_type_kwd"])}if positions := res.JSON[0].ToMap()["positions"].([][]float64); len(positions) != 2 {t.Fatalf("len(positions) = %d, want one tuple per <tr>", len(positions))diff --git a/internal/parser/parser/doc_item.go b/internal/parser/parser/doc_item.goindex 9e26ebe55e..fa7adf08a3 100644--- a/internal/parser/parser/doc_item.go+++ b/internal/parser/parser/doc_item.go@@ -39,13 +39,11 @@ type DocItem interface {ToMap() map[string]any}-// TextItem is a body/heading/list/code span. CKType optionally carries the-// legacy ck_type discriminator (heading/list/code/...); it is omitted from-// ToMap when empty. Extra carries layout/page_number/level/... passthrough.+// TextItem is a body/heading/list/code span. Extra carries+// layout/page_number/level/... passthrough.type TextItem struct {- Text string- CKType string- Extra map[string]any+ Text string+ Extra map[string]any}func (t TextItem) ToMap() map[string]any {@@ -53,20 +51,14 @@ func (t TextItem) ToMap() map[string]any {"text": t.Text,DocTypeKey: DocTypeText,}- if t.CKType != "" {- m["ck_type"] = t.CKType- }return mergeExtra(m, t.Extra)}-// TableItem carries a parsed TableData. CKType is the legacy ck_type-// discriminator (always "table" today, mirroring NewTableDataJSONItem); it is-// emitted for pipeline compatibility and will be dropped in a later pass.+// TableItem carries a parsed TableData.type TableItem struct {Table *entity.TableDataSheet stringPositions Positions- CKType stringExtra map[string]any}@@ -74,7 +66,6 @@ func (t TableItem) ToMap() map[string]any {m := map[string]any{"table": t.Table,DocTypeKey: DocTypeTable,- "ck_type": firstNonEmptyStr(t.CKType, DocTypeTable),}if t.Sheet != "" {m["sheet"] = t.Sheet@@ -86,15 +77,12 @@ func (t TableItem) ToMap() map[string]any {}// ImageItem is an embedded image. Text is the alt text, Image the data-// URL/base64 payload (possibly empty when omitted). CKType carries the legacy-// ck_type discriminator ("image" for xlsx/pptx/html/markdown, empty for docx);-// it is omitted from ToMap when empty.+// URL/base64 payload (possibly empty when omitted).type ImageItem struct {Text stringImage stringSheet stringPositions Positions- CKType stringExtra map[string]any}@@ -104,9 +92,6 @@ func (i ImageItem) ToMap() map[string]any {"image": i.Image,DocTypeKey: DocTypeImage,}- if i.CKType != "" {- m["ck_type"] = i.CKType- }if i.Sheet != "" {m["sheet"] = i.Sheet}@@ -188,9 +173,8 @@ func DocItemFromMap(m map[string]any) DocItem {func textItemFromMap(m map[string]any) DocItem {return TextItem{- Text: stringField(m, "text"),- CKType: stringField(m, "ck_type"),- Extra: extraFromMap(m, "text", DocTypeKey, "ck_type"),+ Text: stringField(m, "text"),+ Extra: extraFromMap(m, "text", DocTypeKey, "ck_type"),}}@@ -207,7 +191,6 @@ func imageItemFromMap(m map[string]any) DocItem {return ImageItem{Text: stringField(m, "text"),Image: stringField(m, "image"),- CKType: stringField(m, "ck_type"),Sheet: stringField(m, "sheet"),Positions: pos,Extra: extraFromMap(m, skip...),@@ -233,7 +216,6 @@ func tableItemFromMap(m map[string]any) DocItem {Table: td,Sheet: stringField(m, "sheet"),Positions: pos,- CKType: stringField(m, "ck_type"),Extra: extraFromMap(m, skip...),}}@@ -287,6 +269,10 @@ func ItemsToMaps(items []DocItem) []map[string]any {// present, so a stray Extra key can never shadow text/table/image/doc_type_kwd.func mergeExtra(m, extra map[string]any) map[string]any {for k, v := range extra {+ // ck_type is retired; a stray Extra key must not resurrect it.+ if k == "ck_type" {+ continue+ }if _, ok := m[k]; !ok {m[k] = v}@@ -294,15 +280,6 @@ func mergeExtra(m, extra map[string]any) map[string]any {return m}-func firstNonEmptyStr(values ...string) string {- for _, v := range values {- if v != "" {- return v- }- }- return ""-}-// ---------------------------------------------------------------------------// Position: strong-typed replacement for the loosely-typed `positions` wire// field. The single wire shape is a 5-tuple array, but it carries TWOdiff --git a/internal/parser/parser/doc_item_test.go b/internal/parser/parser/doc_item_test.goindex 500c405942..59689f994f 100644--- a/internal/parser/parser/doc_item_test.go+++ b/internal/parser/parser/doc_item_test.go@@ -34,18 +34,10 @@ func TestTextItem_ToMap(t *testing.T) {mustMap(t, map[string]any{"text": "", DocTypeKey: DocTypeText},TextItem{}.ToMap())})- t.Run("with text and no ck_type", func(t *testing.T) {+ t.Run("with text", func(t *testing.T) {mustMap(t, map[string]any{"text": "body", DocTypeKey: DocTypeText},TextItem{Text: "body"}.ToMap())})- t.Run("ck_type emitted only when non-empty", func(t *testing.T) {- mustMap(t, map[string]any{"text": "h", DocTypeKey: DocTypeText, "ck_type": "heading"},- TextItem{Text: "h", CKType: "heading"}.ToMap())- // empty CKType must NOT appear as a key.- if _, ok := (TextItem{Text: "h"}).ToMap()["ck_type"]; ok {- t.Error("ck_type present despite empty CKType")- }- })t.Run("extra passthrough", func(t *testing.T) {mustMap(t, map[string]any{"text": "x",@@ -60,21 +52,19 @@ func TestTextItem_ToMap(t *testing.T) {}func TestTextItem_ExtraDoesNotClobberCanonical(t *testing.T) {- // An attacker/stray Extra key must never shadow the typed fields.+ // An attacker/stray Extra key must never shadow the typed fields, and a+ // retired ck_type must not be resurrected through Extra.got := TextItem{- Text: "real",- CKType: "heading",+ Text: "real",Extra: map[string]any{"text": "evil",DocTypeKey: "evil",- "ck_type": "evil","layout": "kept", // not a canonical key -> passes through},}.ToMap()mustMap(t, map[string]any{"text": "real",DocTypeKey: DocTypeText,- "ck_type": "heading","layout": "kept",}, got)}@@ -84,25 +74,15 @@ func TestTableItem_ToMap(t *testing.T) {mustMap(t, map[string]any{"table": (*entity.TableData)(nil),DocTypeKey: DocTypeTable,- "ck_type": DocTypeTable,}, TableItem{}.ToMap())})- t.Run("ck_type falls back to table when empty", func(t *testing.T) {- mustMap(t, map[string]any{- "table": (*entity.TableData)(nil),- DocTypeKey: DocTypeTable,- "ck_type": DocTypeTable,- }, TableItem{CKType: ""}.ToMap())- })- t.Run("ck_type explicit wins", func(t *testing.T) {+ t.Run("sheet and positions", func(t *testing.T) {mustMap(t, map[string]any{"table": (*entity.TableData)(nil),DocTypeKey: DocTypeTable,- "ck_type": "table","sheet": "Sheet1","positions": [][]float64{{0, 1, 1, 2, 2}},}, TableItem{- CKType: "table",Sheet: "Sheet1",Positions: SpreadsheetPositions([][]float64{{0, 1, 1, 2, 2}}),}.ToMap())@@ -119,30 +99,25 @@ func TestTableItem_ToMap(t *testing.T) {}func TestImageItem_ToMap(t *testing.T) {- t.Run("docx-like: no ck_type, empty image", func(t *testing.T) {- // docx image items historically set doc_type_kwd:"image" with no- // ck_type and an empty image when over budget.+ t.Run("docx-like: empty image", func(t *testing.T) {+ // docx image items historically set doc_type_kwd:"image" with an empty+ // image when over budget.mustMap(t, map[string]any{"text": "","image": "",DocTypeKey: DocTypeImage,}, ImageItem{}.ToMap())- if _, ok := (ImageItem{}).ToMap()["ck_type"]; ok {- t.Error("ck_type present despite empty CKType")- }})- t.Run("xlsx-like: ck_type image + sheet + positions", func(t *testing.T) {+ t.Run("xlsx-like: sheet + positions", func(t *testing.T) {mustMap(t, map[string]any{"text": "alt","image": "data:image/png;base64,xxx",DocTypeKey: DocTypeImage,- "ck_type": "image","sheet": "Sheet2","positions": [][]float64{{1, 0, 0, 3, 4}},}, ImageItem{Text: "alt",Image: "data:image/png;base64,xxx",- CKType: "image",Sheet: "Sheet2",Positions: SpreadsheetPositions([][]float64{{1, 0, 0, 3, 4}}),}.ToMap())@@ -166,7 +141,6 @@ func TestRawItem_Passthrough(t *testing.T) {in := map[string]any{"text": "hello",DocTypeKey: DocTypeText,- "ck_type": "heading","layout": "title","arbitrary": []int{1, 2, 3},"nested": map[string]any{"k": "v"},@@ -222,7 +196,6 @@ func TestConstructorWireShapeRegression(t *testing.T) {mustMap(t, map[string]any{"table": td,DocTypeKey: DocTypeTable,- "ck_type": DocTypeTable,"sheet": "S","positions": [][]float64{{0, 0, 0, 1, 1}},}, NewTableDataJSONItem(td, "S", SpreadsheetPositions([][]float64{{0, 0, 0, 1, 1}})).ToMap())@@ -253,7 +226,6 @@ func TestDocItemFromMap_WireShapePreserved(t *testing.T) {textMap := map[string]any{"text": "hello",DocTypeKey: DocTypeText,- "ck_type": "heading","layout": "1","page_number": 3,}@@ -261,13 +233,11 @@ func TestDocItemFromMap_WireShapePreserved(t *testing.T) {"text": "alt","image": "data:image/png;base64,xxx",DocTypeKey: DocTypeImage,- "ck_type": "image","page_number": 2,}tableMap := map[string]any{"table": &entity.TableData{},DocTypeKey: DocTypeTable,- "ck_type": DocTypeTable,"sheet": "s","positions": [][]float64{{1, 2}},"sheet_index": 0,@@ -281,13 +251,11 @@ func TestDocItemFromMap_WireShapePreserved(t *testing.T) {"text": "alt","image": "data:image/png;base64,xxx",DocTypeKey: DocTypeImage,- "ck_type": "image","positions": [][]any{{1, 2, 3, 4}},}tableMapAny := map[string]any{"table": &entity.TableData{},DocTypeKey: DocTypeTable,- "ck_type": DocTypeTable,"positions": [][]any{{1, 2, 3, 4}},}// A table whose "table" value is a JSON-decoded map[string]any (not a typed@@ -300,7 +268,6 @@ func TestDocItemFromMap_WireShapePreserved(t *testing.T) {tableMapJSONDecoded := map[string]any{"table": map[string]any{"grid": []any{}, "header_rows": float64(1)},DocTypeKey: DocTypeTable,- "ck_type": DocTypeTable,}for _, m := range []map[string]any{textMap, imageMap, tableMap, opaqueMap, imageMapAny, tableMapAny, tableMapJSONDecoded} {@@ -325,11 +292,29 @@ func TestDocItemFromMap_WireShapePreserved(t *testing.T) {}}+// A legacy payload may still carry the retired ck_type key. Typed conversion+// must drop it rather than route it through Extra and re-emit it.+func TestDocItemFromMap_DropsRetiredCKType(t *testing.T) {+ legacy := map[string]any{+ "text": "h",+ DocTypeKey: DocTypeText,+ "ck_type": "heading",+ "layout": "heading",+ }+ got := DocItemFromMap(legacy).ToMap()+ if _, ok := got["ck_type"]; ok {+ t.Fatalf("retired ck_type leaked through ToMap: %#v", got)+ }+ if got["layout"] != "heading" {+ t.Fatalf("layout lost during conversion: %#v", got)+ }+}+// DocItemsFromMaps maps a []map[string]any to typed DocItems the same way// DocItemFromMap does, preserving order and skipping nothing.func TestMapsToItems_OrderAndType(t *testing.T) {maps := []map[string]any{- {"text": "a", DocTypeKey: DocTypeText, "ck_type": "text"},+ {"text": "a", DocTypeKey: DocTypeText},{"foo": "bar"},}items := MapsToItems(maps)diff --git a/internal/parser/parser/docx_ir.go b/internal/parser/parser/docx_ir.goindex 77ab144e32..3bb904be0a 100644--- a/internal/parser/parser/docx_ir.go+++ b/internal/parser/parser/docx_ir.go@@ -599,7 +599,7 @@ func appendDOCXParagraphSections(sections []map[string]any, el docxIRElement, he"doc_type_kwd": "text",}if heading {- item["ck_type"] = "heading"+ item["layout"] = "heading"}sections = append(sections, item)}diff --git a/internal/parser/parser/docx_ir_test.go b/internal/parser/parser/docx_ir_test.goindex 23037c6807..d26229c900 100644--- a/internal/parser/parser/docx_ir_test.go+++ b/internal/parser/parser/docx_ir_test.go@@ -29,7 +29,7 @@ func TestBuildDOCXJSONSections_FromJSON(t *testing.T) {got := buildDOCXJSONSections(irJSON, newEmbeddedMediaBudget())want := []map[string]any{- {"text": "Title", "image": nil, "doc_type_kwd": "text", "ck_type": "heading"},+ {"text": "Title", "image": nil, "doc_type_kwd": "text", "layout": "heading"},{"text": "Hello", "image": nil, "doc_type_kwd": "text"},{"text": "", "image": "aGVsbG8=", "doc_type_kwd": "image"},// The table sits under heading "Title", so it carries thediff --git a/internal/parser/parser/docx_parser_test.go b/internal/parser/parser/docx_parser_test.goindex fe4de3724e..d724bf05aa 100644--- a/internal/parser/parser/docx_parser_test.go+++ b/internal/parser/parser/docx_parser_test.go@@ -59,8 +59,8 @@ func TestBuildDOCXJSONSections_Headings(t *testing.T) {if got, ok := item["doc_type_kwd"].(string); !ok || got != "text" {t.Errorf("doc_type_kwd = %q, want %q", got, "text")}- if got, ok := item["ck_type"].(string); !ok || got != "heading" {- t.Errorf("ck_type = %q, want %q", got, "heading")+ if got, ok := item["layout"].(string); !ok || got != "heading" {+ t.Errorf("layout = %q, want %q", got, "heading")}}@@ -147,8 +147,8 @@ func TestBuildDOCXJSONSections_MixedContent(t *testing.T) {if got, _ := sections[3]["doc_type_kwd"].(string); got != "text" {t.Errorf("item[3].doc_type_kwd = %q, want %q", got, "text")}- if got, _ := sections[3]["ck_type"].(string); got != "heading" {- t.Errorf("item[3].ck_type = %q, want %q", got, "heading")+ if got, _ := sections[3]["layout"].(string); got != "heading" {+ t.Errorf("item[3].layout = %q, want %q", got, "heading")}}diff --git a/internal/parser/parser/html_parser.go b/internal/parser/parser/html_parser.goindex 375d61eb0f..a9ebf37524 100644--- a/internal/parser/parser/html_parser.go+++ b/internal/parser/parser/html_parser.go@@ -168,17 +168,17 @@ func walkHTMLBlocksWithState(root *html.Node, out *[]map[string]any, state *html// Emit the <table> as a SINGLE structured doc_type_kwd:"table"// item, in document order. Keeping the full <table>…</table>// markup (not flattened) preserves row/column structure for- // embedding, retrieval, and LLM rendering, and doc_type_kwd/- // ck_type drives downstream table handling (discrete chunk ++ // embedding, retrieval, and LLM rendering, and doc_type_kwd+ // drives downstream table handling (discrete chunk +// table context). We emit ONLY this item — no duplicate// doc_type_kwd:"text" copy — so the table is embedded once and// its markup does not pollute neighbouring prose chunks.emitHTMLTable(child, out, state)continue}- ckType := htmlTagToCkType(tag)+ layout := htmlTagToLayout(tag)trim := tag != "pre" && tag != "textarea"- htmlLeafTextWithState(child, out, ckType, trim, state)+ htmlLeafTextWithState(child, out, layout, trim, state)}}@@ -220,24 +220,26 @@ func isHTMLInlineElement(n *html.Node) bool {return false}-func appendHTMLTextItem(out *[]map[string]any, text, ckType string, trim bool) {+func appendHTMLTextItem(out *[]map[string]any, text, layout string, trim bool) {if trim {text = strings.TrimSpace(text)}if text == "" {return}- *out = append(*out, map[string]any{+ item := map[string]any{"text": text,"doc_type_kwd": "text",- "ck_type": ckType,- })+ }+ if layout != "" {+ item["layout"] = layout+ }+ *out = append(*out, item)}-// htmlTagToCkType maps HTML block tags to the python `ck_type`-// vocabulary used downstream by TitleChunker and similar-// components. Tags not in the map fall back to "text".-func htmlTagToCkType(tag string) string {+// htmlTagToLayout maps an HTML block tag to its structural layout value.+// Media tags are handled by their own emitters, so they have no entry here.+func htmlTagToLayout(tag string) string {switch tag {case "h1", "h2", "h3", "h4", "h5", "h6":return "heading"@@ -247,14 +249,10 @@ func htmlTagToCkType(tag string) string {return "list"case "pre", "code":return "code"- case "table", "tr", "td", "th":- return "table"case "blockquote":return "quote"- case "img":- return "image"}- return "text"+ return ""}// leafWriter accumulates the visible text of an HTML subtree while applying@@ -323,27 +321,28 @@ func (w *leafWriter) hardBreak() {// subtrees are skipped. Whitespace is folded per CSS rules (so// "<h1>Hello world</h1>" becomes "Hello world" and "<br>" survives as a// real line break), while <pre>/<textarea> keep their source formatting-// verbatim. ckType is the block's ck_type (from htmlTagToCkType) applied to the-// accumulated prose item; trim controls whether trailing/leading whitespace is-// collapsed (false for <pre>/<textarea>, which must stay verbatim). Any <table>+// verbatim. layout is the block's structural layout value (from+// htmlTagToLayout) applied to the accumulated prose item; trim controls+// whether trailing/leading whitespace is collapsed (false for+// <pre>/<textarea>, which must stay verbatim). Any <table>// encountered in the subtree is emitted as a single structured// doc_type_kwd:"table" item at its document position (see walkHTMLLeaf's// "table" case) — the prose around it is flushed as ordinary text items, so// the table is never relocated to the end and never duplicated.-func htmlLeafTextWithState(n *html.Node, out *[]map[string]any, ckType string, trim bool, state *htmlWalkState) {+func htmlLeafTextWithState(n *html.Node, out *[]map[string]any, layout string, trim bool, state *htmlWalkState) {var b bytes.Bufferw := &leafWriter{b: &b}- walkHTMLLeaf(n, w, out, ckType, trim, state)- flushLeafText(w, out, ckType, trim)+ walkHTMLLeaf(n, w, out, layout, trim, state)+ flushLeafText(w, out, layout, trim)}// flushLeafText emits any text accumulated in w as a doc_type_kwd:"text" item-// (dropping empty output) and resets the writer. ckType/trim mirror+// (dropping empty output) and resets the writer. layout/trim mirror// appendHTMLTextItem. It is called at block boundaries and at <table> elements// so tables are emitted in their original document position rather than being// relocated. Prose with no specific block tag (e.g. text accumulated just-// before a nested table) is flushed as ck_type "text".-func flushLeafText(w *leafWriter, out *[]map[string]any, ckType string, trim bool) {+// before a nested table) is flushed with no layout.+func flushLeafText(w *leafWriter, out *[]map[string]any, layout string, trim bool) {text := w.b.String()if trim {text = strings.TrimSpace(text)@@ -351,14 +350,14 @@ func flushLeafText(w *leafWriter, out *[]map[string]any, ckType string, trim booif text == "" {return}- appendHTMLTextItem(out, text, ckType, false)+ appendHTMLTextItem(out, text, layout, false)w.b.Reset()w.lastSpace = falsew.lineStart = truew.endsNL = false}-func walkHTMLLeaf(n *html.Node, w *leafWriter, out *[]map[string]any, ckType string, trim bool, state *htmlWalkState) {+func walkHTMLLeaf(n *html.Node, w *leafWriter, out *[]map[string]any, layout string, trim bool, state *htmlWalkState) {switch n.Type {case html.TextNode:w.writeText(n.Data)@@ -378,7 +377,7 @@ func walkHTMLLeaf(n *html.Node, w *leafWriter, out *[]map[string]any, ckType str}return}- flushLeafText(w, out, ckType, trim)+ flushLeafText(w, out, layout, trim)appendHTMLImageItem(out, state, src, htmlAttribute(n, "alt"), "", 0, 0)return}@@ -386,7 +385,7 @@ func walkHTMLLeaf(n *html.Node, w *leafWriter, out *[]map[string]any, ckType str// Verbatim: no folding, no injected block breaks.w.pre = truefor child := n.FirstChild; child != nil; child = child.NextSibling {- walkHTMLLeaf(child, w, out, ckType, trim, state)+ walkHTMLLeaf(child, w, out, layout, trim, state)}w.pre = falsereturn@@ -412,7 +411,7 @@ func walkHTMLLeaf(n *html.Node, w *leafWriter, out *[]map[string]any, ckType strw.hardBreak()}for child := n.FirstChild; child != nil; child = child.NextSibling {- walkHTMLLeaf(child, w, out, ckType, trim, state)+ walkHTMLLeaf(child, w, out, layout, trim, state)}if !w.pre && isBlockTag(n.Data) && w.b.Len() > 0 && !w.endsNL {w.hardBreak()@@ -443,7 +442,6 @@ func emitHTMLTable(n *html.Node, out *[]map[string]any, state *htmlWalkState) {td, _ := tableutil.ParseTableHTML(markup)table := map[string]any{"doc_type_kwd": "table",- "ck_type": "table","table": td,}tableID := ""@@ -521,7 +519,6 @@ func appendHTMLImageItem(out *[]map[string]any, state *htmlWalkState, src, alt,item := map[string]any{"text": strings.TrimSpace(alt),"doc_type_kwd": "image",- "ck_type": "image","media_order": state.mediaOrder,}item["image"] = srcdiff --git a/internal/parser/parser/html_parser_align_test.go b/internal/parser/parser/html_parser_align_test.goindex 5c0d93a621..dcf5f6610e 100644--- a/internal/parser/parser/html_parser_align_test.go+++ b/internal/parser/parser/html_parser_align_test.go@@ -9,7 +9,7 @@ import (// TestHTMLParser_TableProducesStructuredItems asserts the HTML table contract:// a <table> is emitted as exactly ONE structured doc_type_kwd:"table"/-// ck_type:"table" item, in its original document position (not relocated to+// layout-free structured table item, in its original document position (not relocated to// the end), with NO duplicate doc_type_kwd:"text" copy carrying the <table>// markup. Keeping the full <table>…</table> markup (not flattened) preserves// row/column structure for embedding/retrieval/LLM rendering, and the single@@ -61,8 +61,8 @@ func TestHTMLParser_TableProducesStructuredItems(t *testing.T) {if tableCount != 1 {t.Fatalf("structured doc_type_kwd:\"table\" item count = %d, want exactly 1", tableCount)}- if got, want := res.JSON[tableIdx].ToMap()["ck_type"], "table"; got != want {- t.Errorf("structured table item ck_type = %v, want %v", got, want)+ if _, ok := res.JSON[tableIdx].ToMap()["ck_type"]; ok {+ t.Errorf("retired ck_type present on structured table item: %#v", res.JSON[tableIdx].ToMap())}if !strings.Contains(tableText, "<table") ||!strings.Contains(tableText, "Name") ||@@ -144,8 +144,8 @@ func TestHTMLParser_NestedTableProducesStructuredItems(t *testing.T) {if tableCount != 1 {t.Fatalf("structured doc_type_kwd:\"table\" item count = %d, want exactly 1", tableCount)}- if got, want := res.JSON[tableIdx].ToMap()["ck_type"], "table"; got != want {- t.Errorf("structured nested table item ck_type = %v, want %v", got, want)+ if _, ok := res.JSON[tableIdx].ToMap()["ck_type"]; ok {+ t.Errorf("retired ck_type present on structured nested table item: %#v", res.JSON[tableIdx].ToMap())}if !strings.Contains(tableText, "<table") ||!strings.Contains(tableText, "Name") ||@@ -383,25 +383,24 @@ func TestHTMLParser_NestedListItemsPreserved(t *testing.T) {}// The whole top-level <ul> is emitted as ONE text item tagged- // ck_type:"list" (Go's "one item per block-level element" model). Guard+ // layout:"list" (Go's "one item per block-level element" model). Guard// against a regression that would drop the list tag or split it// unexpectedly. Note: a list nested inside a non-list container// (div/section/…) is folded into that container's single text item and- // therefore loses ck_type:"list" (becomes "text") — a fidelity nuance,- // not a collapse, since downstream chunking reads doc_type_kwd not ck_type.+ // therefore loses layout:"list" — a fidelity nuance, not a collapse.var listText stringfor _, it := range ItemsToMaps(res.JSON) {- if it["ck_type"] == "list" {+ if it["layout"] == "list" {listText, _ = it["text"].(string)break}}if listText == "" {- t.Fatalf("top-level <ul> not tagged ck_type:\"list\"; got items: %#v", res.JSON)+ t.Fatalf("top-level <ul> not tagged layout:\"list\"; got items: %#v", res.JSON)}for _, want := range []string{"Item 1", "Item 2", "Sub A", "Sub B"} {if !strings.Contains(listText, want) {- t.Errorf("ck_type:\"list\" item missing %q: %q", want, listText)+ t.Errorf("layout:\"list\" item missing %q: %q", want, listText)}}}@@ -413,7 +412,7 @@ func TestHTMLParser_NestedListItemsPreserved(t *testing.T) {// structure and the LLM / user would see broken code. This mirrors Python's// RAGFlowHtmlParser, which keeps pre/textarea verbatim (_PRE_TAGS in// deepdoc/parser/html_parser.py). Both blocks keep doc_type_kwd:"text"; <pre>-// is tagged ck_type:"code" while <textarea> falls back to ck_type:"text"+// is tagged layout:"code" while <textarea> has no structural layout// (htmlTagToCkType has no textarea case) — a labeling nuance, not a content// divergence. Note: the single leading newline immediately after <pre>/<textarea>// is dropped by the HTML parser per spec (pre/textarea eat one leading newline);@@ -453,8 +452,8 @@ SELECT * FROM tif preItem == nil {t.Fatalf("no <pre> item found; got items: %#v", res.JSON)}- if preItem["ck_type"] != "code" {- t.Errorf("<pre> item ck_type = %v, want code", preItem["ck_type"])+ if preItem["layout"] != "code" {+ t.Errorf("<pre> item layout = %v, want code", preItem["layout"])}preText, _ := preItem["text"].(string)// 4-space and 8-space indents must survive verbatim (a folded path woulddiff --git a/internal/parser/parser/markdown_parser.go b/internal/parser/parser/markdown_parser.goindex 2a20b9df7d..e16c41984a 100644--- a/internal/parser/parser/markdown_parser.go+++ b/internal/parser/parser/markdown_parser.go@@ -386,27 +386,26 @@ func markdownTableCells(line string) []string {//// Tables: a GFM/HTML table is rendered inline as a single <table> HTML// block by renderMarkdownTablesInlineText and kept as one HTML block.-// It is emitted as ONE structured item (doc_type_kwd:"table",-// ck_type:"table") in its original document position — there is no+// It is emitted as ONE structured item (doc_type_kwd:"table")+// in its original document position — there is no// duplicate doc_type_kwd:"text" copy. The downstream chunker consumes// doc_type_kwd:"table" to keep the table whole and attach table context// to neighbouring chunks (chunker/token.go). Non-table HTML blocks-// (<div>, <style>, …) are emitted as ordinary text with no ck_type.+// (<div>, <style>, …) are emitted as ordinary text with no layout.func walkMarkdownBlocksWithImages(ctx context.Context, doc ast.Node, out *[]map[string]any, flatten, fetchRemoteImages bool, budget *embeddedMediaBudget) int {unresolvedImages := 0for _, child := range doc.GetChildren() {- var ckType string+ var layout stringvar docTypeKwd stringvar txt stringswitch n := child.(type) {case *ast.Heading:txt = headingText(n)- ckType = "heading"+ layout = "heading"docTypeKwd = "text"case *ast.Paragraph:txt = leafText(n)- ckType = "text"docTypeKwd = "text"case *ast.List:// Python's _markdown keeps one item per list entry, marker included@@ -420,21 +419,21 @@ func walkMarkdownBlocksWithImages(ctx context.Context, doc ast.Node, out *[]map[*out = append(*out, map[string]any{"text": entry,"doc_type_kwd": "text",- "ck_type": "list",+ "layout": "list",})}continuecase *ast.CodeBlock:txt = leafText(n)- ckType = "code"+ layout = "code"docTypeKwd = "text"case *ast.HTMLBlock:// An HTML block is either an inlined table (kept as a single// HTML block thanks to the blank lines renderMarkdownTablesInline// wraps around it) or a plain HTML block such as <div>/<style>.// Only a table is emitted as a structured table item; everything- // else is treated as ordinary text (no ck_type). We emit exactly- // ONE item (doc_type_kwd:"table"/ck_type:"table") in document+ // else is treated as ordinary text (no layout). We emit exactly+ // ONE item (doc_type_kwd:"table") in document// order — no duplicate doc_type_kwd:"text" copy — so the table is// embedded once and its markup does not pollute prose chunks.txt = leafText(n)@@ -447,13 +446,12 @@ func walkMarkdownBlocksWithImages(ctx context.Context, doc ast.Node, out *[]map[td, _ := tableutil.ParseTableHTML(txt)table := map[string]any{"doc_type_kwd": "table",- "ck_type": "table","table": td,}*out = append(*out, table)continue}- // Non-table HTML block: ordinary text, no ck_type.+ // Non-table HTML block: ordinary text, no layout.docTypeKwd = "text"default:txt = leafText(n)@@ -467,8 +465,8 @@ func walkMarkdownBlocksWithImages(ctx context.Context, doc ast.Node, out *[]map["text": txt,"doc_type_kwd": docTypeKwd,}- if ckType != "" {- item["ck_type"] = ckType+ if layout != "" {+ item["layout"] = layout}// Resolve Markdown images from the AST node of THIS block only, so@@ -719,7 +717,7 @@ func resolveAndValidateHost(ctx context.Context, host string) (net.IP, error) {// headingText returns a canonical ATX heading (marker plus inline text).// Keeping the marker in the parser payload preserves the Markdown structure-// for GeneralChunker and downstream LLM prompts; ck_type alone does not carry+// for GeneralChunker and downstream LLM prompts; layout alone does not carry// the heading level.func headingText(h *ast.Heading) string {var buf bytes.Bufferdiff --git a/internal/parser/parser/markdown_parser_test.go b/internal/parser/parser/markdown_parser_test.goindex 27689d4b3d..b63b101ea5 100644--- a/internal/parser/parser/markdown_parser_test.go+++ b/internal/parser/parser/markdown_parser_test.go@@ -34,8 +34,8 @@ func TestMarkdownParser_ParseWithResult_Basic(t *testing.T) {if got, _ := res.JSON[0].ToMap()["text"].(string); got != "# Hello" {t.Fatalf("first item text = %q, want %q", got, "# Hello")}- if got, _ := res.JSON[0].ToMap()["ck_type"].(string); got != "heading" {- t.Fatalf("first item ck_type = %q, want %q", got, "heading")+ if got, _ := res.JSON[0].ToMap()["layout"].(string); got != "heading" {+ t.Fatalf("first item layout = %q, want %q", got, "heading")}}@@ -52,8 +52,8 @@ func TestMarkdownParser_ParseWithResult_UTF8BOM(t *testing.T) {if got, _ := res.JSON[0].ToMap()["text"].(string); got != "# Hello" {t.Fatalf("first item text = %q, want %q", got, "# Hello")}- if got, _ := res.JSON[0].ToMap()["ck_type"].(string); got != "heading" {- t.Fatalf("first item ck_type = %q, want %q", got, "heading")+ if got, _ := res.JSON[0].ToMap()["layout"].(string); got != "heading" {+ t.Fatalf("first item layout = %q, want %q", got, "heading")}}@@ -79,7 +79,7 @@ func TestMarkdownParser_ListItemsKeepWordBoundaries(t *testing.T) {}var lists []stringfor _, item := range ItemsToMaps(res.JSON) {- if ck, _ := item["ck_type"].(string); ck == "list" {+ if ck, _ := item["layout"].(string); ck == "list" {text, _ := item["text"].(string)lists = append(lists, text)}@@ -141,7 +141,7 @@ func TestMarkdownParser_OrderedListKeepsNumbers(t *testing.T) {}var lists []stringfor _, item := range ItemsToMaps(res.JSON) {- if ck, _ := item["ck_type"].(string); ck == "list" {+ if ck, _ := item["layout"].(string); ck == "list" {text, _ := item["text"].(string)lists = append(lists, text)}@@ -273,8 +273,8 @@ func TestMarkdownParser_ParseWithResult_RendersTableInline(t *testing.T) {switch kd {case "table":tableCount++- if ck, _ := item["ck_type"].(string); ck != "table" {- t.Errorf("table item ck_type = %q, want \"table\"", ck)+ if _, ok := item["ck_type"]; ok {+ t.Errorf("retired ck_type present on table item: %#v", item)}td, ok := item["table"].(*entity.TableData)if !ok || td == nil {@@ -642,8 +642,8 @@ func TestMarkdownParser_TableNotCollapsed(t *testing.T) {if _, ok := item["table"].(*entity.TableData); !ok {t.Fatalf("table item missing *entity.TableData: %#v", item)}- if ck, _ := item["ck_type"].(string); ck != "table" {- t.Fatalf("table item ck_type = %q, want \"table\"", ck)+ if _, ok := item["ck_type"]; ok {+ t.Fatalf("retired ck_type present on table item: %#v", item)}}}@@ -707,8 +707,8 @@ func TestMarkdownParser_TableWithSurroundingText(t *testing.T) {if _, ok := item["table"].(*entity.TableData); !ok {t.Fatalf("table item missing *entity.TableData: %#v", item)}- if ck, _ := item["ck_type"].(string); ck != "table" {- t.Fatalf("table item ck_type = %q, want \"table\"", ck)+ if _, ok := item["ck_type"]; ok {+ t.Fatalf("retired ck_type present on table item: %#v", item)}}}@@ -740,8 +740,8 @@ func TestMarkdownParser_TableWithSurroundingText(t *testing.T) {}// TestMarkdownParser_NonTableHTMLBlockNotTable guards against the blanket-// ck_type:"table" bug: a raw HTML block that is NOT a <table> (e.g. <div>,-// <style>) must be emitted as ordinary text with no ck_type, so downstream+// raw-table bug: a raw HTML block that is NOT a <table> (e.g. <div>,+// <style>) must be emitted as ordinary text with no layout, so downstream// consumers (chunker) do not mistake it for a table.func TestMarkdownParser_NonTableHTMLBlockNotTable(t *testing.T) {ctx := t.Context()@@ -754,8 +754,8 @@ func TestMarkdownParser_NonTableHTMLBlockNotTable(t *testing.T) {for _, item := range ItemsToMaps(res.JSON) {text, _ := item["text"].(string)if strings.Contains(text, "<div") || strings.Contains(text, "<style") {- if ck, ok := item["ck_type"].(string); ok && ck == "table" {- t.Fatalf("non-table HTML block wrongly tagged ck_type:\"table\": %q", text)+ if _, ok := item["ck_type"]; ok {+ t.Fatalf("retired ck_type present on non-table item: %q", text)}if kd, _ := item["doc_type_kwd"].(string); kd == "table" {t.Fatalf("non-table HTML block wrongly tagged doc_type_kwd:\"table\": %q", text)@@ -827,8 +827,8 @@ func TestMarkdownParser_RawHTMLTableHandled(t *testing.T) {if _, ok := item["table"].(*entity.TableData); !ok {t.Fatalf("raw table item missing *entity.TableData: %#v", item)}- if ck, _ := item["ck_type"].(string); ck != "table" {- t.Fatalf("raw table item ck_type = %q, want \"table\"", ck)+ if _, ok := item["ck_type"]; ok {+ t.Fatalf("retired ck_type present on raw table item: %#v", item)}}}diff --git a/internal/parser/parser/office_table_render.go b/internal/parser/parser/office_table_render.goindex 14b9f60119..f91947a3c4 100644--- a/internal/parser/parser/office_table_render.go+++ b/internal/parser/parser/office_table_render.go@@ -96,7 +96,7 @@ func trimCells(row []string) []string {// Every tuple is a spreadsheet position, [sheet, rowStart, rowEnd, colStart,// colEnd]. The same `positions` field carries PDF layout boxes// ([page, left, right, top, bottom]) on PDF items, so a consumer must key the-// vocabulary off sheet_index / ck_type and never reinterpret one as the other.+// vocabulary off sheet_index / doc_type_kwd and never reinterpret one as the other.func buildSheetItems(records [][]string, sheet string, sheetIndex, headerRow int, dataRows []int, images []map[string]any) []map[string]any {sortImagesByAnchor(images)if len(records) == 0 {@@ -356,7 +356,6 @@ cellLoop:item := map[string]any{"text": alt,"doc_type_kwd": "image",- "ck_type": "image","sheet": sheet,"cell": cell,"row_start": row,diff --git a/internal/parser/parser/office_table_render_test.go b/internal/parser/parser/office_table_render_test.goindex 6c04242f8a..b491e30a33 100644--- a/internal/parser/parser/office_table_render_test.go+++ b/internal/parser/parser/office_table_render_test.go@@ -30,8 +30,8 @@ func TestXLSXParserEmitsSegmentedHTMLTable(t *testing.T) {t.Fatalf("items = %d, want one segmented HTML table item", len(res.JSON))}item := res.JSON[0].ToMap()- if item["ck_type"] != "table" || item["doc_type_kwd"] != "table" {- t.Fatalf("item types = ck_type:%v doc_type:%v", item["ck_type"], item["doc_type_kwd"])+ if item["doc_type_kwd"] != "table" {+ t.Fatalf("item doc_type_kwd = %v, want table", item["doc_type_kwd"])}wantText := "<table><caption>Sheet1</caption>\n" +"<tr><th>ID</th><th>Status</th></tr>\n" +@@ -81,7 +81,7 @@ func TestXLSXParserHTML4ExcelIsIgnored(t *testing.T) {if len(res.JSON) != 1 {t.Fatalf("items = %d, want one atomic table item", len(res.JSON))}- if res.JSON[0].ToMap()["ck_type"] != "table" || res.JSON[0].ToMap()["table"] == nil {+ if res.JSON[0].ToMap()["doc_type_kwd"] != "table" || res.JSON[0].ToMap()["table"] == nil {t.Fatalf("html4excel item = %#v", res.JSON[0])}if !reflect.DeepEqual(res.JSON, plainRes.JSON) {diff --git a/internal/parser/parser/parse_result.go b/internal/parser/parser/parse_result.goindex 3afa7225da..3398e8997b 100644--- a/internal/parser/parser/parse_result.go+++ b/internal/parser/parser/parse_result.go@@ -117,6 +117,5 @@ func NewTableDataJSONItem(td *entity.TableData, sheet string, positions PositionTable: td,Sheet: sheet,Positions: positions,- CKType: DocTypeTable,}}diff --git a/internal/parser/parser/parse_result_test.go b/internal/parser/parser/parse_result_test.goindex 6f47acd44c..073084b577 100644--- a/internal/parser/parser/parse_result_test.go+++ b/internal/parser/parser/parse_result_test.go@@ -23,7 +23,7 @@ import ("ragflow/internal/entity")-func TestNewTableDataJSONItem_IncludesDocTypeAndChunkType(t *testing.T) {+func TestNewTableDataJSONItem_IncludesDocType(t *testing.T) {td := &entity.TableData{Grid: [][]entity.TableCell{{{Text: "col"}}, {{Text: "value"}}},HeaderRows: 1,@@ -35,9 +35,6 @@ func TestNewTableDataJSONItem_IncludesDocTypeAndChunkType(t *testing.T) {if got, want := m[DocTypeKey], DocTypeTable; got != want {t.Fatalf("item[%q] = %v, want %v", DocTypeKey, got, want)}- if got, want := m["ck_type"], DocTypeTable; got != want {- t.Fatalf("item[%q] = %v, want %v", "ck_type", got, want)- }got, ok := m["table"].(*entity.TableData)if !ok {t.Fatalf("item[%q] = %T, want *entity.TableData", "table", m["table"])@@ -175,8 +172,8 @@ func TestMarkdownParser_ParseWithResult(t *testing.T) {if txt, _ := heading["text"].(string); !strings.Contains(txt, "Title") {t.Errorf("JSON[0].text = %q, want contains 'Title'", txt)}- if got, _ := heading["ck_type"].(string); got != "heading" {- t.Errorf("JSON[0].ck_type = %q, want heading", got)+ if got, _ := heading["layout"].(string); got != "heading" {+ t.Errorf("JSON[0].layout = %q, want heading", got)}for i, it := range ItemsToMaps(res.JSON) {if _, ok := it["doc_type_kwd"]; !ok {diff --git a/internal/parser/parser/parse_with_result_test.go b/internal/parser/parser/parse_with_result_test.goindex 7c51365021..74b88cce62 100644--- a/internal/parser/parser/parse_with_result_test.go+++ b/internal/parser/parser/parse_with_result_test.go@@ -19,7 +19,7 @@// that did not previously satisfy ParseResultProducer://// - HTMLParser — block-level walker that emits the python-compatible-// {text, doc_type_kwd, ck_type} shape.+// {text, doc_type_kwd, layout} shape.// - TextParser — normalized one-unit output for the text&code family// (.txt / .py / .js / .java / .c / .cpp / .h / .php / .go / .ts// / .sh / .cs / .kt / .sql).@@ -135,7 +135,7 @@ func TestTextParser_ParseWithResult_GBK(t *testing.T) {// TestHTMLParser_ParseWithResult_BlockSplit pins the HTML walker.// Three block elements (heading, paragraph, list) yield three-// items with the python-compatible ck_type vocabulary.+// items with the structural layout vocabulary.func TestHTMLParser_ParseWithResult_BlockSplit(t *testing.T) {ctx := t.Context()p := NewHTMLParser()@@ -154,20 +154,20 @@ func TestHTMLParser_ParseWithResult_BlockSplit(t *testing.T) {if len(res.JSON) != 3 {t.Fatalf("JSON len = %d, want 3 (h1, p, ul)", len(res.JSON))}- if got, want := res.JSON[0].ToMap()["ck_type"], "heading"; got != want {- t.Errorf("JSON[0].ck_type = %v, want %v", got, want)+ if got, want := res.JSON[0].ToMap()["layout"], "heading"; got != want {+ t.Errorf("JSON[0].layout = %v, want %v", got, want)}if got, want := res.JSON[0].ToMap()["text"], "Title"; got != want {t.Errorf("JSON[0].text = %v, want %v", got, want)}- if got, want := res.JSON[1].ToMap()["ck_type"], "paragraph"; got != want {- t.Errorf("JSON[1].ck_type = %v, want %v", got, want)+ if got, want := res.JSON[1].ToMap()["layout"], "paragraph"; got != want {+ t.Errorf("JSON[1].layout = %v, want %v", got, want)}if got, want := res.JSON[1].ToMap()["text"], "First paragraph."; got != want {t.Errorf("JSON[1].text = %v, want %v", got, want)}- if got, want := res.JSON[2].ToMap()["ck_type"], "list"; got != want {- t.Errorf("JSON[2].ck_type = %v, want %v", got, want)+ if got, want := res.JSON[2].ToMap()["layout"], "list"; got != want {+ t.Errorf("JSON[2].layout = %v, want %v", got, want)}if got, want := res.JSON[2].ToMap()["text"], "Item one"; got != want {t.Errorf("JSON[2].text = %v, want %v", got, want)@@ -195,7 +195,7 @@ Tail text}want := []struct {text string- ckType string+ layout string}{{"Intro text", "text"},{"Title", "heading"},@@ -213,8 +213,8 @@ Tail textif got := res.JSON[i].ToMap()["doc_type_kwd"]; got != "text" {t.Errorf("JSON[%d].doc_type_kwd = %v, want text", i, got)}- if got := res.JSON[i].ToMap()["ck_type"]; got != w.ckType {- t.Errorf("JSON[%d].ck_type = %v, want %v", i, got, w.ckType)+ if got := res.JSON[i].ToMap()["layout"]; got != w.layout {+ t.Errorf("JSON[%d].layout = %v, want %v", i, got, w.layout)}}}diff --git a/internal/parser/parser/pdf_parser_paddleocr.go b/internal/parser/parser/pdf_parser_paddleocr.goindex fd829ecd73..c9d7821527 100644--- a/internal/parser/parser/pdf_parser_paddleocr.go+++ b/internal/parser/parser/pdf_parser_paddleocr.go@@ -11,6 +11,18 @@ import ("ragflow/internal/utility")+// isStructuralOCRTextLayout reports whether an item already carries a+// structural block kind produced by markdown normalization. Those values are+// finer than an OCR block label and must win over it.+func isStructuralOCRTextLayout(fields map[string]any) bool {+ layout, _ := fields["layout"].(string)+ switch layout {+ case "heading", "paragraph", "list", "code", "quote":+ return true+ }+ return false+}+// PaddleOCRPDFResult projects OCR regions onto the PDF's page geometry and// emits the structured items consumed by the existing PDF chunk pipeline.func PaddleOCRPDFResult(ctx context.Context, filename string, data []byte, response *models.OCRFileResponse) ParseResult {@@ -63,17 +75,22 @@ func PaddleOCRPDFResult(ctx context.Context, filename string, data []byte, respolabel = "caption"}for _, item := range parsed.JSON {+ _, isTextItem := item.(TextItem)if text, ok := item.(TextItem); ok {if strings.TrimSpace(text.Text) == "" {continue}- if isTitle && text.CKType == "text" {+ layout, _ := text.Extra["layout"].(string)+ if isTitle && layout != "heading" && layout != "list" && layout != "code" {prefix := "## "if label == "doc_title" {prefix = "# "}text.Text = prefix + text.Text- text.CKType = "heading"+ if text.Extra == nil {+ text.Extra = map[string]any{}+ }+ text.Extra["layout"] = "heading"item = text}}@@ -81,10 +98,18 @@ func PaddleOCRPDFResult(ctx context.Context, filename string, data []byte, respo// Page identity is retained even when the block has no usable geometry.fields["page_number"] = page.Index + 1fields["_preserve_order"] = true- if isTitle {- fields["layout"] = "title"+ // Markdown normalization already asserted a structural layout+ // (heading/list/code). The OCR block label is coarser and must+ // not overwrite it; only fill an absent layout.+ if isTitle && isTextItem && !isStructuralOCRTextLayout(fields) {+ // The asserted-title marker the Title chunker reads without+ // the notTitle heuristic guard. PDF model labels that merely+ // look title-like stay on "title".+ fields["layout"] = "heading"} else if label != "" {- fields["layout"] = label+ if existing, _ := fields["layout"].(string); existing == "" {+ fields["layout"] = label+ }}if len(position) > 0 {matrix := [][]any{position}diff --git a/internal/parser/parser/pdf_parser_paddleocr_test.go b/internal/parser/parser/pdf_parser_paddleocr_test.goindex 0530e041bc..caf21ab35c 100644--- a/internal/parser/parser/pdf_parser_paddleocr_test.go+++ b/internal/parser/parser/pdf_parser_paddleocr_test.go@@ -100,12 +100,12 @@ func TestPaddleOCRTitleLabels(t *testing.T) {if err != nil {t.Fatal(err)}- for _, tc := range []struct{ label, text, want, ckType string }{+ for _, tc := range []struct{ label, text, want, layout string }{{"doc_title", "Document title", "# Document title", "heading"},{"paragraph_title", "Section title", "## Section title", "heading"},{"doc_title", "### Existing title", "### Existing title", "heading"},- {"figure_title", "Figure caption", "Figure caption", "text"},- {"table_title", "Table caption", "Table caption", "text"},+ {"figure_title", "Figure caption", "Figure caption", "caption"},+ {"table_title", "Table caption", "Table caption", "caption"},} {t.Run(tc.label+tc.text, func(t *testing.T) {response := &models.OCRFileResponse{Pages: []models.OCRPage{{Index: 0, Blocks: []models.OCRBlock{{Text: tc.text, Label: tc.label}}}}}@@ -114,7 +114,7 @@ func TestPaddleOCRTitleLabels(t *testing.T) {t.Fatal(result.Err)}item := result.JSON[0].ToMap()- if item["ck_type"] != tc.ckType || item["text"] != tc.want {+ if item["layout"] != tc.layout || item["text"] != tc.want {t.Fatalf("title semantics: %v", item)}formatted := pdfItemsToResult("title.pdf", ItemsToMaps(result.JSON), "markdown", 1)@@ -125,6 +125,42 @@ func TestPaddleOCRTitleLabels(t *testing.T) {}}+func TestPaddleOCRBlocksPreserveMarkdownLayout(t *testing.T) {+ data, err := os.ReadFile("testdata/paddleocr_pages.pdf")+ if err != nil {+ t.Fatal(err)+ }+ for _, tc := range []struct{ name, label, text, wantLayout string }{+ {"text heading", "text", "# Section", "heading"},+ {"text code", "text", "```\ncode line\n```", "code"},+ {"text list", "text", "- item one", "list"},+ {"title heading", "paragraph_title", "# Section", "heading"},+ {"title code", "paragraph_title", "```\ncode line\n```", "code"},+ {"title list", "paragraph_title", "- item one", "list"},+ } {+ t.Run(tc.name, func(t *testing.T) {+ // The layout model label is coarser than the Markdown+ // normalization. It must not overwrite a structural block kind,+ // including inside a title-labelled block.+ response := &models.OCRFileResponse{Pages: []models.OCRPage{{Index: 0, Blocks: []models.OCRBlock{{Text: tc.text, Label: tc.label}}}}}+ result := PaddleOCRPDFResult(t.Context(), "layout.pdf", data, response)+ if result.Err != nil {+ t.Fatal(result.Err)+ }+ if len(result.JSON) == 0 {+ t.Fatal("no items emitted")+ }+ item := result.JSON[0].ToMap()+ if item["layout"] != tc.wantLayout {+ t.Fatalf("layout = %v, want %q (markdown structural layout must win over the OCR label): %v", item["layout"], tc.wantLayout, item)+ }+ if text, _ := item["text"].(string); strings.HasPrefix(text, "## ") {+ t.Errorf("structural item must not be promoted with a heading prefix: %q", text)+ }+ })+ }+}+func TestPaddleOCRPDFResultRejectsRenderedEmptyText(t *testing.T) {for _, label := range []string{"text", "doc_title", "paragraph_title"} {t.Run(label, func(t *testing.T) {diff --git a/internal/parser/parser/pptx_ir.go b/internal/parser/parser/pptx_ir.goindex 35351c9bdf..81a20ae2db 100644--- a/internal/parser/parser/pptx_ir.go+++ b/internal/parser/parser/pptx_ir.go@@ -69,7 +69,6 @@ func buildPPTXJSONSections(irJSON string, budget *embeddedMediaBudget) ([]map[st"text": "","image": nil,"doc_type_kwd": "image",- "ck_type": "image","slide_number": i + 1,"media_order": mediaOrder,}diff --git a/internal/parser/parser/xls_parser_test.go b/internal/parser/parser/xls_parser_test.goindex 95224ff5b2..0a9a2e0bbd 100644--- a/internal/parser/parser/xls_parser_test.go+++ b/internal/parser/parser/xls_parser_test.go@@ -225,8 +225,8 @@ func TestXLSParser_SpreadsheetJSONOutput(t *testing.T) {t.Fatalf("res.JSON should have structured table items")}item := res.JSON[0].ToMap()- if item["ck_type"] != "table" {- t.Errorf("table item ck_type = %v, want table", item["ck_type"])+ if item["doc_type_kwd"] != "table" {+ t.Errorf("table item doc_type_kwd = %v, want table", item["doc_type_kwd"])}text := tableItemHTML(t, item)if !strings.Contains(text, "<tr><th>Header1</th><th>Header2</th></tr>") ||@@ -271,8 +271,8 @@ func TestXLSXParser_JSONOutput(t *testing.T) {if len(res.JSON) != 1 {t.Fatalf("res.JSON = %#v, want one segmented HTML table item", res.JSON)}- if res.JSON[0].ToMap()["ck_type"] != "table" {- t.Errorf("item ck_type = %v, want table", res.JSON[0].ToMap()["ck_type"])+ if res.JSON[0].ToMap()["doc_type_kwd"] != "table" {+ t.Errorf("item doc_type_kwd = %v, want table", res.JSON[0].ToMap()["doc_type_kwd"])}// Even if configured to legacy "html", format is unified to "json"diff --git a/internal/parser/parser/xls_tcadp_test.go b/internal/parser/parser/xls_tcadp_test.goindex 7768be6cc6..84a6fe4973 100644--- a/internal/parser/parser/xls_tcadp_test.go+++ b/internal/parser/parser/xls_tcadp_test.go@@ -228,7 +228,7 @@ func TestCSVParser_ParseWithResult_DefaultCSVBehavior(t *testing.T) {if len(res.JSON) != 1 {t.Fatalf("JSON items = %d, want one HTML table item", len(res.JSON))}- if res.JSON[0].ToMap()["ck_type"] != "table" {+ if res.JSON[0].ToMap()["doc_type_kwd"] != "table" {t.Fatalf("JSON item = %#v; want a table item", res.JSON[0])}}diff --git a/internal/parser/parser/xlsx_parse_method_test.go b/internal/parser/parser/xlsx_parse_method_test.goindex 43ea5bf7a4..643728a757 100644--- a/internal/parser/parser/xlsx_parse_method_test.go+++ b/internal/parser/parser/xlsx_parse_method_test.go@@ -177,7 +177,7 @@ func TestCSVParser_DeepDocParseMethod(t *testing.T) {if len(res.JSON) != 1 {t.Fatalf("JSON items = %d, want one HTML table item", len(res.JSON))}- if res.JSON[0].ToMap()["ck_type"] != "table" {+ if res.JSON[0].ToMap()["doc_type_kwd"] != "table" {t.Fatalf("JSON item = %#v, want a table item", res.JSON[0])}}diff --git a/internal/service/file/file_content_test.go b/internal/service/file/file_content_test.goindex b50ea119e9..e72bf1d1dc 100644--- a/internal/service/file/file_content_test.go+++ b/internal/service/file/file_content_test.go@@ -83,7 +83,7 @@ func TestParseResultText_SpreadsheetSegmentsAreTheirOwnMarkup(t *testing.T) {result, err := parseResultText(parser.ParseResult{OutputFormat: "json",JSON: parser.MapsToItems([]map[string]any{- {"text": segment, "doc_type_kwd": "table", "ck_type": "table", "sheet": "Data", "sheet_index": 1},+ {"text": segment, "doc_type_kwd": "table", "sheet": "Data", "sheet_index": 1},}),})if err != nil {diff --git a/tools/mdprobe/main.go b/tools/mdprobe/main.goindex c9e12088f3..3703173a78 100644--- a/tools/mdprobe/main.go+++ b/tools/mdprobe/main.go@@ -50,9 +50,9 @@ func main() {m := item.ToMap()text, _ := m["text"].(string)total += len(text)- ck := fmt.Sprint(m["ck_type"])+ layout := fmt.Sprint(m["layout"])docType := fmt.Sprint(m["doc_type_kwd"])- fmt.Printf(" [%3d] %-7s %-6s %6d chars %q\n", i, ck, docType, len(text), truncate(text, 90))+ fmt.Printf(" [%3d] %-7s %-6s %6d chars %q\n", i, layout, docType, len(text), truncate(text, 90))}fmt.Printf("parser emitted %d chars total (file %d)\n", total, len(data))