Skip to content

RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.

refactor(ingestion): retire ck_type in favor of layout and doc_type (#21259)

jay7772111ca59d5113e
diff --git a/internal/ingestion/component/chunker/a1a2_regression_test.go b/internal/ingestion/component/chunker/a1a2_regression_test.go
index 110f8dfa60..a6f171697e 100644
--- a/internal/ingestion/component/chunker/a1a2_regression_test.go
+++ b/internal/ingestion/component/chunker/a1a2_regression_test.go
@@ -83,7 +83,7 @@ func TestCropImageChunks_UploadFailureRedactsChunkText(t *testing.T) {
eng := mockCropEngine{}
pos := jsonPositions(t, []float64{1, 10, 100, 10, 100})
secret := "SECRET_DOC_TEXT_9f3a"
- chunks := []schema.ChunkDoc{{CKType: "image", Text: secret, PDFPositions: pos}}
+ chunks := []schema.ChunkDoc{{DocType: "image", Text: secret, PDFPositions: pos}}
cropImageChunks(ctx, eng, chunks)
for _, e := range observedLogs.All() {
diff --git a/internal/ingestion/component/chunker/block_kind.go b/internal/ingestion/component/chunker/block_kind.go
new file mode 100644
index 0000000000..1a32d4ebf5
--- /dev/null
+++ b/internal/ingestion/component/chunker/block_kind.go
@@ -0,0 +1,50 @@
+// Copyright 2026 The InfiniFlow Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package chunker
+
+import "ragflow/internal/ingestion/component/schema"
+
+// structuralLayouts are the non-media block kinds a parser asserts on the
+// layout channel. They gate heading detection and line splitting; they never
+// describe content type (doc_type_kwd owns that) and are cleared before the
+// chunker output boundary.
+//
+// "title" is deliberately absent: it is the PDF models' heuristic label and
+// goes through the Title chunker's notTitle guard, while "heading" is an
+// asserted title that bypasses that guard.
+var structuralLayouts = map[string]bool{
+ "heading": true,
+ "paragraph": true,
+ "list": true,
+ "code": true,
+ "quote": true,
+}
+
+func isStructuralLayout(layout string) bool {
+ return structuralLayouts[layout]
+}
+
+// clearStructuralLayout drops the block-kind marker after a chunker has
+// merged or split the block into plain body text. PDF model layouts
+// (title/figure/equation/...) pass through unchanged.
+func clearStructuralLayout(ck *schema.ChunkDoc) {
+ if isStructuralLayout(ck.Layout) {
+ ck.Layout = ""
+ }
+}
+
+func isHeadingLayout(layout string) bool {
+ return layout == "heading"
+}
diff --git a/internal/ingestion/component/chunker/block_kind_contract_cgo_test.go b/internal/ingestion/component/chunker/block_kind_contract_cgo_test.go
new file mode 100644
index 0000000000..66e657d230
--- /dev/null
+++ b/internal/ingestion/component/chunker/block_kind_contract_cgo_test.go
@@ -0,0 +1,46 @@
+// Copyright 2026 The InfiniFlow Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+//go:build cgo
+
+package chunker
+
+import (
+ "testing"
+
+ deepdoctype "ragflow/internal/deepdoc/parser/type"
+)
+
+// TestCropTitleChunksNeverEmitsChunkStructure pins that the Title-family crop
+// path keeps the output shape identical whether or not a PDF engine is
+// available: retired ck_type and structural layout never leave the component.
+func TestCropTitleChunksNeverEmitsChunkStructure(t *testing.T) {
+ for _, engine := range []deepdoctype.PDFEngine{nil, mockCropEngine{}} {
+ in := []map[string]any{{
+ "text": "body",
+ "doc_type_kwd": "text",
+ "ck_type": "heading",
+ "layout": "heading",
+ }}
+ got := cropTitleChunks(t.Context(), engine, in)
+ for i, ck := range got {
+ if kind, ok := ck["ck_type"]; ok {
+ t.Errorf("engine=%T chunk[%d] emits retired ck_type %v: %#v", engine, i, kind, ck)
+ }
+ if layout, ok := ck["layout"]; ok {
+ t.Errorf("engine=%T chunk[%d] leaks structural layout %v: %#v", engine, i, layout, ck)
+ }
+ }
+ }
+}
diff --git a/internal/ingestion/component/chunker/block_kind_contract_test.go b/internal/ingestion/component/chunker/block_kind_contract_test.go
new file mode 100644
index 0000000000..86f7283fa6
--- /dev/null
+++ b/internal/ingestion/component/chunker/block_kind_contract_test.go
@@ -0,0 +1,223 @@
+// Copyright 2026 The InfiniFlow Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package chunker
+
+import (
+ "testing"
+
+ "ragflow/internal/ingestion/component/schema"
+)
+
+// TestChunkerOutputDropsRetiredCKType pins the output contract: ck_type is a
+// retired parser key and no chunker may emit it, even when the input payload
+// still carries a legacy ck_type.
+func TestChunkerOutputDropsRetiredCKType(t *testing.T) {
+ cases := []struct {
+ name string
+ run func(t *testing.T) []map[string]any
+ }{
+ {
+ name: "general",
+ run: func(t *testing.T) []map[string]any {
+ c, err := NewGeneralChunker(map[string]any{"chunk_token_size": 128})
+ if err != nil {
+ t.Fatalf("NewGeneralChunker: %v", err)
+ }
+ out, err := c.Invoke(t.Context(), nil, map[string]any{
+ "name": "t.md", "file_type": "md", "output_format": "json",
+ "json": []map[string]any{
+ {"text": "# Heading", "doc_type_kwd": "text", "layout": "heading", "ck_type": "heading"},
+ {"text": "body", "doc_type_kwd": "text", "ck_type": "text"},
+ },
+ })
+ if err != nil {
+ t.Fatalf("GeneralChunker.Invoke: %v", err)
+ }
+ return outputChunks(t, out)
+ },
+ },
+ {
+ name: "token",
+ run: func(t *testing.T) []map[string]any {
+ c, err := NewTokenChunker(map[string]any{"chunk_token_size": 128})
+ if err != nil {
+ t.Fatalf("NewTokenChunker: %v", err)
+ }
+ out, err := c.Invoke(t.Context(), nil, map[string]any{
+ "name": "t.md", "output_format": "json",
+ "json": []map[string]any{
+ {"text": "# Heading", "doc_type_kwd": "text", "layout": "heading", "ck_type": "heading"},
+ {"text": "body", "doc_type_kwd": "text", "ck_type": "text"},
+ },
+ })
+ if err != nil {
+ t.Fatalf("TokenChunker.Invoke: %v", err)
+ }
+ return outputChunks(t, out)
+ },
+ },
+ {
+ name: "one",
+ run: func(t *testing.T) []map[string]any {
+ c, err := NewOneChunker(nil)
+ if err != nil {
+ t.Fatalf("NewOneChunker: %v", err)
+ }
+ out, err := c.Invoke(t.Context(), nil, map[string]any{
+ "name": "t.txt", "output_format": "text", "text": "hello",
+ })
+ if err != nil {
+ t.Fatalf("OneChunker.Invoke: %v", err)
+ }
+ return outputChunks(t, out)
+ },
+ },
+ {
+ name: "title",
+ run: func(t *testing.T) []map[string]any {
+ c, err := NewGroupTitleChunker(map[string]any{"levels": [][]string{{`^# `}}})
+ if err != nil {
+ t.Fatalf("NewGroupTitleChunker: %v", err)
+ }
+ out, err := c.Invoke(t.Context(), nil, map[string]any{
+ "name": "t.md",
+ "chunks": []map[string]any{
+ {"text": "# Heading", "doc_type_kwd": "text", "layout": "heading", "ck_type": "heading"},
+ {"text": "body", "doc_type_kwd": "text", "ck_type": "text"},
+ },
+ })
+ if err != nil {
+ t.Fatalf("GroupTitleChunker.Invoke: %v", err)
+ }
+ return outputChunks(t, out)
+ },
+ },
+ }
+
+ for _, tc := range cases {
+ t.Run(tc.name, func(t *testing.T) {
+ chunks := tc.run(t)
+ if len(chunks) == 0 {
+ t.Fatal("no chunks emitted")
+ }
+ for i, ck := range chunks {
+ if kind, ok := ck["ck_type"]; ok {
+ t.Errorf("chunk[%d] emits retired ck_type %v: %#v", i, kind, ck)
+ }
+ if layout, ok := ck["layout"].(string); ok && isStructuralLayout(layout) {
+ t.Errorf("chunk[%d] leaks structural layout %q: %#v", i, layout, ck)
+ }
+ }
+ })
+ }
+}
+
+// TestChunkOutputBoundaryClearsStructuralLayout pins which layout values are
+// chunker-internal. Structural block kinds are consumed before the boundary
+// and must not leak; PDF model layouts pass through unchanged.
+func TestChunkOutputBoundaryClearsStructuralLayout(t *testing.T) {
+ out := chunkOutputs([]schema.ChunkDoc{
+ {Text: "heading", Layout: "heading"},
+ {Text: "paragraph", Layout: "paragraph"},
+ {Text: "code", Layout: "code"},
+ {Text: "figure", DocType: "image", Layout: "figure"},
+ {Text: "title", Layout: "title"},
+ {Text: "equation", Layout: "equation"},
+ })
+ chunks := outputChunks(t, out)
+ for i := 0; i < 3; i++ {
+ if layout, ok := chunks[i]["layout"]; ok {
+ t.Errorf("chunk[%d] leaked structural layout %v: %#v", i, layout, chunks[i])
+ }
+ }
+ for i, want := range []string{"figure", "title", "equation"} {
+ if got := chunks[3+i]["layout"]; got != want {
+ t.Errorf("chunk[%d] layout = %v, want %q (PDF model layout must pass through)", 3+i, got, want)
+ }
+ }
+}
+
+// TestTitleReadsLayoutNotRetiredCKType pins that a legacy ck_type alone no
+// longer promotes a line to a heading; only layout does.
+func TestTitleReadsLayoutNotRetiredCKType(t *testing.T) {
+ c, err := NewGroupTitleChunker(map[string]any{"levels": [][]string{{`^# `}}})
+ if err != nil {
+ t.Fatalf("NewGroupTitleChunker: %v", err)
+ }
+ out, err := c.Invoke(t.Context(), nil, map[string]any{
+ "name": "t.docx",
+ "chunks": []map[string]any{
+ {"text": "Introduction", "doc_type_kwd": "text", "ck_type": "heading"},
+ {"text": "Body.", "doc_type_kwd": "text"},
+ {"text": "Conclusion", "doc_type_kwd": "text", "ck_type": "heading"},
+ {"text": "Final.", "doc_type_kwd": "text"},
+ },
+ })
+ if err != nil {
+ t.Fatalf("Invoke: %v", err)
+ }
+ if chunks := outputChunks(t, out); len(chunks) != 1 {
+ t.Fatalf("legacy ck_type must not create headings: got %d chunks, want 1", len(chunks))
+ }
+}
+
+// TestMarkdownImageHeadingFoldsIntoPreviousChunk pins the base chunk
+// boundary for an image-only heading: the image is a media attachment and
+// folds into the preceding chunk.
+func TestMarkdownImageHeadingFoldsIntoPreviousChunk(t *testing.T) {
+ c, err := NewGeneralChunker(map[string]any{"chunk_token_size": 512})
+ if err != nil {
+ t.Fatalf("NewGeneralChunker: %v", err)
+ }
+ out, err := c.Invoke(t.Context(), nil, map[string]any{
+ "name": "t.md", "file_type": "md", "output_format": "json",
+ "json": []map[string]any{
+ {"text": "intro", "doc_type_kwd": "text"},
+ // An image-only heading: markdown labels it as an image but keeps
+ // the heading layout. Base folded it into the preceding chunk.
+ {"text": "##", "doc_type_kwd": "image", "image": "data:image/png;base64,AAAA", "layout": "heading"},
+ {"text": "after", "doc_type_kwd": "text"},
+ },
+ })
+ if err != nil {
+ t.Fatalf("GeneralChunker.Invoke: %v", err)
+ }
+ if chunks := outputChunks(t, out); len(chunks) != 1 {
+ t.Fatalf("image-only heading must fold into the preceding chunk: got %d chunks, want 1", len(chunks))
+ }
+}
+
+// TestTitleKeepsHTMLParagraphAtomic pins the line-split gate: a paragraph
+// layout keeps a multi-line block as one record, so an inner line that looks
+// like a heading cannot create a spurious section.
+func TestTitleKeepsHTMLParagraphAtomic(t *testing.T) {
+ c, err := NewGroupTitleChunker(map[string]any{"levels": [][]string{{`^# `}}})
+ if err != nil {
+ t.Fatalf("NewGroupTitleChunker: %v", err)
+ }
+ out, err := c.Invoke(t.Context(), nil, map[string]any{
+ "name": "t.html",
+ "chunks": []map[string]any{
+ {"text": "intro\n# not a heading", "doc_type_kwd": "text", "layout": "paragraph"},
+ {"text": "body", "doc_type_kwd": "text"},
+ },
+ })
+ if err != nil {
+ t.Fatalf("Invoke: %v", err)
+ }
+ if chunks := outputChunks(t, out); len(chunks) != 1 {
+ t.Fatalf("paragraph must stay atomic: got %d chunks, want 1", len(chunks))
+ }
+}
diff --git a/internal/ingestion/component/chunker/common.go b/internal/ingestion/component/chunker/common.go
index ea99778323..4ef02a6645 100644
--- a/internal/ingestion/component/chunker/common.go
+++ b/internal/ingestion/component/chunker/common.go
@@ -216,15 +216,10 @@ func itemDocType(it schema.ChunkDoc) string {
return "text"
}
-// isMediaChunk reports whether a chunk is an image or table region. It checks
-// CKType first (general/token paths set it) and falls back to DocType
-// (group/hierarchy forward parser output that carries only doc_type_kwd), so
-// it classifies media regardless of which chunker produced the chunk.
+// isMediaChunk reports whether a chunk is an image or table region. The
+// content type lives on doc_type_kwd; layout never describes media.
func isMediaChunk(ck schema.ChunkDoc) bool {
- typ := strings.ToLower(strings.TrimSpace(ck.CKType))
- if typ == "" {
- typ = strings.ToLower(strings.TrimSpace(ck.DocType))
- }
+ typ := itemDocType(ck)
return typ == "image" || typ == "table"
}
@@ -310,6 +305,11 @@ func chunkOutputs(chunks []schema.ChunkDoc) map[string]any {
materialized := make([]schema.ChunkDoc, len(chunks))
for i := range chunks {
materialized[i] = materializeMediaContext(chunks[i])
+ // Structural layout is chunker-internal: it is consumed by merge,
+ // split, and crop, and must not cross the output boundary. PDF model
+ // layouts (title/figure/equation/...) are not structural and pass
+ // through unchanged.
+ clearStructuralLayout(&materialized[i])
}
return map[string]any{
"output_format": "chunks",
diff --git a/internal/ingestion/component/chunker/coverage_blind_spots_test.go b/internal/ingestion/component/chunker/coverage_blind_spots_test.go
index 312583b740..5baa972172 100644
--- a/internal/ingestion/component/chunker/coverage_blind_spots_test.go
+++ b/internal/ingestion/component/chunker/coverage_blind_spots_test.go
@@ -25,7 +25,7 @@ import (
)
func TestSplitTextParserSentencesKeepsSentenceUnits(t *testing.T) {
- item := schema.ChunkDoc{Text: "First!Second?第三。Fourth;", DocType: "text", CKType: "text"}
+ item := schema.ChunkDoc{Text: "First!Second?第三。Fourth;", DocType: "text"}
parts := splitTextParserSentences(item)
if got, want := len(parts), 4; got != want {
t.Fatalf("sentence parts = %d, want %d: %#v", got, want, parts)
@@ -136,8 +136,8 @@ func TestGeneralChunkerMarkdownLongHeadingKeepsFollowingTableAtomic(t *testing.T
"file_type": "md",
"output_format": "json",
"json": []map[string]any{
- {"text": longHeading, "doc_type_kwd": "text", "ck_type": "heading"},
- {"text": "<table><tr><td>A</td></tr></table>", "doc_type_kwd": "table", "ck_type": "table"},
+ {"text": longHeading, "doc_type_kwd": "text", "layout": "heading"},
+ {"text": "<table><tr><td>A</td></tr></table>", "doc_type_kwd": "table"},
},
})
if err != nil {
@@ -147,10 +147,10 @@ func TestGeneralChunkerMarkdownLongHeadingKeepsFollowingTableAtomic(t *testing.T
if len(chunks) != 2 {
t.Fatalf("chunks = %#v, want separate heading and table chunks", chunks)
}
- if chunks[0]["ck_type"] != "heading" || chunks[0]["text"] != strings.TrimSpace(longHeading) {
+ if chunks[0]["doc_type_kwd"] != "text" || chunks[0]["text"] != strings.TrimSpace(longHeading) {
t.Errorf("heading chunk = %#v", chunks[0])
}
- if chunks[1]["ck_type"] != "table" || chunks[1]["text"] != "<table><tr><td>A</td></tr></table>" {
+ if chunks[1]["doc_type_kwd"] != "table" || chunks[1]["text"] != "<table><tr><td>A</td></tr></table>" {
t.Errorf("table chunk = %#v", chunks[1])
}
}
diff --git a/internal/ingestion/component/chunker/crop_title_test.go b/internal/ingestion/component/chunker/crop_title_test.go
index a64acd4a86..58256d756f 100644
--- a/internal/ingestion/component/chunker/crop_title_test.go
+++ b/internal/ingestion/component/chunker/crop_title_test.go
@@ -24,11 +24,10 @@ import (
// TestCropTitleChunks_CropsPositionedChunk pins residual A: chunks
// produced by the Title/Group/Hierarchy chunkers must be cropped on demand
// (mirroring the TokenChunker JSON path). The chunk shape mirrors the real
-// Group/Hierarchy output: it carries doc_type_kwd + positions but NO ck_type
-// (buildChunksFromRecordGroups never sets ck_type). cropTitleChunks must
-// derive ck_type from doc_type_kwd so needsCrop (pdfcrop_cgo.go:151) fires.
+// Group/Hierarchy output: it carries doc_type_kwd + positions and no
+// block-kind field. cropImageChunks derives the crop type from doc_type_kwd.
// A text chunk with positions gets a rendered preview; a chunk without
-// positions is left untouched; and the derived ck_type must not leak into
+// positions is left untouched; and the retired ck_type must not appear in
// the returned chunk shape.
func TestCropTitleChunks_CropsPositionedChunk(t *testing.T) {
ctx := context.Background()
@@ -50,7 +49,7 @@ func TestCropTitleChunks_CropsPositionedChunk(t *testing.T) {
t.Errorf("chunk without positions should not be cropped, got %v", got[1]["image"])
}
if _, ok := got[0]["ck_type"]; ok {
- t.Errorf("derived ck_type must not leak into output chunk: %v", got[0])
+ t.Errorf("retired ck_type must not appear in output chunk: %v", got[0])
}
}
diff --git a/internal/ingestion/component/chunker/custom_delim_test.go b/internal/ingestion/component/chunker/custom_delim_test.go
index 382a5e0c94..06ba1f33ca 100644
--- a/internal/ingestion/component/chunker/custom_delim_test.go
+++ b/internal/ingestion/component/chunker/custom_delim_test.go
@@ -14,8 +14,7 @@ import (
//
// Plain text/markdown/html inputs must not gain doc_type_kwd merely because a
// custom delimiter is configured: Python emits only text for those paths.
-// Structured JSON inputs keep their source doc_type_kwd, while ck_type remains
-// on Go text chunks for downstream crop dispatch.
+// Structured JSON inputs keep their source doc_type_kwd.
const backtickNewline = "`\n`"
@@ -54,16 +53,16 @@ func assertPlainCustomDelimiterMetadata(t *testing.T, chunks []map[string]any) {
if _, ok := chunk["doc_type_kwd"]; ok {
t.Errorf("chunk[%d] must not synthesize doc_type_kwd: %v", i, chunk)
}
- if got := chunk["ck_type"]; got != "text" {
- t.Errorf("chunk[%d] ck_type: want %q got %v", i, "text", got)
+ if _, ok := chunk["ck_type"]; ok {
+ t.Errorf("chunk[%d] must not emit the retired ck_type: %v", i, chunk)
}
}
}
-// TestCustomDelimTextChildrenKeepCKType covers primary and child splitting in
-// combination: child chunks remain unstructured but retain Go's live CKType,
-// and every delimiter (primary "::" and children "--") is kept.
-func TestCustomDelimTextChildrenKeepCKType(t *testing.T) {
+// TestCustomDelimTextChildrenKeepDelimiters covers primary and child splitting
+// in combination: child chunks remain unstructured and every delimiter
+// (primary "::" and children "--") is kept.
+func TestCustomDelimTextChildrenKeepDelimiters(t *testing.T) {
params := map[string]any{
"chunk_token_size": float64(128),
"delimiters": []string{"`::`"},
diff --git a/internal/ingestion/component/chunker/general.go b/internal/ingestion/component/chunker/general.go
index 57c03cc6d7..9a63747458 100644
--- a/internal/ingestion/component/chunker/general.go
+++ b/internal/ingestion/component/chunker/general.go
@@ -266,7 +266,6 @@ func (c *GeneralChunkerComponent) chunkPDF(ctx context.Context, db *gorm.DB, ups
media := cloneChunkDoc(unit)
media.Text = normalizeGeneralNewlines(media.Text)
media.DocType = itemDocType(media)
- media.CKType = media.DocType
chunks = append(chunks, media)
}
}
@@ -491,7 +490,6 @@ func groupDOCXUnitsByRecordBoundary(units []schema.ChunkDoc, recordPrefixes []st
flush()
media := cloneChunkDoc(unit)
media.DocType = itemDocType(media)
- media.CKType = media.DocType
grouped = append(grouped, media)
continue
}
@@ -500,7 +498,7 @@ func groupDOCXUnitsByRecordBoundary(units []schema.ChunkDoc, recordPrefixes []st
}
text := cloneChunkDoc(unit)
text.DocType = "text"
- text.CKType = "text"
+ clearStructuralLayout(&text)
if docxTextStartsRecordBoundary(text.Text, recordPrefixes) {
sawBoundary = true
flush()
@@ -742,7 +740,7 @@ type markdownSplit struct {
}
// splitMarkdownUnits expands only text units while retaining the semantic
-// parser type on every piece. Markdown heading detection depends on ck_type,
+// parser type on every piece. Markdown heading detection depends on layout,
// so the generic splitter's type normalization cannot be used here.
func splitMarkdownUnits(units []schema.ChunkDoc, pattern *regexp.Regexp, keepDelimiter bool) markdownSplit {
result := markdownSplit{
@@ -753,12 +751,15 @@ func splitMarkdownUnits(units []schema.ChunkDoc, pattern *regexp.Regexp, keepDel
for _, unit := range units {
unit = cloneChunkDoc(unit)
unit.Text = normalizeGeneralNewlines(itemTextOrFallback(unit))
- if !keepDelimiter || unit.CKType == "heading" {
+ if !keepDelimiter || isHeadingLayout(unit.Layout) {
unit.Text = strings.TrimSpace(unit.Text)
}
unit.DocType = itemDocType(unit)
+ // Media units are content attachments, not block-kind boundaries.
+ // Base normalized their CKType to DocType here, so an image-only
+ // heading folded into the preceding chunk; preserve that behavior.
if unit.DocType == "table" || unit.DocType == "image" {
- unit.CKType = unit.DocType
+ clearStructuralLayout(&unit)
}
shortHeading := isShortMarkdownHeading(unit)
if unit.DocType != "text" || pattern == nil || !pattern.MatchString(unit.Text) {
@@ -838,7 +839,7 @@ func mergeMarkdownUnits(parts markdownSplit, target int, overlapPct float64) []s
joinSep = "\n"
}
if itemDocType(unit) == "table" {
- if current >= 0 && merged[current].CKType == "heading" && previousShortHeading {
+ if current >= 0 && isHeadingLayout(merged[current].Layout) && previousShortHeading {
mergedImage, imagesOK := mergeMarkdownImagesChecked(merged[current].Image, unit.Image)
if !imagesOK {
merged = append(merged, cloneChunkDoc(unit))
@@ -859,7 +860,6 @@ func mergeMarkdownUnits(parts markdownSplit, target int, overlapPct float64) []s
mergeGeneralMetadata(&table, heading)
table.Image = mergedImage
table.DocType = "table"
- table.CKType = "table"
table.ContextAbove = stripTextLinesFromMediaContextAbove(table.ContextAbove, heading.Text)
merged[current] = table
current = -1
@@ -879,9 +879,6 @@ func mergeMarkdownUnits(parts markdownSplit, target int, overlapPct float64) []s
}
standalone := cloneChunkDoc(unit)
standalone.DocType = "image"
- if standalone.CKType == "" {
- standalone.CKType = "image"
- }
standalone.TKNums = intPtr(generalUnitTokens(standalone))
merged = append(merged, standalone)
continue
@@ -889,9 +886,6 @@ func mergeMarkdownUnits(parts markdownSplit, target int, overlapPct float64) []s
unit = cloneChunkDoc(unit)
unit.DocType = "text"
- if unit.CKType == "" {
- unit.CKType = "text"
- }
unit.TKNums = intPtr(generalUnitTokens(unit))
if current < 0 {
merged = append(merged, unit)
@@ -902,13 +896,13 @@ func mergeMarkdownUnits(parts markdownSplit, target int, overlapPct float64) []s
previous := &merged[current]
unitTokens := generalUnitTokens(unit)
- if startsBlock && unit.CKType == "heading" && previous.CKType != "heading" {
+ if startsBlock && isHeadingLayout(unit.Layout) && !isHeadingLayout(previous.Layout) {
merged = append(merged, unit)
current = len(merged) - 1
currentTokens = unitTokens
continue
}
- forceMerge := previous.CKType == "heading" && previousShortHeading
+ forceMerge := isHeadingLayout(previous.Layout) && previousShortHeading
mergedImage, imagesOK := mergeMarkdownImagesChecked(previous.Image, unit.Image)
if !imagesOK {
merged = append(merged, unit)
@@ -940,8 +934,8 @@ func mergeMarkdownUnits(parts markdownSplit, target int, overlapPct float64) []s
mergeGeneralChunk(previous, unit, joinSep)
previous.Image = mergedImage
previous.DocType = "text"
- if startsBlock || unit.CKType != "heading" {
- previous.CKType = "text"
+ if startsBlock || !isHeadingLayout(unit.Layout) {
+ clearStructuralLayout(previous)
}
currentTokens = projected
previous.TKNums = intPtr(currentTokens)
@@ -950,7 +944,7 @@ func mergeMarkdownUnits(parts markdownSplit, target int, overlapPct float64) []s
}
func isShortMarkdownHeading(unit schema.ChunkDoc) bool {
- if unit.CKType != "heading" {
+ if !isHeadingLayout(unit.Layout) {
return false
}
return tokenizeStr(markdownHeadingContent(unit.Text)) < 50
@@ -1190,7 +1184,7 @@ func mergeGeneralUnits(units []schema.ChunkDoc, target int, overlapPct float64,
count := generalUnitTokens(unit)
unit.DocType = "text"
- unit.CKType = "text"
+ clearStructuralLayout(&unit)
unit.TKNums = intPtr(count)
if count > target {
merged = append(merged, cloneChunkDoc(unit))
@@ -1335,7 +1329,7 @@ func splitGeneralUnitsPreservingRecordPrefix(units []schema.ChunkDoc, pattern *r
unit = cloneChunkDoc(unit)
unit.Text = normalizeGeneralNewlines(itemTextOrFallback(unit))
unit.DocType = itemDocType(unit)
- unit.CKType = unit.DocType
+ clearStructuralLayout(&unit)
prefix, rest := peelLeadingRecordPrefix(unit.Text, recordPrefixes)
if unit.DocType != "text" || pattern == nil || !pattern.MatchString(rest) {
unit.Text = prefix + rest
@@ -1387,7 +1381,7 @@ func splitGeneralUnits(units []schema.ChunkDoc, pattern *regexp.Regexp, keepDeli
unit = cloneChunkDoc(unit)
unit.Text = normalizeGeneralNewlines(itemTextOrFallback(unit))
unit.DocType = itemDocType(unit)
- unit.CKType = unit.DocType
+ clearStructuralLayout(&unit)
if unit.DocType != "text" || pattern == nil || !pattern.MatchString(unit.Text) {
unit.TKNums = intPtr(tokenizeStr(unit.Text))
result = append(result, unit)
diff --git a/internal/ingestion/component/chunker/general_block_delimiter_test.go b/internal/ingestion/component/chunker/general_block_delimiter_test.go
index db51ad6ad8..bece09f171 100644
--- a/internal/ingestion/component/chunker/general_block_delimiter_test.go
+++ b/internal/ingestion/component/chunker/general_block_delimiter_test.go
@@ -145,15 +145,15 @@ func TestGeneralChunkerPDFAndMarkdownCustomMediaContextDropsMarker(t *testing.T)
func TestGeneralChunkerMarkdownPunctuationPreservesMediaBoundaries(t *testing.T) {
for _, tc := range []struct {
- name, docType, ckType, first, media string
+ name, docType, layout, first, media string
contextSize int
want []string
}{
{"heading and table", "table", "heading", "标题。还有标题!", "<table>A</table>", 512,
[]string{"标题。还有标题!\n<table>A</table>后文。还有一句。", "后文。还有一句。"}},
- {"image attachment", "image", "text", "前文。还有一句。", "图注。", 0,
+ {"image attachment", "image", "", "前文。还有一句。", "图注。", 0,
[]string{"前文。还有一句。\n图注。\n后文。还有一句。"}},
- {"standalone image", "image", "text", "前文。还有一句。", "图注。", 512,
+ {"standalone image", "image", "", "前文。还有一句。", "图注。", 512,
[]string{"前文。还有一句。", "前文。还有一句。图注。后文。还有一句。", "后文。还有一句。"}},
} {
t.Run(tc.name, func(t *testing.T) {
@@ -164,10 +164,14 @@ func TestGeneralChunkerMarkdownPunctuationPreservesMediaBoundaries(t *testing.T)
if err != nil {
t.Fatal(err)
}
+ first := map[string]any{"text": tc.first, "doc_type_kwd": "text"}
+ if tc.layout != "" {
+ first["layout"] = tc.layout
+ }
out, err := c.Invoke(t.Context(), nil, map[string]any{
"name": "document.md", "file_type": "md", "output_format": "json",
"json": []map[string]any{
- {"text": tc.first, "doc_type_kwd": "text", "ck_type": tc.ckType},
+ first,
{"text": tc.media, "doc_type_kwd": tc.docType, "img_id": "media-id"},
{"text": "后文。还有一句。", "doc_type_kwd": "text"},
},
diff --git a/internal/ingestion/component/chunker/general_context_regression_test.go b/internal/ingestion/component/chunker/general_context_regression_test.go
index e33dc23f0f..f12a4d0ed8 100644
--- a/internal/ingestion/component/chunker/general_context_regression_test.go
+++ b/internal/ingestion/component/chunker/general_context_regression_test.go
@@ -60,7 +60,7 @@ func TestGeneralChunkerMarkdownFoldedHeadingDropsPartialContext(t *testing.T) {
out, err := c.Invoke(t.Context(), nil, map[string]any{
"name": "test.md", "file_type": "md", "output_format": "json",
"json": []map[string]any{
- {"text": "# 标题。还有标题!", "doc_type_kwd": "text", "ck_type": "heading"},
+ {"text": "# 标题。还有标题!", "doc_type_kwd": "text", "layout": "heading"},
{"text": "<table>A</table>", "doc_type_kwd": "table"},
},
})
@@ -91,7 +91,7 @@ func TestGeneralChunkerMarkdownLongHeadingUsesSourceClassification(t *testing.T)
out, err := c.Invoke(t.Context(), nil, map[string]any{
"name": "test.md", "file_type": "md", "output_format": "json",
"json": []map[string]any{
- {"text": heading, "doc_type_kwd": "text", "ck_type": "heading"},
+ {"text": heading, "doc_type_kwd": "text", "layout": "heading"},
{"text": "body", "doc_type_kwd": "text"},
},
})
diff --git a/internal/ingestion/component/chunker/general_merge_test.go b/internal/ingestion/component/chunker/general_merge_test.go
index 6717a209fb..20e9924574 100644
--- a/internal/ingestion/component/chunker/general_merge_test.go
+++ b/internal/ingestion/component/chunker/general_merge_test.go
@@ -28,9 +28,9 @@ import (
func TestMergeGeneralUnitsAllowsOneBoundaryOverflow(t *testing.T) {
units := []schema.ChunkDoc{
- {Text: "alpha", DocType: "text", CKType: "text", TKNums: intPtr(2)},
- {Text: "beta", DocType: "text", CKType: "text", TKNums: intPtr(2)},
- {Text: "gamma", DocType: "text", CKType: "text", TKNums: intPtr(2)},
+ {Text: "alpha", DocType: "text", TKNums: intPtr(2)},
+ {Text: "beta", DocType: "text", TKNums: intPtr(2)},
+ {Text: "gamma", DocType: "text", TKNums: intPtr(2)},
}
got := mergeGeneralUnits(units, 3, 0, "\n")
@@ -58,8 +58,8 @@ func TestSameSpreadsheetTableUsesPositionSheet(t *testing.T) {
func TestMergeGeneralUnitsMergesWhenCurrentEqualsCap(t *testing.T) {
units := []schema.ChunkDoc{
- {Text: "at cap", DocType: "text", CKType: "text", TKNums: intPtr(3)},
- {Text: "overflow", DocType: "text", CKType: "text", TKNums: intPtr(1)},
+ {Text: "at cap", DocType: "text", TKNums: intPtr(3)},
+ {Text: "overflow", DocType: "text", TKNums: intPtr(1)},
}
got := mergeGeneralUnits(units, 3, 0, "\n")
@@ -70,9 +70,9 @@ func TestMergeGeneralUnitsMergesWhenCurrentEqualsCap(t *testing.T) {
func TestMergeGeneralUnitsKeepsOversizedAtomWhole(t *testing.T) {
units := []schema.ChunkDoc{
- {Text: "before", DocType: "text", CKType: "text", TKNums: intPtr(2)},
- {Text: "oversized atom stays whole", DocType: "text", CKType: "text", TKNums: intPtr(8)},
- {Text: "after", DocType: "text", CKType: "text", TKNums: intPtr(1)},
+ {Text: "before", DocType: "text", TKNums: intPtr(2)},
+ {Text: "oversized atom stays whole", DocType: "text", TKNums: intPtr(8)},
+ {Text: "after", DocType: "text", TKNums: intPtr(1)},
}
got := mergeGeneralUnits(units, 3, 0, "\n")
@@ -83,9 +83,9 @@ func TestMergeGeneralUnitsKeepsOversizedAtomWhole(t *testing.T) {
func TestMergeGeneralUnitsAppliesUnconditionalCharacterOverlap(t *testing.T) {
units := []schema.ChunkDoc{
- {Text: "abcdefghij@@1\t0\t1\t2\t3##", DocType: "text", CKType: "text", TKNums: intPtr(2)},
- {Text: "klmnopqrst", DocType: "text", CKType: "text", TKNums: intPtr(2)},
- {Text: "uvwxyzABCD", DocType: "text", CKType: "text", TKNums: intPtr(2)},
+ {Text: "abcdefghij@@1\t0\t1\t2\t3##", DocType: "text", TKNums: intPtr(2)},
+ {Text: "klmnopqrst", DocType: "text", TKNums: intPtr(2)},
+ {Text: "uvwxyzABCD", DocType: "text", TKNums: intPtr(2)},
}
got := mergeGeneralUnits(units, 2, 50, "\n")
@@ -106,9 +106,9 @@ func TestApplyGeneralOverlapDoesNotAccumulatePositions(t *testing.T) {
return json.RawMessage(fmt.Sprintf("[[%d,0,10,0,5]]", page))
}
chunks := []schema.ChunkDoc{
- {Text: "alpha beta", DocType: "text", CKType: "text", PDFPositions: position(1), Positions: position(1)},
- {Text: "gamma delta", DocType: "text", CKType: "text", PDFPositions: position(2), Positions: position(2)},
- {Text: "epsilon zeta", DocType: "text", CKType: "text", PDFPositions: position(3), Positions: position(3)},
+ {Text: "alpha beta", DocType: "text", PDFPositions: position(1), Positions: position(1)},
+ {Text: "gamma delta", DocType: "text", PDFPositions: position(2), Positions: position(2)},
+ {Text: "epsilon zeta", DocType: "text", PDFPositions: position(3), Positions: position(3)},
}
got := applyGeneralOverlap(chunks, 50, "\n")
@@ -125,10 +125,10 @@ func TestApplyGeneralOverlapDoesNotAccumulatePositions(t *testing.T) {
func TestMergeMarkdownUnitsCarriesCharacterOverlapIntoNextBudget(t *testing.T) {
units := []schema.ChunkDoc{
- {Text: "alpha beta", DocType: "text", CKType: "text"},
- {Text: "gamma delta", DocType: "text", CKType: "text"},
- {Text: "epsilon zeta", DocType: "text", CKType: "text"},
- {Text: "eta theta", DocType: "text", CKType: "text"},
+ {Text: "alpha beta", DocType: "text"},
+ {Text: "gamma delta", DocType: "text"},
+ {Text: "epsilon zeta", DocType: "text"},
+ {Text: "eta theta", DocType: "text"},
}
for i := range units {
units[i].TKNums = intPtr(tokenizeStr(units[i].Text))
@@ -153,7 +153,6 @@ func TestMergeMarkdownUnitsOverlapKeepsCurrentChunkMetadata(t *testing.T) {
{
Text: "alpha beta",
DocType: "text",
- CKType: "text",
Image: "previous-image",
ImgID: "previous-image-id",
PageNumber: &previousPage,
@@ -163,7 +162,6 @@ func TestMergeMarkdownUnitsOverlapKeepsCurrentChunkMetadata(t *testing.T) {
{
Text: "gamma delta",
DocType: "text",
- CKType: "text",
PageNumber: &currentPage,
Positions: json.RawMessage(`[[0,0,10,20,30]]`),
Extra: map[string]json.RawMessage{"current": json.RawMessage(`true`)},
@@ -203,11 +201,10 @@ func TestMergeMarkdownUnitsOverlapKeepsCurrentChunkMetadata(t *testing.T) {
func TestMergeMarkdownUnitsStripsFoldedHeadingFromTableContextAbove(t *testing.T) {
units := []schema.ChunkDoc{
- {Text: "## Section", DocType: "text", CKType: "heading", TKNums: intPtr(1)},
+ {Text: "## Section", DocType: "text", Layout: "heading", TKNums: intPtr(1)},
{
Text: "<table></table>",
DocType: "table",
- CKType: "table",
ContextAbove: "## Section",
TKNums: intPtr(1),
},
@@ -226,12 +223,11 @@ func TestMergeMarkdownUnitsStripsFoldedHeadingFromTableContextAbove(t *testing.T
func TestMergeMarkdownUnitsStripsOnlyFoldedHeadingFromTableContextAbove(t *testing.T) {
units := []schema.ChunkDoc{
- {Text: "intro", DocType: "text", CKType: "text", TKNums: intPtr(1)},
- {Text: "## Section", DocType: "text", CKType: "heading", TKNums: intPtr(1)},
+ {Text: "intro", DocType: "text", TKNums: intPtr(1)},
+ {Text: "## Section", DocType: "text", Layout: "heading", TKNums: intPtr(1)},
{
Text: "<table></table>",
DocType: "table",
- CKType: "table",
ContextAbove: "intro\n## Section",
TKNums: intPtr(1),
},
@@ -247,32 +243,31 @@ func TestMergeMarkdownUnitsStripsOnlyFoldedHeadingFromTableContextAbove(t *testi
func TestMergeMarkdownUnitsKeepsImageWithMediaContextStandalone(t *testing.T) {
units := []schema.ChunkDoc{
- {Text: "before", DocType: "text", CKType: "text", TKNums: intPtr(1)},
+ {Text: "before", DocType: "text", TKNums: intPtr(1)},
{
Text: "figure",
DocType: "image",
- CKType: "image",
ContextAbove: "before",
ContextBelow: "after",
Image: "data:image/png;base64,AAAA",
TKNums: intPtr(1),
},
- {Text: "after", DocType: "text", CKType: "text", TKNums: intPtr(1)},
+ {Text: "after", DocType: "text", TKNums: intPtr(1)},
}
got := mergeMarkdownUnits(markdownSplit{chunks: units, blockStarts: []bool{true, true, true}, shortHeadings: []bool{false, false, false}}, 10, 0)
if len(got) != 3 {
t.Fatalf("chunks = %d, want text, image, text", len(got))
}
- if got[1].CKType != "image" || got[1].ContextAbove != "before" || got[1].ContextBelow != "after" {
+ if itemDocType(got[1]) != "image" || got[1].ContextAbove != "before" || got[1].ContextBelow != "after" {
t.Fatalf("image chunk = %#v, want preserved media context", got[1])
}
}
func TestMergeMarkdownUnitsStartsNewChunkForIncomingHeading(t *testing.T) {
units := []schema.ChunkDoc{
- {Text: "Background details", DocType: "text", CKType: "text", TKNums: intPtr(1)},
- {Text: "## Architecture", DocType: "text", CKType: "heading", TKNums: intPtr(1)},
- {Text: "The architecture is split into parser and chunker.", DocType: "text", CKType: "text", TKNums: intPtr(1)},
+ {Text: "Background details", DocType: "text", TKNums: intPtr(1)},
+ {Text: "## Architecture", DocType: "text", Layout: "heading", TKNums: intPtr(1)},
+ {Text: "The architecture is split into parser and chunker.", DocType: "text", TKNums: intPtr(1)},
}
got := mergeMarkdownUnits(markdownSplit{chunks: units, blockStarts: []bool{true, true, true}, shortHeadings: []bool{false, true, false}}, 3, 0)
@@ -282,14 +277,14 @@ func TestMergeMarkdownUnitsStartsNewChunkForIncomingHeading(t *testing.T) {
}) {
t.Fatalf("markdown chunks = %q, want body and heading/body chunks", texts)
}
- if got[1].CKType != "text" {
- t.Fatalf("heading/body chunk CKType = %q, want text after heading absorption", got[1].CKType)
+ if got[1].Layout != "" {
+ t.Fatalf("heading/body chunk Layout = %q, want cleared after heading absorption", got[1].Layout)
}
}
func TestShortMarkdownHeadingBudgetIgnoresATXMarker(t *testing.T) {
content := strings.TrimSpace(strings.Repeat("word ", 49))
- unit := schema.ChunkDoc{CKType: "heading", Text: "## " + content}
+ unit := schema.ChunkDoc{Layout: "heading", Text: "## " + content}
if tokenizeStr(content) >= 50 {
t.Fatalf("test heading content unexpectedly reached threshold: %d", tokenizeStr(content))
}
@@ -304,9 +299,9 @@ func TestShortMarkdownHeadingBudgetIgnoresATXMarker(t *testing.T) {
func TestMergeGeneralUnitsOverlapDoesNotCopyPreviousMediaMetadata(t *testing.T) {
previousPage := 4
units := []schema.ChunkDoc{
- {Text: "first", DocType: "text", CKType: "text", TKNums: intPtr(2), Image: "previous-image", ImgID: "previous-image-id", PageNumber: &previousPage, Extra: map[string]json.RawMessage{"source": json.RawMessage(`"previous"`)}},
- {Text: "second", DocType: "text", CKType: "text", TKNums: intPtr(2)},
- {Text: "third", DocType: "text", CKType: "text", TKNums: intPtr(2)},
+ {Text: "first", DocType: "text", TKNums: intPtr(2), Image: "previous-image", ImgID: "previous-image-id", PageNumber: &previousPage, Extra: map[string]json.RawMessage{"source": json.RawMessage(`"previous"`)}},
+ {Text: "second", DocType: "text", TKNums: intPtr(2)},
+ {Text: "third", DocType: "text", TKNums: intPtr(2)},
}
got := mergeGeneralUnits(units, 4, 50, "\n")
@@ -337,16 +332,16 @@ func TestTakeContextSentencesPreservesBoundarySelection(t *testing.T) {
func TestMergeGeneralUnitsKeepsMediaAtomicAndBreaksTextRun(t *testing.T) {
units := []schema.ChunkDoc{
- {Text: "before", DocType: "text", CKType: "text", TKNums: intPtr(1)},
- {Text: "A|B", DocType: "table", CKType: "table", TKNums: intPtr(2)},
- {Text: "after", DocType: "text", CKType: "text", TKNums: intPtr(1)},
+ {Text: "before", DocType: "text", TKNums: intPtr(1)},
+ {Text: "A|B", DocType: "table", TKNums: intPtr(2)},
+ {Text: "after", DocType: "text", TKNums: intPtr(1)},
}
got := mergeGeneralUnits(units, 10, 0, "\n")
if texts := generalChunkTexts(got); !reflect.DeepEqual(texts, []string{"before", "A|B", "after"}) {
t.Fatalf("texts = %q", texts)
}
- if got[1].DocType != "table" || got[1].CKType != "table" {
+ if itemDocType(got[1]) != "table" {
t.Errorf("table type changed: %#v", got[1])
}
}
@@ -355,7 +350,6 @@ func TestMergeGeneralUnitsCombinesPositionsAndMetadata(t *testing.T) {
first := schema.ChunkDoc{
Text: "alpha",
DocType: "text",
- CKType: "text",
TKNums: intPtr(1),
Image: "image-a",
Positions: json.RawMessage(`[[0,0,10,0,10]]`),
@@ -366,7 +360,6 @@ func TestMergeGeneralUnitsCombinesPositionsAndMetadata(t *testing.T) {
second := schema.ChunkDoc{
Text: "beta",
DocType: "text",
- CKType: "text",
TKNums: intPtr(1),
Positions: json.RawMessage(`[[0,0,10,20,30]]`),
PDFPositions: json.RawMessage(`[[0,0,10,20,30]]`),
diff --git a/internal/ingestion/component/chunker/general_positions_slice_test.go b/internal/ingestion/component/chunker/general_positions_slice_test.go
index e1f1c6b87e..d0c3ca21e9 100644
--- a/internal/ingestion/component/chunker/general_positions_slice_test.go
+++ b/internal/ingestion/component/chunker/general_positions_slice_test.go
@@ -49,7 +49,6 @@ func TestSplitGeneralChildrenSlicesPositionsAbut(t *testing.T) {
chunk := schema.ChunkDoc{
Text: "aaa;bbb",
DocType: "text",
- CKType: "text",
PDFPositions: json.RawMessage(`[[1,10,20,0,100]]`),
Positions: json.RawMessage(`[[1,10,20,0,100]]`),
}
diff --git a/internal/ingestion/component/chunker/general_test.go b/internal/ingestion/component/chunker/general_test.go
index c337efafb0..093e99f5cf 100644
--- a/internal/ingestion/component/chunker/general_test.go
+++ b/internal/ingestion/component/chunker/general_test.go
@@ -80,7 +80,6 @@ func spreadsheetSegmentItem(sheet string, header []string, rows [][]string, shee
// so text-bound assertions (e.g. TokenChunker boundary tests) stay valid.
"text": tableutil.RenderTableHTML(td),
"doc_type_kwd": "table",
- "ck_type": "table",
"sheet_index": sheetIndex,
"positions": matrix,
}
@@ -319,8 +318,8 @@ func TestGeneralChunkerMarkdownShortHeadingForcesNextUnit(t *testing.T) {
"file_type": "md",
"output_format": "json",
"json": []map[string]any{
- {"text": "Title", "doc_type_kwd": "text", "ck_type": "heading"},
- {"text": "body", "doc_type_kwd": "text", "ck_type": "text"},
+ {"text": "Title", "doc_type_kwd": "text", "layout": "heading"},
+ {"text": "body", "doc_type_kwd": "text"},
},
})
if err != nil {
@@ -346,8 +345,8 @@ func TestGeneralChunkerMarkdownLongHeadingUsesNormalCap(t *testing.T) {
"file_type": "md",
"output_format": "json",
"json": []map[string]any{
- {"text": longHeading, "doc_type_kwd": "text", "ck_type": "heading"},
- {"text": "body", "doc_type_kwd": "text", "ck_type": "text"},
+ {"text": longHeading, "doc_type_kwd": "text", "layout": "heading"},
+ {"text": "body", "doc_type_kwd": "text"},
},
})
if err != nil {
@@ -375,7 +374,7 @@ func TestGeneralChunkerMarkdownImageMergesWithTextAndPreservesPayload(t *testing
"file_type": "md",
"output_format": "json",
"json": []map[string]any{
- {"text": "Title", "doc_type_kwd": "text", "ck_type": "heading"},
+ {"text": "Title", "doc_type_kwd": "text", "layout": "heading"},
{"text": "figure caption", "doc_type_kwd": "image", "image": "data:image/png;base64,AAAA"},
},
})
@@ -516,9 +515,9 @@ func TestGeneralChunkerMarkdownShortHeadingKeepsFollowingTableAtomic(t *testing.
"file_type": "md",
"output_format": "json",
"json": []map[string]any{
- {"text": "Title", "doc_type_kwd": "text", "ck_type": "heading"},
- {"text": "<table><tr><td>A</td></tr></table>", "doc_type_kwd": "table", "ck_type": "table"},
- {"text": "body", "doc_type_kwd": "text", "ck_type": "text"},
+ {"text": "Title", "doc_type_kwd": "text", "layout": "heading"},
+ {"text": "<table><tr><td>A</td></tr></table>", "doc_type_kwd": "table"},
+ {"text": "body", "doc_type_kwd": "text"},
},
})
if err != nil {
@@ -534,9 +533,6 @@ func TestGeneralChunkerMarkdownShortHeadingKeepsFollowingTableAtomic(t *testing.
if got := chunks[0]["doc_type_kwd"]; got != "table" {
t.Errorf("table doc_type_kwd = %v, want table", got)
}
- if got := chunks[0]["ck_type"]; got != "table" {
- t.Errorf("table ck_type = %v, want table", got)
- }
}
func TestGeneralChunkerDOCXMediaDoesNotBreakTextMerge(t *testing.T) {
@@ -639,7 +635,7 @@ func TestGeneralChunkerMarkdownTableDoesNotDuplicateShortHeadingInContext(t *tes
"file_type": "md",
"output_format": "json",
"json": []map[string]any{
- {"text": "## Section", "doc_type_kwd": "text", "ck_type": "heading"},
+ {"text": "## Section", "doc_type_kwd": "text", "layout": "heading"},
{"text": "<table><tr><td>A</td></tr></table>", "doc_type_kwd": "table"},
},
})
@@ -678,7 +674,7 @@ func TestGeneralChunkerMarkdownTableKeepsEarlierContextWhenHeadingFolds(t *testi
"output_format": "json",
"json": []map[string]any{
{"text": "intro", "doc_type_kwd": "text"},
- {"text": "## Section", "doc_type_kwd": "text", "ck_type": "heading"},
+ {"text": "## Section", "doc_type_kwd": "text", "layout": "heading"},
{"text": "<table><tr><td>A</td></tr></table>", "doc_type_kwd": "table"},
},
})
@@ -803,11 +799,11 @@ func TestGeneralChunkerDOCXAttachesMediaContextBeforeTextMerge(t *testing.T) {
func TestGeneralMediaContextSeparatesAdjacentSourceUnits(t *testing.T) {
units := []schema.ChunkDoc{
- {Text: "before one", DocType: "text", CKType: "text", TKNums: intPtr(1)},
- {Text: "before two", DocType: "text", CKType: "text", TKNums: intPtr(1)},
- {Text: "", DocType: "table", CKType: "table"},
- {Text: "after one", DocType: "text", CKType: "text", TKNums: intPtr(1)},
- {Text: "after two", DocType: "text", CKType: "text", TKNums: intPtr(1)},
+ {Text: "before one", DocType: "text", TKNums: intPtr(1)},
+ {Text: "before two", DocType: "text", TKNums: intPtr(1)},
+ {Text: "", DocType: "table"},
+ {Text: "after one", DocType: "text", TKNums: intPtr(1)},
+ {Text: "after two", DocType: "text", TKNums: intPtr(1)},
}
if got, want := collectGeneralMediaContext(units, 2, 10, true, contextSentencePattern), "before one\nbefore two"; got != want {
@@ -855,8 +851,8 @@ func TestGeneralChunkerDOCXCustomDelimiterGroupsMultiParagraphRecords(t *testing
func TestGroupDOCXUnitsByRecordBoundaryInactiveWithoutBoundary(t *testing.T) {
units := []schema.ChunkDoc{
- {Text: "plain paragraph one", DocType: "text", CKType: "text"},
- {Text: "plain paragraph two", DocType: "text", CKType: "text"},
+ {Text: "plain paragraph one", DocType: "text"},
+ {Text: "plain paragraph two", DocType: "text"},
}
got, grouped := groupDOCXUnitsByRecordBoundary(units, []string{"问:"})
if grouped {
@@ -1013,9 +1009,9 @@ func TestSortPDFUnitsKeepsUnpositionedMediaInInputOrder(t *testing.T) {
return json.RawMessage(fmt.Sprintf("[[1,0,10,%g,%g]]", top, top+5))
}
units := []schema.ChunkDoc{
- {Text: "before", DocType: "text", CKType: "text", Positions: position(10)},
- {Text: "figure", DocType: "image", CKType: "image"},
- {Text: "after", DocType: "text", CKType: "text", Positions: position(30)},
+ {Text: "before", DocType: "text", Positions: position(10)},
+ {Text: "figure", DocType: "image"},
+ {Text: "after", DocType: "text", Positions: position(30)},
}
got := sortPDFUnits(units)
@@ -1044,7 +1040,7 @@ func TestGeneralChunkerSpreadsheetHeaderOnlyPreservesHeaderChunk(t *testing.T) {
t.Fatalf("chunks = %#v, want one header-only chunk", chunks)
}
text, _ := chunks[0]["text"].(string)
- if chunks[0]["ck_type"] != "table" || text == "" || !strings.Contains(text, "Name") || !strings.Contains(text, "Amount") {
+ if chunks[0]["doc_type_kwd"] != "table" || text == "" || !strings.Contains(text, "Name") || !strings.Contains(text, "Amount") {
t.Fatalf("header-only chunk = %#v", chunks[0])
}
}
@@ -1063,7 +1059,7 @@ func TestGeneralChunkerSpreadsheetAttachesImageContext(t *testing.T) {
"output_format": "json",
"json": []map[string]any{
spreadsheetSegmentItem("Sheet1", []string{"Name"}, [][]string{{"Revenue"}}, 1, 2),
- {"text": "B2", "doc_type_kwd": "image", "ck_type": "image", "image": "figure", "sheet_index": 1},
+ {"text": "B2", "doc_type_kwd": "image", "image": "figure", "sheet_index": 1},
spreadsheetSegmentItem("Sheet1", []string{"Name"}, [][]string{{"Growth"}}, 1, 3),
},
})
@@ -1074,7 +1070,7 @@ func TestGeneralChunkerSpreadsheetAttachesImageContext(t *testing.T) {
if len(chunks) != 3 {
t.Fatalf("chunks = %#v, want segment, image, segment", chunks)
}
- if chunks[1]["ck_type"] != "image" {
+ if chunks[1]["doc_type_kwd"] != "image" {
t.Fatalf("image chunk = %#v", chunks[1])
}
assertMaterializedMediaContext(t, chunks[1], "RevenueB2Growth")
@@ -1091,7 +1087,7 @@ func TestGeneralChunkerSpreadsheetImageContextStopsAtSheetBoundary(t *testing.T)
"output_format": "json",
"json": []map[string]any{
spreadsheetSegmentItem("Sheet1", []string{"Name"}, [][]string{{"Sheet one"}}, 1, 2),
- {"text": "B2", "doc_type_kwd": "image", "ck_type": "image", "image": "figure", "sheet_index": 1},
+ {"text": "B2", "doc_type_kwd": "image", "image": "figure", "sheet_index": 1},
spreadsheetSegmentItem("Sheet2", []string{"Name"}, [][]string{{"Sheet two"}}, 2, 2),
},
})
@@ -1243,9 +1239,8 @@ func TestGeneralChunkerMediaContextReachesChunkIDAndIndexContent(t *testing.T) {
}
const wantText = "before<table><tr><td>A</td></tr></table>after"
- // The table chunk is selected by the STORED type column: ck_type is chunker
- // bookkeeping and the index boundary strips it (it has no chunk-store
- // column, and a strict engine rejects the whole insert over one).
+ // The table chunk is selected by the STORED type column: doc_type_kwd is
+ // the content-type contract and the index boundary keeps it.
var table map[string]any
for _, ck := range chunks {
if ck["doc_type_kwd"] == "table" {
@@ -1347,7 +1342,6 @@ func TestGeneralChunkerNonSpreadsheetTableNotMediaContext(t *testing.T) {
table := map[string]any{
"text": "<table><caption>Doc</caption>\n<tr><th>Name</th></tr>\n<tr><td>Revenue</td></tr>\n</table>\n",
"doc_type_kwd": "table",
- "ck_type": "table",
}
out, err := component.Invoke(t.Context(), nil, map[string]any{
"name": "document.docx",
@@ -1355,7 +1349,7 @@ func TestGeneralChunkerNonSpreadsheetTableNotMediaContext(t *testing.T) {
"output_format": "json",
"json": []map[string]any{
table,
- {"text": "B2", "doc_type_kwd": "image", "ck_type": "image", "image": "figure"},
+ {"text": "B2", "doc_type_kwd": "image", "image": "figure"},
table,
},
})
@@ -1366,7 +1360,7 @@ func TestGeneralChunkerNonSpreadsheetTableNotMediaContext(t *testing.T) {
if len(chunks) != 3 {
t.Fatalf("chunks = %#v, want table, image, table", chunks)
}
- if chunks[1]["ck_type"] != "image" {
+ if chunks[1]["doc_type_kwd"] != "image" {
t.Fatalf("image chunk = %#v", chunks[1])
}
if got, _ := chunks[1]["text"].(string); got != "B2" {
diff --git a/internal/ingestion/component/chunker/golden_parity_test.go b/internal/ingestion/component/chunker/golden_parity_test.go
index a88851675e..2b982fdf2b 100644
--- a/internal/ingestion/component/chunker/golden_parity_test.go
+++ b/internal/ingestion/component/chunker/golden_parity_test.go
@@ -246,7 +246,7 @@ func invokeChunker(t *testing.T, tc parityCase) []map[string]any {
//
// Deriving both sets per case rather than hardcoding an ignore list matters:
// the Go-only field set is not constant across components (TokenChunker emits
-// ck_type/tk_nums/id, TitleChunker emits only id), so a fixed list would both
+// tk_nums/id, TitleChunker emits only id), so a fixed list would both
// over- and under-match.
func compareChunks(t *testing.T, want, got []map[string]any, allowedExtra map[string]string) {
t.Helper()
@@ -392,9 +392,9 @@ func extraFieldsToStrip(rules []diffRule) map[string]bool {
// used for go_snapshot files. Go's json encoder sorts map keys, so two runs
// with equal content produce byte-identical output.
//
-// The Go-only decorative fields declared in known_diffs.json (id, ck_type,
+// The Go-only decorative fields declared in known_diffs.json (id,
// tk_nums) are stripped first: `id` is non-deterministic (derived from a
-// per-run doc id), and `ck_type`/`tk_nums` are Go-internal metadata already
+// per-run doc id), and `tk_nums` are Go-internal metadata already
// covered by the extra_fields allow-list. Snapshotting them would make the
// ratchet non-reproducible across runs, defeating its purpose.
func serializeChunks(caseID string, chunks []map[string]any, strip map[string]bool) string {
diff --git a/internal/ingestion/component/chunker/group.go b/internal/ingestion/component/chunker/group.go
index e217ab227e..dec608e3e0 100644
--- a/internal/ingestion/component/chunker/group.go
+++ b/internal/ingestion/component/chunker/group.go
@@ -548,7 +548,6 @@ func recordsFromStructured(items []schema.ChunkDoc) []lineRecord {
imgID: imgID,
image: image,
layout: it.Layout,
- ckType: it.CKType,
pdfPositions: it.PDFPositions,
positions: it.Positions,
preserveOrder: it.PreserveOrder,
@@ -562,7 +561,7 @@ func recordsFromStructured(items []schema.ChunkDoc) []lineRecord {
// so heading regexes can see headings after the first line. Structured
// media/table items and positioned text remain atomic, and the image
// payload stays out of per-line records so a blob is never duplicated.
- plainText := itemDocType(it) == "text" && (it.CKType == "" || it.CKType == "text")
+ plainText := itemDocType(it) == "text" && !isStructuralLayout(it.Layout)
unpositioned := len(it.PDFPositions) == 0 && len(it.Positions) == 0
if plainText && unpositioned && strings.Contains(text, "\n") {
for _, line := range strings.Split(normalizeGeneralNewlines(text), "\n") {
diff --git a/internal/ingestion/component/chunker/hierarchy_test.go b/internal/ingestion/component/chunker/hierarchy_test.go
index 5ed21ccc7a..c421eb1f82 100644
--- a/internal/ingestion/component/chunker/hierarchy_test.go
+++ b/internal/ingestion/component/chunker/hierarchy_test.go
@@ -444,12 +444,12 @@ func TestHierarchyTitleChunker_ShortSingleCJKLineFilter(t *testing.T) {
}
}
-// TestHierarchyTitleChunker_CKTypeHeadingFallback verifies that
-// records with ck_type "heading" (from office_oxide DOCX parsing)
+// TestHierarchyTitleChunker_LayoutHeadingFallback verifies that
+// records with layout "heading" (from office_oxide DOCX parsing)
// are treated as heading nodes in the tree, even when their text
// doesn't match any regex pattern. Without the fix, such records
// are treated as body text — all content merges into one chunk.
-func TestHierarchyTitleChunker_CKTypeHeadingFallback(t *testing.T) {
+func TestHierarchyTitleChunker_LayoutHeadingFallback(t *testing.T) {
c, err := NewHierarchyTitleChunker(map[string]any{
"hierarchy": 2,
"levels": [][]string{{`^# `}, {`^## `}},
@@ -458,11 +458,11 @@ func TestHierarchyTitleChunker_CKTypeHeadingFallback(t *testing.T) {
t.Fatalf("NewHierarchyTitleChunker: %v", err)
}
items := []map[string]any{
- {"text": "Introduction", "doc_type_kwd": "text", "ck_type": "heading"},
+ {"text": "Introduction", "doc_type_kwd": "text", "layout": "heading"},
{"text": "Intro body.", "doc_type_kwd": "text"},
- {"text": "Background", "doc_type_kwd": "text", "ck_type": "heading"},
+ {"text": "Background", "doc_type_kwd": "text", "layout": "heading"},
{"text": "Background body.", "doc_type_kwd": "text"},
- {"text": "Conclusion", "doc_type_kwd": "text", "ck_type": "heading"},
+ {"text": "Conclusion", "doc_type_kwd": "text", "layout": "heading"},
{"text": "Final words.", "doc_type_kwd": "text"},
}
out, err := c.Invoke(t.Context(), nil, map[string]any{
@@ -479,7 +479,7 @@ func TestHierarchyTitleChunker_CKTypeHeadingFallback(t *testing.T) {
// Without the fix: 1 chunk (all text merged as body under root).
// With the fix: 3 chunks (one per heading + its body).
if len(chunks) != 3 {
- t.Fatalf("len(chunks) = %d, want 3 (3 ck_type=headings should produce 3 chunks)", len(chunks))
+ t.Fatalf("len(chunks) = %d, want 3 (3 layout=headings should produce 3 chunks)", len(chunks))
}
for _, ck := range chunks {
text, _ := ck["text"].(string)
diff --git a/internal/ingestion/component/chunker/live_parity_test.go b/internal/ingestion/component/chunker/live_parity_test.go
index 7785ee4153..6e5858ffcc 100644
--- a/internal/ingestion/component/chunker/live_parity_test.go
+++ b/internal/ingestion/component/chunker/live_parity_test.go
@@ -35,7 +35,7 @@ package chunker
// silently-collapsed baseline into a hard failure instead of a false pass.
//
// Comparison logic is shared with the golden harness: chunkDiffs reports every
-// difference, allowedExtraFields tolerates the Go-only id/ck_type/tk_nums
+// difference, allowedExtraFields tolerates the Go-only id/tk_nums
// fields, and known_diffs.json is consulted so documented divergences are
// reported (not failed) rather than surprising the reader.
diff --git a/internal/ingestion/component/chunker/one.go b/internal/ingestion/component/chunker/one.go
index 53ae5fd1fe..52a685ae49 100644
--- a/internal/ingestion/component/chunker/one.go
+++ b/internal/ingestion/component/chunker/one.go
@@ -115,7 +115,6 @@ func emitOne(text, docType string) map[string]any {
// scope here.
Text: removeTag(text),
DocType: docType,
- CKType: docType,
}})
}
@@ -145,7 +144,6 @@ func emitOneFromItems(items, chunks []schema.ChunkDoc) map[string]any {
// Strip parser coordinate tags (see emitOne).
Text: removeTag(text),
DocType: docType,
- CKType: docType,
Image: it.Image,
Positions: it.Positions,
PDFPositions: it.PDFPositions,
@@ -170,7 +168,7 @@ func emitOneFromItems(items, chunks []schema.ChunkDoc) map[string]any {
if strings.TrimSpace(merged) == "" && img == "" {
return emptyOutputs()
}
- out := schema.ChunkDoc{Text: merged, DocType: "text", CKType: "text"}
+ out := schema.ChunkDoc{Text: merged, DocType: "text"}
// Multi-item merge produces a single text-only chunk mirroring Python
// one.py:166-168. Per-item Positions/PDFPositions are intentionally
// not carried — merging coordinates from different source items would
diff --git a/internal/ingestion/component/chunker/one_tag_test.go b/internal/ingestion/component/chunker/one_tag_test.go
index 8f61bf62d4..123ea8fad9 100644
--- a/internal/ingestion/component/chunker/one_tag_test.go
+++ b/internal/ingestion/component/chunker/one_tag_test.go
@@ -28,8 +28,8 @@ import (
// coordinate tags into the single emitted chunk.
func TestOneChunkerStripsCoordTagsJSON(t *testing.T) {
items := []schema.ChunkDoc{
- {Text: "Sentence one@@1\t2\t3\t4##", CKType: "text"},
- {Text: "Sentence two@@1\t2\t3\t4##", CKType: "text"},
+ {Text: "Sentence one@@1\t2\t3\t4##"},
+ {Text: "Sentence two@@1\t2\t3\t4##"},
}
out := emitOneFromItems(items, nil)
chunks, _ := out["chunks"].([]map[string]any)
diff --git a/internal/ingestion/component/chunker/pdfcrop_cgo.go b/internal/ingestion/component/chunker/pdfcrop_cgo.go
index 59c2557014..2e0b448177 100644
--- a/internal/ingestion/component/chunker/pdfcrop_cgo.go
+++ b/internal/ingestion/component/chunker/pdfcrop_cgo.go
@@ -278,15 +278,9 @@ func cropImageChunks(ctx context.Context, engine deepdoctype.PDFEngine, chunks [
// region (Python restore_pdf_text_previews). A pre-existing Image is never
// re-cropped — cropImageChunks honors that separately.
//
-// CKType is the chunker-layer refinement (heading/text/image/table/
-// image/table). General and token chunkers always set it before cropping, so
-// they are handled by the CKType branch above. Group and hierarchy chunkers
-// forward the parser's output verbatim, which carries only the coarser
-// doc_type_kwd (no ck_type). For those, fall back to DocType so image/table/
-// text regions are still cropped on demand — otherwise every figure/table/
-// text preview in group/hierarchy would be silently skipped. When CKType is
-// set (general/token) it takes priority, so headings (CKType "heading") stay
-// excluded from preview cropping.
+// The content type on doc_type_kwd alone decides the crop kind. Headings are
+// text chunks and get the same text preview as any other body text, matching
+// the historical behavior.
func needsCrop(ck schema.ChunkDoc) bool {
switch cropChunkType(ck) {
case "image", "table", "text":
@@ -296,9 +290,10 @@ func needsCrop(ck schema.ChunkDoc) bool {
}
}
+// cropChunkType returns the content type that selects the crop. Callers must
+// pass normalized chunks: General/Token clear structural layout before
+// cropping, so passing raw parser items would lose their block-kind
+// distinction here.
func cropChunkType(ck schema.ChunkDoc) string {
- if ck.CKType != "" {
- return ck.CKType
- }
- return ck.DocType
+ return itemDocType(ck)
}
diff --git a/internal/ingestion/component/chunker/pdfcrop_cgo_test.go b/internal/ingestion/component/chunker/pdfcrop_cgo_test.go
index 15447b1512..897e8628ef 100644
--- a/internal/ingestion/component/chunker/pdfcrop_cgo_test.go
+++ b/internal/ingestion/component/chunker/pdfcrop_cgo_test.go
@@ -72,29 +72,28 @@ func TestNeedsCrop(t *testing.T) {
ck schema.ChunkDoc
want bool
}{
- // --- CKType set (general / token paths): unchanged behavior ---
- {"image with positions", schema.ChunkDoc{CKType: "image", PDFPositions: jsonPositions(t, []float64{1, 10, 100, 10, 100})}, true},
- {"table with positions", schema.ChunkDoc{CKType: "table", Positions: jsonPositions(t, []float64{1, 10, 100, 10, 100})}, true},
- {"text with positions", schema.ChunkDoc{CKType: "text", PDFPositions: jsonPositions(t, []float64{1, 10, 100, 10, 100})}, true},
- {"image without positions", schema.ChunkDoc{CKType: "image"}, false},
- {"heading excluded via CKType", schema.ChunkDoc{CKType: "heading", PDFPositions: jsonPositions(t, []float64{1, 10, 100, 10, 100})}, false},
- {"unknown type via CKType", schema.ChunkDoc{CKType: "equation", PDFPositions: jsonPositions(t, []float64{1, 10, 100, 10, 100})}, false},
-
- // --- CKType empty: fall back to DocType (group / hierarchy paths,
- // whose parser output carries no ck_type). This is the fix for the
- // bug where figure/image/table in group/hierarchy were never cropped.
+ // Content type comes from doc_type_kwd.
+ {"image with positions", schema.ChunkDoc{DocType: "image", PDFPositions: jsonPositions(t, []float64{1, 10, 100, 10, 100})}, true},
+ {"table with positions", schema.ChunkDoc{DocType: "table", Positions: jsonPositions(t, []float64{1, 10, 100, 10, 100})}, true},
+ {"text with positions", schema.ChunkDoc{DocType: "text", PDFPositions: jsonPositions(t, []float64{1, 10, 100, 10, 100})}, true},
+ {"image without positions", schema.ChunkDoc{DocType: "image"}, false},
+ {"image with heading layout still crops", schema.ChunkDoc{DocType: "image", Layout: "heading", PDFPositions: jsonPositions(t, []float64{1, 10, 100, 10, 100})}, true},
+ {"heading text previews like text", schema.ChunkDoc{Layout: "heading", DocType: "text", PDFPositions: jsonPositions(t, []float64{1, 10, 100, 10, 100})}, true},
+ // A non-media model layout (equation) is not structural, so it falls
+ // back to its content type and is previewed as text.
+ {"equation layout falls back to text", schema.ChunkDoc{Layout: "equation", DocType: "text", PDFPositions: jsonPositions(t, []float64{1, 10, 100, 10, 100})}, true},
+
+ // Group/Hierarchy forward parser output whose content type is already
+ // on doc_type_kwd; no block-kind derivation is involved.
{"doc image fallback", schema.ChunkDoc{DocType: "image", PDFPositions: jsonPositions(t, []float64{1, 10, 100, 10, 100})}, true},
{"doc table fallback", schema.ChunkDoc{DocType: "table", Positions: jsonPositions(t, []float64{1, 10, 100, 10, 100})}, true},
{"doc text fallback", schema.ChunkDoc{DocType: "text", PDFPositions: jsonPositions(t, []float64{1, 10, 100, 10, 100})}, true},
- // Heading-tagged blocks carry doc_type_kwd "text" but
- // must NOT be previewed when CKType is empty — only a plain text body
- // chunk legitimately has CKType empty AND should be previewed. Since
- // group/hierarchy cannot distinguish heading from body at this layer,
- // aligning with Python (which previews every text region) we still crop
- // doc_type_kwd=="text"; the heading-collision guard lives in the
- // general/token paths where CKType is set.
+ // Headings are text chunks and are previewed too, matching Python's
+ // restore_pdf_text_previews.
{"doc image without positions", schema.ChunkDoc{DocType: "image"}, false},
- {"doc unknown type", schema.ChunkDoc{DocType: "equation", PDFPositions: jsonPositions(t, []float64{1, 10, 100, 10, 100})}, false},
+ // Unknown doc_type values normalize to text (itemDocType), matching
+ // every other content-type dispatch.
+ {"unknown doc type normalizes to text", schema.ChunkDoc{DocType: "equation", PDFPositions: jsonPositions(t, []float64{1, 10, 100, 10, 100})}, true},
}
for _, tc := range cases {
if got := needsCrop(tc.ck); got != tc.want {
@@ -110,17 +109,17 @@ func TestCropImageChunks_CropsImageTableAndText(t *testing.T) {
pos := jsonPositions(t, []float64{1, 10, 100, 10, 100})
chunks := []schema.ChunkDoc{
- {CKType: "image", Text: "img", PDFPositions: pos},
- {CKType: "table", Text: "tbl", PDFPositions: pos},
- {CKType: "text", PDFPositions: pos}, // restored preview (Chunker-1.3)
- {CKType: "image", Image: "data:image/png;base64,preexisting"}, // preserved
+ {DocType: "image", Text: "img", PDFPositions: pos},
+ {DocType: "table", Text: "tbl", PDFPositions: pos},
+ {PDFPositions: pos}, // restored preview (Chunker-1.3)
+ {DocType: "image", Image: "data:image/png;base64,preexisting"}, // preserved
}
out := cropImageChunks(ctx, eng, chunks)
if len(out) != len(chunks) {
t.Fatalf("len(out) = %d, want %d", len(out), len(chunks))
}
for i, ck := range out {
- switch ck.CKType {
+ switch itemDocType(ck) {
case "text":
// Chunker-1.3: text chunks with PDF positions get a rendered
// preview, mirroring Python restore_pdf_text_previews.
@@ -148,10 +147,9 @@ func TestCropImageChunks_CropsImageTableAndText(t *testing.T) {
// img_id upload is owned by imageUploadDecorator (image_upload.go) and is
// not asserted here.
// TestCropImageChunks_DocTypeFallbackCropsImageTableText covers the
-// group/hierarchy regression fix: those chunkers forward parser output that
-// carries doc_type_kwd but no ck_type. needsCrop must fall back to DocType so
-// image/table/text regions are still cropped on demand (the bug was that they
-// were silently skipped because ck_type was empty).
+// group/hierarchy path: those chunkers forward parser output whose content
+// type is already on doc_type_kwd. needsCrop reads DocType so image/table/text
+// regions are cropped on demand.
func TestCropImageChunks_DocTypeFallbackCropsImageTableText(t *testing.T) {
ctx := context.Background()
eng := assertZeroPageEngine{}
@@ -160,11 +158,11 @@ func TestCropImageChunks_DocTypeFallbackCropsImageTableText(t *testing.T) {
chunks := []schema.ChunkDoc{
{DocType: "image", Text: "img", PDFPositions: pos},
{DocType: "table", Text: "tbl", PDFPositions: pos},
- // A plain text body chunk from group/hierarchy has CKType empty and
- // doc_type_kwd "text"; the DocType fallback must crop its preview,
- // mirroring Python restore_pdf_text_previews. (Real group/hierarchy
- // headings are also doc_type_kwd "text" with empty CKType and are
- // intentionally previewed too — full alignment with Python.)
+ // A plain text body chunk from group/hierarchy has no block-kind
+ // marker and doc_type_kwd "text"; DocType drives its preview,
+ // mirroring Python restore_pdf_text_previews. Real group/hierarchy
+ // headings are also doc_type_kwd "text" and are intentionally
+ // previewed too — full alignment with Python.
{DocType: "text", PDFPositions: pos},
{DocType: "image", Image: "data:image/png;base64,preexisting"}, // preserved
}
@@ -193,31 +191,12 @@ func TestCropImageChunks_DocTypeFallbackCropsImageTableText(t *testing.T) {
}
}
-// TestCropImageChunks_HeadingExcludedByCKType verifies that the general/token
-// path still excludes headings: a chunk with CKType "heading" is never
-// previewed even when it carries positions — the CKType branch takes priority
-// over the DocType fallback.
-func TestCropImageChunks_HeadingExcludedByCKType(t *testing.T) {
- ctx := context.Background()
- pos := jsonPositions(t, []float64{1, 10, 100, 10, 100})
- chunks := []schema.ChunkDoc{
- {CKType: "heading", DocType: "text", PDFPositions: pos},
- }
- out := cropImageChunks(ctx, mockCropEngine{}, chunks)
- if len(out) != 1 {
- t.Fatalf("len(out) = %d, want 1", len(out))
- }
- if out[0].Image != "" {
- t.Errorf("heading chunk: image = %q, want no preview (CKType takes priority)", out[0].Image)
- }
-}
-
func TestRestorePDFTextPreview(t *testing.T) {
ctx := context.Background()
pos := jsonPositions(t, []float64{1, 10, 100, 10, 100})
- withPos := schema.ChunkDoc{CKType: "text", PDFPositions: pos}
- withoutPos := schema.ChunkDoc{CKType: "text", Text: "plain text, no coordinates"}
+ withPos := schema.ChunkDoc{PDFPositions: pos}
+ withoutPos := schema.ChunkDoc{Text: "plain text, no coordinates"}
out := cropImageChunks(ctx, mockCropEngine{}, []schema.ChunkDoc{withPos, withoutPos})
if len(out) != 2 {
@@ -235,8 +214,8 @@ func TestCropImageChunks_NilEnginePassesThrough(t *testing.T) {
ctx := context.Background()
pos := jsonPositions(t, []float64{1, 10, 100, 10, 100})
chunks := []schema.ChunkDoc{
- {CKType: "image", PDFPositions: pos},
- {CKType: "table", PDFPositions: pos},
+ {DocType: "image", PDFPositions: pos},
+ {DocType: "table", PDFPositions: pos},
}
out := cropImageChunks(ctx, nil, chunks)
if len(out) != 2 {
@@ -254,7 +233,7 @@ func TestCropImageChunks_RenderFailureSkipsChunk(t *testing.T) {
// mockCropEngine renders a real image, so a non-empty crop is expected.
pos := jsonPositions(t, []float64{1, 10, 100, 10, 100})
chunks := []schema.ChunkDoc{
- {CKType: "image", Text: "cap", PDFPositions: pos},
+ {DocType: "image", Text: "cap", PDFPositions: pos},
}
out := cropImageChunks(ctx, mockCropEngine{}, chunks)
if len(out) != 1 {
@@ -316,9 +295,9 @@ func TestCropImageChunks_StreamingUpload(t *testing.T) {
eng := mockCropEngine{}
pos := jsonPositions(t, []float64{1, 10, 100, 10, 100})
chunks := []schema.ChunkDoc{
- {CKType: "image", Text: "img-chunk", PDFPositions: pos},
- {CKType: "table", Text: "tbl-chunk", PDFPositions: pos},
- {CKType: "text", Text: "txt-chunk", PDFPositions: pos},
+ {DocType: "image", Text: "img-chunk", PDFPositions: pos},
+ {DocType: "table", Text: "tbl-chunk", PDFPositions: pos},
+ {Text: "txt-chunk", PDFPositions: pos},
}
out := cropImageChunks(ctx, eng, chunks)
if len(out) != len(chunks) {
@@ -327,10 +306,10 @@ func TestCropImageChunks_StreamingUpload(t *testing.T) {
for i, ck := range out {
if ck.ImgID == "" {
- t.Errorf("chunk %d (%s): ImgID empty, want uploaded id", i, ck.CKType)
+ t.Errorf("chunk %d (%s): ImgID empty, want uploaded id", i, itemDocType(ck))
}
if ck.Image != "" {
- t.Errorf("chunk %d (%s): Image = %q, want cleared after upload", i, ck.CKType, ck.Image)
+ t.Errorf("chunk %d (%s): Image = %q, want cleared after upload", i, itemDocType(ck), ck.Image)
}
}
if len(rec.calls) != len(chunks) {
@@ -376,7 +355,7 @@ func TestCropImageChunks_NoUploadWhenKBAbsent(t *testing.T) {
eng := mockCropEngine{}
pos := jsonPositions(t, []float64{1, 10, 100, 10, 100})
- chunks := []schema.ChunkDoc{{CKType: "image", Text: "img", PDFPositions: pos}}
+ chunks := []schema.ChunkDoc{{DocType: "image", Text: "img", PDFPositions: pos}}
out := cropImageChunks(ctx, eng, chunks)
if len(rec.calls) != 0 {
@@ -404,7 +383,7 @@ func TestCropImageChunks_UploadFailureFallsThroughToBatchPass(t *testing.T) {
eng := mockCropEngine{}
pos := jsonPositions(t, []float64{1, 10, 100, 10, 100})
- chunks := []schema.ChunkDoc{{CKType: "image", Text: "img-chunk", PDFPositions: pos}}
+ chunks := []schema.ChunkDoc{{DocType: "image", Text: "img-chunk", PDFPositions: pos}}
out := cropImageChunks(ctx, eng, chunks)
if out[0].ImgID != "" {
@@ -436,7 +415,7 @@ func TestCropImageChunks_StreamingUploadUsesFinalizedText(t *testing.T) {
// ContextAbove is folded into the body by materializeMediaContext, so the
// finalized text differs from the raw Text.
chunks := []schema.ChunkDoc{
- {CKType: "image", Text: "body", ContextAbove: "ABOVE ", PDFPositions: pos},
+ {DocType: "image", Text: "body", ContextAbove: "ABOVE ", PDFPositions: pos},
}
out := cropImageChunks(ctx, eng, chunks)
@@ -486,11 +465,11 @@ func TestCropImageChunks_SkipsEmptyTextMedia(t *testing.T) {
eng := mockCropEngine{}
pos := jsonPositions(t, []float64{1, 10, 100, 10, 100})
emptyMedia := []schema.ChunkDoc{
- {CKType: "image", PDFPositions: pos}, // no text
- {CKType: "table", PDFPositions: pos}, // no text
- {CKType: "image", Text: "@@1\t2##", PDFPositions: pos}, // position tag only, no body
- {CKType: "table", ContextAbove: "", ContextBelow: "", PDFPositions: pos}, // explicit empty
- {DocType: "image", PDFPositions: pos}, // doc-type fallback, no text
+ {DocType: "image", PDFPositions: pos}, // no text
+ {DocType: "table", PDFPositions: pos}, // no text
+ {DocType: "image", Text: "@@1\t2##", PDFPositions: pos}, // position tag only, no body
+ {DocType: "table", ContextAbove: "", ContextBelow: "", PDFPositions: pos}, // explicit empty
+ {DocType: "image", PDFPositions: pos}, // doc-type fallback, no text
}
out := cropImageChunks(ctx, eng, emptyMedia)
if len(out) != len(emptyMedia) {
@@ -510,7 +489,7 @@ func TestCropImageChunks_SkipsEmptyTextMedia(t *testing.T) {
// Sanity: a captioned media chunk is still cropped and uploaded.
captioned := []schema.ChunkDoc{
- {CKType: "image", Text: "a real figure caption", PDFPositions: pos},
+ {DocType: "image", Text: "a real figure caption", PDFPositions: pos},
}
out2 := cropImageChunks(ctx, eng, captioned)
if len(out2) != 1 {
@@ -555,7 +534,7 @@ func TestCropImageChunks_StreamingUploadUsesCanonicalID(t *testing.T) {
// Text carries a parser position tag (@@x\t y##) AND a caption, no media
// context — materializeMediaContext short-circuits, so the tag-stripped
// canonical text is "fig caption".
- in := schema.ChunkDoc{CKType: "image", Text: "fig caption@@1\t2##", PDFPositions: pos}
+ in := schema.ChunkDoc{DocType: "image", Text: "fig caption@@1\t2##", PDFPositions: pos}
out := cropImageChunks(ctx, eng, []schema.ChunkDoc{in})
if len(out) != 1 {
diff --git a/internal/ingestion/component/chunker/pdfcrop_concurrency_test.go b/internal/ingestion/component/chunker/pdfcrop_concurrency_test.go
index b98a735ad5..f29afda359 100644
--- a/internal/ingestion/component/chunker/pdfcrop_concurrency_test.go
+++ b/internal/ingestion/component/chunker/pdfcrop_concurrency_test.go
@@ -22,7 +22,7 @@ func TestCropImageChunks_ConcurrentCropsAll(t *testing.T) {
const n = 64
chunks := make([]schema.ChunkDoc, n)
for i := 0; i < n; i++ {
- chunks[i] = schema.ChunkDoc{CKType: "image", Text: "cap", PDFPositions: pos}
+ chunks[i] = schema.ChunkDoc{DocType: "image", Text: "cap", PDFPositions: pos}
}
out := cropImageChunks(ctx, eng, chunks)
if len(out) != n {
@@ -48,7 +48,7 @@ func TestCropImageChunks_ConcurrentPageCacheReuse(t *testing.T) {
const n = 200
chunks := make([]schema.ChunkDoc, n)
for i := 0; i < n; i++ {
- chunks[i] = schema.ChunkDoc{CKType: "table", Text: "cap", Positions: pos}
+ chunks[i] = schema.ChunkDoc{DocType: "table", Text: "cap", Positions: pos}
}
out := cropImageChunks(ctx, eng, chunks)
for i, ck := range out {
diff --git a/internal/ingestion/component/chunker/pdfcrop_progress_cgo_test.go b/internal/ingestion/component/chunker/pdfcrop_progress_cgo_test.go
index 93d2bd2b58..e72d4529c1 100644
--- a/internal/ingestion/component/chunker/pdfcrop_progress_cgo_test.go
+++ b/internal/ingestion/component/chunker/pdfcrop_progress_cgo_test.go
@@ -44,17 +44,14 @@ func TestCropImageChunks_ReportsFractions(t *testing.T) {
{
Text: "content 1",
Positions: jsonPositions(t, []float64{1, 0, 100, 0, 100}),
- CKType: "text",
},
{
Text: "content 2",
Positions: jsonPositions(t, []float64{1, 100, 200, 100, 200}),
- CKType: "text",
},
{
Text: "content 3",
Positions: jsonPositions(t, []float64{1, 200, 300, 200, 300}),
- CKType: "text",
},
}
@@ -91,8 +88,7 @@ func TestCropImageChunks_ZeroTasksReportsNothing(t *testing.T) {
chunks := []schema.ChunkDoc{
{
- Text: "no positions",
- CKType: "text",
+ Text: "no positions",
},
}
@@ -116,12 +112,10 @@ func TestCropImageChunks_CancelledContextDoesNotReport(t *testing.T) {
{
Text: "content 1",
Positions: jsonPositions(t, []float64{1, 0, 100, 0, 100}),
- CKType: "text",
},
{
Text: "content 2",
Positions: jsonPositions(t, []float64{1, 100, 200, 100, 200}),
- CKType: "text",
},
}
diff --git a/internal/ingestion/component/chunker/pdfcrop_text_test.go b/internal/ingestion/component/chunker/pdfcrop_text_test.go
index d460422613..805362ba3c 100644
--- a/internal/ingestion/component/chunker/pdfcrop_text_test.go
+++ b/internal/ingestion/component/chunker/pdfcrop_text_test.go
@@ -42,10 +42,10 @@ func TestCropImageChunksPreservesCompleteTextLines(t *testing.T) {
ck schema.ChunkDoc
wantRows int
}{
- {"text", schema.ChunkDoc{Text: "two lines", CKType: "text", PDFPositions: pos}, 48},
+ {"text", schema.ChunkDoc{Text: "two lines", PDFPositions: pos}, 48},
{"text fallback", schema.ChunkDoc{Text: "two lines", DocType: "text", Positions: pos}, 48},
- {"image", schema.ChunkDoc{Text: "figure", CKType: "image", PDFPositions: pos}, 24},
- {"table", schema.ChunkDoc{Text: "table", CKType: "table", PDFPositions: pos}, 24},
+ {"image", schema.ChunkDoc{Text: "figure", DocType: "image", PDFPositions: pos}, 24},
+ {"table", schema.ChunkDoc{Text: "table", DocType: "table", PDFPositions: pos}, 24},
} {
t.Run(tc.name, func(t *testing.T) {
out := cropImageChunks(t.Context(), textCropEngine{page: page}, []schema.ChunkDoc{tc.ck})
@@ -83,7 +83,7 @@ func TestCropImageChunksKeepsSeparateMediaBoxes(t *testing.T) {
pos := jsonPositions(t,
[]float64{1, 10, 60, 112.0 / 3, 162.0 / 3},
[]float64{1, 10, 60, 162.0 / 3, 212.0 / 3})
- out := cropImageChunks(t.Context(), textCropEngine{page: page}, []schema.ChunkDoc{{Text: typ, CKType: typ, PDFPositions: pos}})
+ out := cropImageChunks(t.Context(), textCropEngine{page: page}, []schema.ChunkDoc{{Text: typ, DocType: typ, PDFPositions: pos}})
img, err := util.DecodeBase64PNG(strings.TrimPrefix(out[0].Image, "data:image/png;base64,"))
if err != nil {
t.Fatal(err)
diff --git a/internal/ingestion/component/chunker/qa.go b/internal/ingestion/component/chunker/qa.go
index fb09b86f3b..e10c1efb6a 100644
--- a/internal/ingestion/component/chunker/qa.go
+++ b/internal/ingestion/component/chunker/qa.go
@@ -423,10 +423,9 @@ func extractQAJSON(items []schema.ChunkDoc, fileType string) []qaPair {
// block that merely opens with "<table" text (no row) stays on the
// prose path instead of silently pairing nothing. (Python's qa.py
// drops plain-text table payloads such as PDF tables entirely; the
- // walker keeps their pairs.) Pre-upgrade row-IR payloads
- // (ck_type: table_row/table_header with cells) hold no markup and
- // fall through to the text extractor, so documents from before this
- // wire must be re-parsed rather than re-chunked.
+ // walker keeps their pairs.) Pre-upgrade row-IR payloads hold no
+ // markup and fall through to the text extractor, so documents from
+ // before this wire must be re-parsed rather than re-chunked.
var rows [][]string
if td != nil {
rows = td.CellTexts()
diff --git a/internal/ingestion/component/chunker/qa_test.go b/internal/ingestion/component/chunker/qa_test.go
index 9a1e8785a4..9fdc5e3fb4 100644
--- a/internal/ingestion/component/chunker/qa_test.go
+++ b/internal/ingestion/component/chunker/qa_test.go
@@ -197,7 +197,6 @@ func TestQAChunker_JSONCSVNameUsesStrictRowShapeWithoutFileType(t *testing.T) {
"json": []map[string]any{{
"text": "<table><tr><td>question</td><td></td><td>unexpected</td></tr></table>",
"doc_type_kwd": "table",
- "ck_type": "table",
}},
}
out, err := comp.Invoke(t.Context(), nil, inputs)
diff --git a/internal/ingestion/component/chunker/table.go b/internal/ingestion/component/chunker/table.go
index cf455aba8e..3bfae3bc1e 100644
--- a/internal/ingestion/component/chunker/table.go
+++ b/internal/ingestion/component/chunker/table.go
@@ -114,7 +114,7 @@ func (c *TableChunkerComponent) invoke(_ context.Context, inputs map[string]any)
// tableItems returns the per-row records, preferring JSONResult and
// falling back to Chunks. Each table row becomes exactly one chunk; every
// other payload record passes through as one chunk — including pre-upgrade
-// row-IR records (ck_type: table_row/table_header with cells), which hold no
+// pre-upgrade row-IR records, which hold no
// markup and therefore keep no per-row positions; documents from before this
// wire must be re-parsed rather than re-chunked.
func tableItems(items, chunks []schema.ChunkDoc) []schema.ChunkDoc {
diff --git a/internal/ingestion/component/chunker/table_test.go b/internal/ingestion/component/chunker/table_test.go
index 86f7b39c3c..82e5189b9f 100644
--- a/internal/ingestion/component/chunker/table_test.go
+++ b/internal/ingestion/component/chunker/table_test.go
@@ -45,7 +45,7 @@ func tableChunksOf(t *testing.T, inputs map[string]any) []map[string]any {
// tableItem builds a chunker JSON input item carrying the structured TableData
// contract a post-migration producer emits (derived from markup), plus any
-// extra fields (positions, sheet_index, ck_type, ...). It deliberately does
+// extra fields (positions, sheet_index, ...). It deliberately does
// NOT set "text": the chunker derives a plain text body from TableData.
func tableItem(t *testing.T, html string, extra ...map[string]any) map[string]any {
t.Helper()
@@ -125,7 +125,7 @@ func TestTableChunkerPreservesHeaderOnlyTable(t *testing.T) {
if len(chunks) != 1 {
t.Fatalf("got %d chunks, want the header-only table preserved", len(chunks))
}
- if chunks[0]["ck_type"] != "table" {
+ if chunks[0]["doc_type_kwd"] != "table" {
t.Fatalf("chunk = %#v, want a table chunk", chunks[0])
}
if got, ok := chunks[0]["text"].(string); !ok || got == "" {
@@ -154,7 +154,7 @@ func TestTableChunkerPreservesHeaderOnlySheetAlongsideDataSheet(t *testing.T) {
!strings.Contains(row0, "<td>paid</td>") || !strings.Contains(row0, "<th>ID</th>") {
t.Fatalf("chunk0 = %#v", chunks[0])
}
- if chunks[1]["ck_type"] != "table" || chunks[1]["table"] != nil {
+ if chunks[1]["doc_type_kwd"] != "table" || chunks[1]["table"] != nil {
t.Fatalf("chunk1 = %#v, want the header-only segment whole with TableData cleared", chunks[1])
}
}
@@ -174,7 +174,6 @@ func TestTableChunker_ExpandsHTMLRowsWithAlignedPositions(t *testing.T) {
"output_format": "json",
"json": []map[string]any{
tableItem(t, table, map[string]any{
- "ck_type": "table",
"sheet_index": 1,
"positions": [][]float64{{1, 1, 1, 1, 2}, {1, 2, 2, 1, 2}, {1, 3, 3, 1, 2}},
}),
@@ -238,7 +237,6 @@ func TestTableChunker_NonSpreadsheetPositionsNotAttached(t *testing.T) {
"output_format": "json",
"json": []map[string]any{
tableItem(t, table, map[string]any{
- "ck_type": "table",
"positions": [][]float64{{1, 1, 1, 1, 1}, {1, 2, 2, 1, 1}, {1, 3, 3, 1, 1}},
}),
},
@@ -267,7 +265,7 @@ func TestTableChunker_HeaderOnlyHTMLTableKeepsWholePayload(t *testing.T) {
"name": "template.xlsx",
"output_format": "json",
"json": []map[string]any{
- tableItem(t, table, map[string]any{"ck_type": "table"}),
+ tableItem(t, table, map[string]any{}),
},
})
if len(chunks) != 1 {
diff --git a/internal/ingestion/component/chunker/table_wire_invariant_test.go b/internal/ingestion/component/chunker/table_wire_invariant_test.go
index 7502bf0557..c7d08ddc5d 100644
--- a/internal/ingestion/component/chunker/table_wire_invariant_test.go
+++ b/internal/ingestion/component/chunker/table_wire_invariant_test.go
@@ -55,9 +55,8 @@ func TestTableWireHasNoRowIRChunkTypes(t *testing.T) {
continue
}
for i, chunk := range chunks {
- kind, _ := chunk["ck_type"].(string)
- if kind == "table_header" || kind == "table_row" {
- t.Errorf("%s chunk[%d] ck_type = %q: the row IR must not come back", name, i, kind)
+ if kind, ok := chunk["ck_type"]; ok {
+ t.Errorf("%s chunk[%d] emits retired ck_type %v: the row IR must not come back", name, i, kind)
}
}
tested++
diff --git a/internal/ingestion/component/chunker/testdata/parity/README.md b/internal/ingestion/component/chunker/testdata/parity/README.md
index 64b6501818..c2597b05d7 100644
--- a/internal/ingestion/component/chunker/testdata/parity/README.md
+++ b/internal/ingestion/component/chunker/testdata/parity/README.md
@@ -98,7 +98,7 @@ Field comparison is deliberately asymmetric:
field is justified rather than assumed harmless.
The allowed set is derived per case rather than hardcoded, because it is not
-constant across components: `TokenChunker` emits `ck_type`/`tk_nums`/`id` while
+constant across components: `TokenChunker` emits `tk_nums`/`id` while
`TitleChunker` emits only `id`.
## No CGO required
diff --git a/internal/ingestion/component/chunker/testdata/parity/known_diffs.json b/internal/ingestion/component/chunker/testdata/parity/known_diffs.json
index c0c3ff696c..fb904a0048 100644
--- a/internal/ingestion/component/chunker/testdata/parity/known_diffs.json
+++ b/internal/ingestion/component/chunker/testdata/parity/known_diffs.json
@@ -15,9 +15,9 @@
"tag": "go_intentional",
"kind": "extra_fields",
"applies_to": ["token__*"],
- "fields": ["ck_type", "tk_nums"],
+ "fields": ["tk_nums"],
"permanent": true,
- "reason": "Python builds both fields internally (token_chunker.py:141,144) but _finalize_json_chunks (token_chunker.py:256-259) whitelists only text/doc_type_kwd/_pdf_positions/mom/img_id, dropping them. Go keeps them because they are still live downstream: needsCrop (pdfcrop_cgo.go:151) reads ck_type, and tokenizer.go:615 forwards tk_nums. The divergence follows from Go cropping after the chunker returns, whereas Python crops inside _invoke (token_chunker.py:370) before finalizing."
+ "reason": "Python builds tk_nums internally (token_chunker.py:144) but _finalize_json_chunks (token_chunker.py:256-259) whitelists only text/doc_type_kwd/_pdf_positions/mom/img_id, dropping it. Go keeps it because tokenizer.go forwards tk_nums to the Tokenizer."
},
{
"id": "token-hardcap-chunk-count",
diff --git a/internal/ingestion/component/chunker/title.go b/internal/ingestion/component/chunker/title.go
index 4ff195dbc4..282df0f280 100644
--- a/internal/ingestion/component/chunker/title.go
+++ b/internal/ingestion/component/chunker/title.go
@@ -337,7 +337,9 @@ func resolveTitleLevels(records []lineRecord, p *titleChunkerParam) []int {
out[i] = lvl
continue
}
- if rec.ckType == "heading" {
+ // Only the asserted "heading" marker bypasses the notTitle guard;
+ // model heuristics use "title" and go through matchLayoutLevel below.
+ if isHeadingLayout(rec.layout) {
out[i] = fallbackLevel
continue
}
@@ -640,7 +642,6 @@ type lineRecord struct {
imgID *string
image string
layout string
- ckType string
pdfPositions json.RawMessage
positions json.RawMessage
preserveOrder bool
diff --git a/internal/ingestion/component/chunker/token.go b/internal/ingestion/component/chunker/token.go
index 68cf9c0e4e..6351a056f4 100644
--- a/internal/ingestion/component/chunker/token.go
+++ b/internal/ingestion/component/chunker/token.go
@@ -286,41 +286,39 @@ func stripChunkerRuntimeTimestamps(inputs map[string]any) map[string]any {
// cropTitleChunks crops image/table/text previews for chunks produced by
// the Title/Group/Hierarchy chunkers, mirroring the TokenChunker JSON path
-// (cropImageChunks at token.go:513). A nil engine — or an
-// empty chunk list — leaves chunks unchanged (best-effort, matching the
-// on-demand PDF crop contract used by the TokenChunker path).
+// (cropImageChunks at token.go:513). A nil engine (or an undecodable payload)
+// skips cropping, but the returned maps are still normalized so retired
+// ck_type and chunker-internal structural layout never leave the component.
func cropTitleChunks(ctx context.Context, engine deepdoctype.PDFEngine, chunks []map[string]any) []map[string]any {
- if engine == nil || len(chunks) == 0 {
+ if len(chunks) == 0 {
+ return chunks
+ }
+ if engine == nil {
+ normalizeTitleChunkMaps(chunks)
return chunks
}
docs, _, err := schema.ChunkDocsFromAny(chunks)
if err != nil || len(docs) == 0 {
+ normalizeTitleChunkMaps(chunks)
return chunks
}
- // The Title/Group/Hierarchy chunkers emit doc_type_kwd but not the
- // ck_type field that cropImageChunks' needsCrop consults
- // (pdfcrop_cgo.go:151). Derive ck_type from doc_type_kwd so the crop
- // decision matches the TokenChunker path. The derived ck_type is
- // stripped from the returned maps so the downstream chunk shape is
- // unchanged (setting ck_type in the real output would also change
- // how a downstream TokenChunker merges these chunks — a separate
- // concern, out of scope here).
- for i := range docs {
- if docs[i].CKType == "" {
- switch docs[i].DocType {
- case "image", "table":
- docs[i].CKType = docs[i].DocType
- default:
- docs[i].CKType = "text"
- }
- }
- }
cropped := cropImageChunks(ctx, engine, docs)
- out := schema.ChunkDocsToMaps(cropped)
- for _, m := range out {
- delete(m, "ck_type")
+ for i := range cropped {
+ clearStructuralLayout(&cropped[i])
+ }
+ return schema.ChunkDocsToMaps(cropped)
+}
+
+// normalizeTitleChunkMaps keeps the Title-family output shape independent of
+// whether a PDF engine is available: retired ck_type and chunker-internal
+// structural layout never leave the component.
+func normalizeTitleChunkMaps(chunks []map[string]any) {
+ for _, chunk := range chunks {
+ delete(chunk, "ck_type")
+ if layout, _ := chunk["layout"].(string); isStructuralLayout(layout) {
+ delete(chunk, "layout")
+ }
}
- return out
}
// invokeTextPayload handles plain-text input (output_format in
@@ -370,10 +368,9 @@ func (c *TokenChunkerComponent) chunkPerSegment(text string, delimPattern, child
}
textDocs := make([]schema.ChunkDoc, 0, len(cleaned))
for _, s := range cleaned {
- // Python's unstructured text path emits only text here. Keep CKType for
- // Go's downstream crop dispatch, but do not synthesize structured-input
- // metadata such as doc_type_kwd.
- textDocs = append(textDocs, schema.ChunkDoc{Text: s, CKType: "text"})
+ // Python's unstructured text path emits only text here. Do not
+ // synthesize structured-input metadata such as doc_type_kwd.
+ textDocs = append(textDocs, schema.ChunkDoc{Text: s})
}
// A backtick children delimiter is also custom and is dropped; a bare
// children delimiter (e.g. "。") is retained losslessly like the main
@@ -484,7 +481,7 @@ func (c *TokenChunkerComponent) mergeByTokenSize(text string, delimPattern, chil
if sub == "" {
continue
}
- units = append(units, schema.ChunkDoc{Text: sub, TKNums: intPtr(tokenizeStr(sub)), CKType: "text"})
+ units = append(units, schema.ChunkDoc{Text: sub, TKNums: intPtr(tokenizeStr(sub))})
}
} else {
sections := []string{text}
@@ -496,7 +493,7 @@ func (c *TokenChunkerComponent) mergeByTokenSize(text string, delimPattern, chil
t := "\n" + sec
tk := tokenizeStr(t)
if tk <= target {
- units = append(units, schema.ChunkDoc{Text: t, TKNums: intPtr(tk), CKType: "text"})
+ units = append(units, schema.ChunkDoc{Text: t, TKNums: intPtr(tk)})
continue
}
// Oversized section: split on production sentence delimiters into
@@ -515,10 +512,10 @@ func (c *TokenChunkerComponent) mergeByTokenSize(text string, delimPattern, chil
}
hadPart = true
seg := "\n" + part
- units = append(units, schema.ChunkDoc{Text: seg, TKNums: intPtr(tokenizeStr(seg)), CKType: "text"})
+ units = append(units, schema.ChunkDoc{Text: seg, TKNums: intPtr(tokenizeStr(seg))})
}
if !hadPart {
- units = append(units, schema.ChunkDoc{Text: t, TKNums: intPtr(tokenizeStr(t)), CKType: "text"})
+ units = append(units, schema.ChunkDoc{Text: t, TKNums: intPtr(tokenizeStr(t))})
}
}
}
@@ -730,17 +727,17 @@ func separateItemBoundaries(perItem [][]schema.ChunkDoc) {
continue
}
firstIndex, lastIndex := 0, len(units)-1
- for firstIndex <= lastIndex && units[firstIndex].CKType == "text" && units[firstIndex].Text == "" {
+ for firstIndex <= lastIndex && itemDocType(units[firstIndex]) == "text" && units[firstIndex].Text == "" {
firstIndex++
}
- for lastIndex >= firstIndex && units[lastIndex].CKType == "text" && units[lastIndex].Text == "" {
+ for lastIndex >= firstIndex && itemDocType(units[lastIndex]) == "text" && units[lastIndex].Text == "" {
lastIndex--
}
if firstIndex > lastIndex {
continue
}
first := &units[firstIndex]
- if previous != nil && previous.CKType == "text" && first.CKType == "text" &&
+ if previous != nil && itemDocType(*previous) == "text" && itemDocType(*first) == "text" &&
strings.TrimRightFunc(previous.Text, unicode.IsSpace) == previous.Text &&
strings.TrimLeftFunc(first.Text, unicode.IsSpace) == first.Text {
first.Text = "\n" + first.Text
@@ -803,7 +800,7 @@ func chunkFromItem(it schema.ChunkDoc, delimPattern *regexp.Regexp, keepDelim bo
// buildChunkMap constructs the python-compatible chunk payload.
//
-// The chunker output carries the basic text+doc_type_kwd+ck_type
+// The chunker output carries the basic text+doc_type_kwd
// fields plus the per-chunk meta fields the python
// rag/flow/chunker/token_chunker.py emits:
//
@@ -824,7 +821,6 @@ func buildChunkDoc(it schema.ChunkDoc, ckType, text, ctxAbove, ctxBelow string)
out := schema.ChunkDoc{
Text: text,
DocType: ckType,
- CKType: ckType,
TKNums: intPtr(tokenizeStr(text)),
Mom: it.Mom,
ImgID: it.ImgID,
@@ -837,6 +833,9 @@ func buildChunkDoc(it schema.ChunkDoc, ckType, text, ctxAbove, ctxBelow string)
SheetIndex: it.SheetIndex,
Extra: cloneExtra(it.Extra),
}
+ // buildChunkDoc rewrites every unit to its content type, so the parser's
+ // block-kind assertion no longer applies to the emitted chunk.
+ clearStructuralLayout(&out)
if ctxAbove != "" {
out.ContextAbove = ctxAbove
}
@@ -884,12 +883,12 @@ func attachMediaContext(chunks []schema.ChunkDoc, tableCtx, imageCtx int) []sche
return chunks
}
for i, ck := range chunks {
- ckType := ck.CKType
- if ckType != "table" && ckType != "image" {
+ docType := itemDocType(ck)
+ if docType != "table" && docType != "image" {
continue
}
ctx := imageCtx
- if ckType == "table" {
+ if docType == "table" {
ctx = tableCtx
}
if ctx <= 0 {
@@ -913,7 +912,7 @@ func collectContext(chunks []schema.ChunkDoc, i, ctxTokens int, above bool) stri
if above {
pos = i - 1
for pos >= 0 && remain > 0 {
- if chunks[pos].CKType == "text" {
+ if itemDocType(chunks[pos]) == "text" {
tk := intValue(chunks[pos].TKNums)
txt := chunks[pos].Text
if tk >= remain {
@@ -929,7 +928,7 @@ func collectContext(chunks []schema.ChunkDoc, i, ctxTokens int, above bool) stri
} else {
pos = i + 1
for pos < len(chunks) && remain > 0 {
- if chunks[pos].CKType == "text" {
+ if itemDocType(chunks[pos]) == "text" {
tk := intValue(chunks[pos].TKNums)
txt := chunks[pos].Text
if tk >= remain {
@@ -1088,7 +1087,7 @@ func mergeUnits(units []schema.ChunkDoc, target int, overlapPct float64, joinSep
prevIdx := -1
for i := range units {
ck := units[i]
- if ck.CKType != "text" {
+ if itemDocType(ck) != "text" {
merged = append(merged, cloneChunkDoc(ck))
mergedItems = append(mergedItems, nil)
prevIdx = -1
@@ -1224,7 +1223,7 @@ func expandOversizedUnits(units []schema.ChunkDoc, target int) []schema.ChunkDoc
}
out := make([]schema.ChunkDoc, 0, len(units))
for _, ck := range units {
- if ck.CKType != "text" {
+ if itemDocType(ck) != "text" {
out = append(out, ck)
continue
}
@@ -1286,7 +1285,7 @@ func splitOversizedText(ck schema.ChunkDoc, target int) []schema.ChunkDoc {
wsPending += part
continue
}
- p := schema.ChunkDoc{Text: wsPending + part, DocType: ck.DocType, CKType: "text"}
+ p := schema.ChunkDoc{Text: wsPending + part, DocType: ck.DocType}
wsPending = ""
if n := tokenizeStr(p.Text); n > target {
pieces = append(pieces, hardSplitPiece(p.Text, ck.DocType, target)...)
@@ -1312,7 +1311,7 @@ func splitOversizedText(ck schema.ChunkDoc, target int) []schema.ChunkDoc {
if n := tokenizeStr(wsPending); n > target {
pieces = append(pieces, hardSplitPiece(wsPending, ck.DocType, target)...)
} else {
- pieces = append(pieces, schema.ChunkDoc{Text: wsPending, DocType: ck.DocType, TKNums: intPtr(n), CKType: "text"})
+ pieces = append(pieces, schema.ChunkDoc{Text: wsPending, DocType: ck.DocType, TKNums: intPtr(n)})
}
}
}
@@ -1345,7 +1344,7 @@ func splitOversizedText(ck schema.ChunkDoc, target int) []schema.ChunkDoc {
inherited := cloneChunkDoc(ck)
inherited.Text = p.Text
inherited.TKNums = p.TKNums
- inherited.CKType = "text"
+ clearStructuralLayout(&inherited)
inherited.PDFPositions = pdfSliced[i]
inherited.Positions = posSliced[i]
pieces[i] = inherited
@@ -1404,7 +1403,7 @@ func hardSplitPiece(text string, docType string, target int) []schema.ChunkDoc {
if target <= 0 || len(tokens) <= target {
// target <= 0 is unreachable from validated configs; the byte loop
// would emit the whole text as one piece, so mirror that here.
- return []schema.ChunkDoc{{Text: text, DocType: docType, TKNums: intPtr(tokenizeStr(text)), CKType: "text"}}
+ return []schema.ChunkDoc{{Text: text, DocType: docType, TKNums: intPtr(tokenizeStr(text))}}
}
// offsets[i] = byte length of the first i tokens decoded. Token ids repeat
@@ -1485,7 +1484,7 @@ func hardSplitPiece(text string, docType string, target int) []schema.ChunkDoc {
q = p + 1 // never stall (defensive; unreachable above)
}
piece := text[offsets[p]:offsets[q]]
- out = append(out, schema.ChunkDoc{Text: piece, DocType: docType, TKNums: intPtr(tokenizeStr(piece)), CKType: "text"})
+ out = append(out, schema.ChunkDoc{Text: piece, DocType: docType, TKNums: intPtr(tokenizeStr(piece))})
p = q
}
return out
@@ -1527,17 +1526,17 @@ func hardSplitPieceByteLoop(text string, docType string, target int) []schema.Ch
// Safety: never stall. An empty cut or a whole-text cut means no
// progress is possible; emit the remaining text as the last piece.
if rest != "" {
- out = append(out, schema.ChunkDoc{Text: rest, DocType: docType, TKNums: intPtr(tokenizeStr(rest)), CKType: "text"})
+ out = append(out, schema.ChunkDoc{Text: rest, DocType: docType, TKNums: intPtr(tokenizeStr(rest))})
}
rest = ""
break
}
head = rest[:cut]
- out = append(out, schema.ChunkDoc{Text: head, DocType: docType, TKNums: intPtr(tokenizeStr(head)), CKType: "text"})
+ out = append(out, schema.ChunkDoc{Text: head, DocType: docType, TKNums: intPtr(tokenizeStr(head))})
rest = rest[cut:]
}
if rest != "" {
- out = append(out, schema.ChunkDoc{Text: rest, DocType: docType, TKNums: intPtr(tokenizeStr(rest)), CKType: "text"})
+ out = append(out, schema.ChunkDoc{Text: rest, DocType: docType, TKNums: intPtr(tokenizeStr(rest))})
}
return out
}
@@ -1820,7 +1819,7 @@ func applyChildrenDelimText(docs []schema.ChunkDoc, pattern *regexp.Regexp, keep
text = leading + text
leading = ""
}
- child := schema.ChunkDoc{CKType: d.CKType, Mom: mom}
+ child := schema.ChunkDoc{DocType: d.DocType, Layout: d.Layout, Mom: mom}
setChunkText(&child, text)
out = append(out, child)
}
diff --git a/internal/ingestion/component/chunker/token_batch1_test.go b/internal/ingestion/component/chunker/token_batch1_test.go
index 32e3946853..ce9ca63e5b 100644
--- a/internal/ingestion/component/chunker/token_batch1_test.go
+++ b/internal/ingestion/component/chunker/token_batch1_test.go
@@ -88,9 +88,9 @@ func TestMergeByTokenSizeFromJSON_OverlapStripsTags(t *testing.T) {
}
items := [][]schema.ChunkDoc{
{
- {Text: aText, DocType: "text", CKType: "text", TKNums: intPtr(aN)},
- {Text: bText, DocType: "text", CKType: "text", TKNums: intPtr(bN)},
- {Text: cText, DocType: "text", CKType: "text", TKNums: intPtr(cN)},
+ {Text: aText, DocType: "text", TKNums: intPtr(aN)},
+ {Text: bText, DocType: "text", TKNums: intPtr(bN)},
+ {Text: cText, DocType: "text", TKNums: intPtr(cN)},
},
}
got := mergeByTokenSizeFromJSON(items, budget, 30.0)
@@ -150,11 +150,11 @@ func TestMergeByTokenSizeFromJSON_NonTextBoundaryResetsMergeRun(t *testing.T) {
items := [][]schema.ChunkDoc{
{
- {Text: t1, DocType: "text", CKType: "text", TKNums: intPtr(t1N)},
- {Text: t2, DocType: "text", CKType: "text", TKNums: intPtr(t2N)},
- {Text: "[image]", DocType: "image", CKType: "image", TKNums: intPtr(1)},
- {Text: t3, DocType: "text", CKType: "text", TKNums: intPtr(t3N)},
- {Text: t4, DocType: "text", CKType: "text", TKNums: intPtr(t4N)},
+ {Text: t1, DocType: "text", TKNums: intPtr(t1N)},
+ {Text: t2, DocType: "text", TKNums: intPtr(t2N)},
+ {Text: "[image]", DocType: "image", TKNums: intPtr(1)},
+ {Text: t3, DocType: "text", TKNums: intPtr(t3N)},
+ {Text: t4, DocType: "text", TKNums: intPtr(t4N)},
},
}
got := mergeByTokenSizeFromJSON(items, budget, 0.0)
@@ -163,7 +163,7 @@ func TestMergeByTokenSizeFromJSON_NonTextBoundaryResetsMergeRun(t *testing.T) {
if len(merged) != 3 {
var texts []string
for _, c := range merged {
- texts = append(texts, c.CKType+":"+c.Text)
+ texts = append(texts, itemDocType(c)+":"+c.Text)
}
t.Fatalf("want 3 chunks (merged + non-text + merged), got %d: %v", len(merged), texts)
}
@@ -174,8 +174,8 @@ func TestMergeByTokenSizeFromJSON_NonTextBoundaryResetsMergeRun(t *testing.T) {
if merged[0].Text != t1+t2 {
t.Errorf("chunk[0] should be the merged T1+T2 chunk: got %q", merged[0].Text)
}
- if merged[1].CKType != "image" {
- t.Errorf("chunk[1] should be the non-text image chunk, got CKType=%q text=%q", merged[1].CKType, merged[1].Text)
+ if itemDocType(merged[1]) != "image" {
+ t.Errorf("chunk[1] should be the non-text image chunk, got doc_type=%q text=%q", itemDocType(merged[1]), merged[1].Text)
}
wantMerged := t3 + t4
if merged[2].Text != wantMerged {
@@ -207,9 +207,9 @@ func TestMergeByTokenSizeFromJSON_UnderCapNoOverflow(t *testing.T) {
}
items := [][]schema.ChunkDoc{
{
- {Text: aText, DocType: "text", CKType: "text", TKNums: intPtr(aN)},
- {Text: bText, DocType: "text", CKType: "text", TKNums: intPtr(bN)},
- {Text: cText, DocType: "text", CKType: "text", TKNums: intPtr(cN)},
+ {Text: aText, DocType: "text", TKNums: intPtr(aN)},
+ {Text: bText, DocType: "text", TKNums: intPtr(bN)},
+ {Text: cText, DocType: "text", TKNums: intPtr(cN)},
},
}
got := mergeByTokenSizeFromJSON(items, budget, 0.0)
@@ -247,8 +247,8 @@ func TestMergeByTokenSizeFromJSON_UnderCapNoOverflow(t *testing.T) {
func clampOverlapFixture() [][]schema.ChunkDoc {
return [][]schema.ChunkDoc{
{
- {Text: strings.Repeat("word ", 20), DocType: "text", CKType: "text", TKNums: intPtr(130)},
- {Text: "body", DocType: "text", CKType: "text", TKNums: intPtr(5)},
+ {Text: strings.Repeat("word ", 20), DocType: "text", TKNums: intPtr(130)},
+ {Text: "body", DocType: "text", TKNums: intPtr(5)},
},
}
}
@@ -289,8 +289,8 @@ func TestMergeByTokenSizeFromJSON_ClampsOverlappedPct(t *testing.T) {
func TestMergeByTokenSizeFromJSON_EmptyPrevKeepsChunk(t *testing.T) {
items := [][]schema.ChunkDoc{
{
- {Text: "", DocType: "text", CKType: "text", TKNums: intPtr(5)},
- {Text: "keepme", DocType: "text", CKType: "text", TKNums: intPtr(5)},
+ {Text: "", DocType: "text", TKNums: intPtr(5)},
+ {Text: "keepme", DocType: "text", TKNums: intPtr(5)},
},
}
got := mergeByTokenSizeFromJSON(items, 128, 0)
diff --git a/internal/ingestion/component/chunker/token_delim_lossless_boundary_test.go b/internal/ingestion/component/chunker/token_delim_lossless_boundary_test.go
index 653097af19..3e8082d20e 100644
--- a/internal/ingestion/component/chunker/token_delim_lossless_boundary_test.go
+++ b/internal/ingestion/component/chunker/token_delim_lossless_boundary_test.go
@@ -64,7 +64,7 @@ func TestDelimiterLossless_BoundaryReconstruction(t *testing.T) {
// adjacent child rather than dropped, so the child texts still reconstruct "A\n\nB".
func TestSplitByChildrenKeepsWhitespaceOnlyPiece(t *testing.T) {
pat := compileDelimPattern([]string{"\n"})
- doc := schema.ChunkDoc{Text: "A\n\nB", DocType: "text", CKType: "text"}
+ doc := schema.ChunkDoc{Text: "A\n\nB", DocType: "text"}
out := splitByChildren([]schema.ChunkDoc{doc}, pat, true)
var got strings.Builder
for _, c := range out {
@@ -80,7 +80,7 @@ func TestSplitByChildrenKeepsWhitespaceOnlyPiece(t *testing.T) {
// text-children split helper.
func TestApplyChildrenDelimTextKeepsWhitespaceOnlyPiece(t *testing.T) {
pat := compileDelimPattern([]string{"\n"})
- doc := schema.ChunkDoc{Text: "A\n\nB", CKType: "text"}
+ doc := schema.ChunkDoc{Text: "A\n\nB"}
out := applyChildrenDelimText([]schema.ChunkDoc{doc}, pat, true)
var got strings.Builder
for _, c := range out {
diff --git a/internal/ingestion/component/chunker/token_delim_lossless_followup_test.go b/internal/ingestion/component/chunker/token_delim_lossless_followup_test.go
index 086dfbd81e..ca8582c891 100644
--- a/internal/ingestion/component/chunker/token_delim_lossless_followup_test.go
+++ b/internal/ingestion/component/chunker/token_delim_lossless_followup_test.go
@@ -41,8 +41,8 @@ import (
func TestApplyChildrenDelimText_NoCrossParentFold(t *testing.T) {
pattern := regexp.MustCompile("\n")
docs := []schema.ChunkDoc{
- {Text: "\nA", CKType: "text"},
- {Text: "\nA", CKType: "text"},
+ {Text: "\nA"},
+ {Text: "\nA"},
}
out := applyChildrenDelimText(docs, pattern, true)
if len(out) != 2 {
@@ -73,7 +73,7 @@ func TestChunkFromItem_RetainsWhitespacePieces(t *testing.T) {
}
for _, c := range cases {
t.Run(c.name, func(t *testing.T) {
- it := schema.ChunkDoc{Text: c.text, CKType: "text"}
+ it := schema.ChunkDoc{Text: c.text}
out := chunkFromItem(it, pattern, true)
got := make([]string, 0, len(out))
for _, d := range out {
@@ -92,7 +92,7 @@ func TestChunkFromItem_RetainsWhitespacePieces(t *testing.T) {
func TestChunkFromItem_LosslessWhitespaceReconstruct(t *testing.T) {
pattern := regexp.MustCompile("\n")
src := "alpha\n\nbeta\n\ngamma"
- it := schema.ChunkDoc{Text: src, CKType: "text"}
+ it := schema.ChunkDoc{Text: src}
out := chunkFromItem(it, pattern, true)
var recon string
for _, d := range out {
diff --git a/internal/ingestion/component/chunker/token_hardcap_test.go b/internal/ingestion/component/chunker/token_hardcap_test.go
index 30361c327d..2e0493c35c 100644
--- a/internal/ingestion/component/chunker/token_hardcap_test.go
+++ b/internal/ingestion/component/chunker/token_hardcap_test.go
@@ -44,10 +44,10 @@ import (
func TestMergeUnits_UnderCapRunningSum(t *testing.T) {
const target = 50
units := []schema.ChunkDoc{
- {Text: "u1", TKNums: intPtr(25), CKType: "text"},
- {Text: "u2", TKNums: intPtr(25), CKType: "text"},
- {Text: "u3", TKNums: intPtr(25), CKType: "text"},
- {Text: "u4", TKNums: intPtr(25), CKType: "text"},
+ {Text: "u1", TKNums: intPtr(25)},
+ {Text: "u2", TKNums: intPtr(25)},
+ {Text: "u3", TKNums: intPtr(25)},
+ {Text: "u4", TKNums: intPtr(25)},
}
got := mergeUnits(units, target, 0, "")
if len(got) != 2 {
@@ -64,15 +64,15 @@ func TestMergeUnits_UnderCapRunningSum(t *testing.T) {
// still merges (<= target), prev+incoming > target starts a fresh chunk.
func TestMergeUnits_BoundaryExactFit(t *testing.T) {
fit := mergeUnits([]schema.ChunkDoc{
- {Text: "a", TKNums: intPtr(25), CKType: "text"},
- {Text: "b", TKNums: intPtr(25), CKType: "text"},
+ {Text: "a", TKNums: intPtr(25)},
+ {Text: "b", TKNums: intPtr(25)},
}, 50, 0, "")
if len(fit) != 1 {
t.Fatalf("exact-fit join must merge, got %d chunks", len(fit))
}
over := mergeUnits([]schema.ChunkDoc{
- {Text: "a", TKNums: intPtr(30), CKType: "text"},
- {Text: "b", TKNums: intPtr(30), CKType: "text"},
+ {Text: "a", TKNums: intPtr(30)},
+ {Text: "b", TKNums: intPtr(30)},
}, 50, 0, "")
if len(over) != 2 {
t.Fatalf("overflowing join must start a fresh chunk, got %d chunks", len(over))
@@ -86,7 +86,7 @@ func TestMergeUnits_ExpandsOversizedUnit(t *testing.T) {
const target = 30
body := strings.Repeat("word ", 100) // ~100 tokens, no sentence delimiter
got := mergeUnits([]schema.ChunkDoc{
- {Text: body, CKType: "text", TKNums: intPtr(tokenizeStr(body))},
+ {Text: body, TKNums: intPtr(tokenizeStr(body))},
}, target, 0, "")
if len(got) < 2 {
t.Fatalf("oversized unit must be split, got %d chunk(s)", len(got))
@@ -114,7 +114,7 @@ func TestMergeUnits_ExpandsOversizedSentence(t *testing.T) {
t.Fatal("fixture: tokenize returned 0 for sentence")
}
got := mergeUnits([]schema.ChunkDoc{
- {Text: body, CKType: "text", TKNums: intPtr(tokenizeStr(body))},
+ {Text: body, TKNums: intPtr(tokenizeStr(body))},
}, target, 0, "")
if len(got) < 2 {
t.Fatalf("oversized unit must be sentence-split, got %d chunk(s)", len(got))
@@ -139,7 +139,7 @@ func TestMergeUnits_SubChunksKeepPositions(t *testing.T) {
body := strings.Repeat("word ", 100)
pos := []byte(`[{"page":1}]`)
got := mergeUnits([]schema.ChunkDoc{
- {Text: body, CKType: "text", TKNums: intPtr(tokenizeStr(body)), PDFPositions: pos},
+ {Text: body, TKNums: intPtr(tokenizeStr(body)), PDFPositions: pos},
}, target, 0, "")
if len(got) < 2 {
t.Fatalf("want >=2 chunks, got %d", len(got))
@@ -154,7 +154,7 @@ func TestMergeUnits_SubChunksKeepPositions(t *testing.T) {
// TestMergeUnits_AtomicNonText verifies non-text chunks (tables/images) are
// never split even when their token count exceeds the target.
func TestMergeUnits_AtomicNonText(t *testing.T) {
- table := schema.ChunkDoc{Text: strings.Repeat("t", 500), CKType: "table", TKNums: intPtr(500)}
+ table := schema.ChunkDoc{Text: strings.Repeat("t", 500), DocType: "table", TKNums: intPtr(500)}
got := mergeUnits([]schema.ChunkDoc{table}, 50, 0, "")
if len(got) != 1 || got[0].Text != table.Text {
t.Fatalf("table chunk must pass through unchanged, got %#v", got)
@@ -168,7 +168,7 @@ func TestMergeUnits_ExpandedPiecesKeepDocType(t *testing.T) {
const target = 30
body := strings.Repeat("word ", 100)
got := mergeUnits([]schema.ChunkDoc{
- {Text: body, DocType: "text", CKType: "text", TKNums: intPtr(tokenizeStr(body))},
+ {Text: body, DocType: "text", TKNums: intPtr(tokenizeStr(body))},
}, target, 0, "")
if len(got) < 2 {
t.Fatalf("oversized unit must be split, got %d chunk(s)", len(got))
@@ -188,7 +188,7 @@ func TestMergeUnits_HardSplitNeverSplitsCoordTag(t *testing.T) {
const tag = "@@1\t2\t3\t4##"
body := strings.Repeat("word ", 30) + tag + strings.Repeat("pad ", 30)
got := mergeUnits([]schema.ChunkDoc{
- {Text: body, CKType: "text", TKNums: intPtr(tokenizeStr(body))},
+ {Text: body, TKNums: intPtr(tokenizeStr(body))},
}, target, 0, "")
if len(got) < 3 {
t.Fatalf("want multiple pieces, got %d", len(got))
@@ -213,8 +213,8 @@ func TestMergeUnits_HardSplitNeverSplitsCoordTag(t *testing.T) {
func TestMergeUnits_OverlapStrictCap(t *testing.T) {
const target = 40
units := []schema.ChunkDoc{
- {Text: strings.Repeat("word ", 30), CKType: "text"}, // ~30 tokens
- {Text: strings.Repeat("pad ", 38), CKType: "text"}, // ~38 tokens, near target
+ {Text: strings.Repeat("word ", 30)}, // ~30 tokens
+ {Text: strings.Repeat("pad ", 38)}, // ~38 tokens, near target
}
got := mergeUnits(units, target, 20, "\n")
if len(got) < 2 {
@@ -238,7 +238,7 @@ func TestMergeUnits_ExpansionPreservesWhitespaceFragments(t *testing.T) {
const target = 30
body := strings.Repeat("word ", 60) + "\n\n" + strings.Repeat("pad ", 60)
got := mergeUnits([]schema.ChunkDoc{
- {Text: body, CKType: "text", TKNums: intPtr(tokenizeStr(body))},
+ {Text: body, TKNums: intPtr(tokenizeStr(body))},
}, target, 0, "")
if len(got) < 2 {
t.Fatalf("oversized unit must be split, got %d chunk(s)", len(got))
@@ -262,7 +262,6 @@ func TestMergeUnits_ExpandedPiecesCarryMetadata(t *testing.T) {
unit := schema.ChunkDoc{
Text: body,
DocType: "text",
- CKType: "text",
TKNums: intPtr(tokenizeStr(body)),
PDFPositions: []byte(`[{"page":1}]`),
Mom: "parent-id",
@@ -299,7 +298,7 @@ func TestMergeByTokenSizeFromJSON_HardCapStrict(t *testing.T) {
var items []schema.ChunkDoc
for i := 0; i < 12; i++ {
text := strings.Repeat("w ", 12)
- items = append(items, schema.ChunkDoc{Text: text, DocType: "text", CKType: "text", TKNums: intPtr(tokenizeStr(text))})
+ items = append(items, schema.ChunkDoc{Text: text, DocType: "text", TKNums: intPtr(tokenizeStr(text))})
}
got := mergeByTokenSizeFromJSON([][]schema.ChunkDoc{items}, budget, 0)
if len(got[0]) < 2 {
@@ -325,8 +324,8 @@ func TestMergeUnits_JoinSepAddsTokens(t *testing.T) {
t.Skipf("fixture: joinSep newline adds no token for these texts (joined=%d target=%d)", joinedTK, target)
}
got := mergeUnits([]schema.ChunkDoc{
- {Text: a, TKNums: intPtr(na), CKType: "text"},
- {Text: b, TKNums: intPtr(nb), CKType: "text"},
+ {Text: a, TKNums: intPtr(na)},
+ {Text: b, TKNums: intPtr(nb)},
}, target, 0, "\n")
if len(got) != 2 {
t.Fatalf("joined text exceeds target via joinSep: want 2 chunks, got %d", len(got))
@@ -349,7 +348,7 @@ func TestExpandOversizedUnits_AllWhitespaceOverCap(t *testing.T) {
if n := tokenizeStr(ws); n <= target {
t.Skipf("fixture: whitespace tokenizes to %d <= target %d", n, target)
}
- got := expandOversizedUnits([]schema.ChunkDoc{{Text: ws, CKType: "text"}}, target)
+ got := expandOversizedUnits([]schema.ChunkDoc{{Text: ws}}, target)
if len(got) < 2 {
t.Fatalf("all-whitespace unit over cap must be hard-split, got %d chunk(s)", len(got))
}
diff --git a/internal/ingestion/component/chunker/token_json_overlap_test.go b/internal/ingestion/component/chunker/token_json_overlap_test.go
index bbe517092d..075e896a1b 100644
--- a/internal/ingestion/component/chunker/token_json_overlap_test.go
+++ b/internal/ingestion/component/chunker/token_json_overlap_test.go
@@ -59,9 +59,9 @@ func TestMergeByTokenSizeFromJSON_TKNumsConsistency(t *testing.T) {
}
items := [][]schema.ChunkDoc{
{
- {Text: aText, DocType: "text", CKType: "text", TKNums: intPtr(aN)},
- {Text: bText, DocType: "text", CKType: "text", TKNums: intPtr(bN)},
- {Text: cText, DocType: "text", CKType: "text", TKNums: intPtr(cN)},
+ {Text: aText, DocType: "text", TKNums: intPtr(aN)},
+ {Text: bText, DocType: "text", TKNums: intPtr(bN)},
+ {Text: cText, DocType: "text", TKNums: intPtr(cN)},
},
}
got := mergeByTokenSizeFromJSON(items, budget, overlapPct)
diff --git a/internal/ingestion/component/chunker/token_merge_alloc_test.go b/internal/ingestion/component/chunker/token_merge_alloc_test.go
index 2ac85421fa..cfe146b665 100644
--- a/internal/ingestion/component/chunker/token_merge_alloc_test.go
+++ b/internal/ingestion/component/chunker/token_merge_alloc_test.go
@@ -126,7 +126,6 @@ func benchMergeUnitsJSON(b *testing.B, n int) {
for i := 0; i < n; i++ {
units[i] = schema.ChunkDoc{
Text: strings.Repeat("word ", 5),
- CKType: "text",
TKNums: intPtr(5),
PDFPositions: append(json.RawMessage(nil), pos...),
}
diff --git a/internal/ingestion/component/chunker/token_merge_parity_test.go b/internal/ingestion/component/chunker/token_merge_parity_test.go
index 6ccf0bb681..ab38e0d4a2 100644
--- a/internal/ingestion/component/chunker/token_merge_parity_test.go
+++ b/internal/ingestion/component/chunker/token_merge_parity_test.go
@@ -91,7 +91,7 @@ func goMergeGroupsOracle(paragraphs []string, cap int, overlapPct float64) []str
if sub == "" {
continue
}
- units = append(units, schema.ChunkDoc{Text: sub, TKNums: intPtr(tokenizeStr(sub)), CKType: "text"})
+ units = append(units, schema.ChunkDoc{Text: sub, TKNums: intPtr(tokenizeStr(sub))})
}
merged := mergeUnits(units, cap, overlapPct, "")
out := make([]string, 0, len(merged))
@@ -239,7 +239,7 @@ func goMergeWithOverlapOracle(paragraphs []string, chunkTokenSize int, overlappe
if sub == "" {
continue
}
- units = append(units, schema.ChunkDoc{Text: sub, TKNums: intPtr(tokenizeStr(sub)), CKType: "text"})
+ units = append(units, schema.ChunkDoc{Text: sub, TKNums: intPtr(tokenizeStr(sub))})
}
merged := mergeUnits(units, chunkTokenSize, overlappedPercent, "")
out := make([]string, 0, len(merged))
diff --git a/internal/ingestion/component/chunker/token_merge_units_test.go b/internal/ingestion/component/chunker/token_merge_units_test.go
index 632956deda..0cf94e33b3 100644
--- a/internal/ingestion/component/chunker/token_merge_units_test.go
+++ b/internal/ingestion/component/chunker/token_merge_units_test.go
@@ -216,7 +216,10 @@ func TestMergeUnitsMatchesHardCapOracle(t *testing.T) {
t.Run(c.name, func(t *testing.T) {
units := make([]schema.ChunkDoc, len(c.texts))
for i, tx := range c.texts {
- units[i] = schema.ChunkDoc{Text: tx, TKNums: intPtr(c.tkNums[i]), CKType: c.kinds[i]}
+ units[i] = schema.ChunkDoc{Text: tx, TKNums: intPtr(c.tkNums[i])}
+ if c.kinds[i] == "image" || c.kinds[i] == "table" {
+ units[i].DocType = c.kinds[i]
+ }
}
want := hardCapMergeUnitsOracle(c.texts, c.tkNums, c.kinds, c.target, c.overlap, c.joinSep)
got := mergeUnits(units, c.target, c.overlap, c.joinSep)
@@ -255,8 +258,8 @@ func TestMergeUnitsOverlapPrefixIsTagFree(t *testing.T) {
for _, c := range cases {
t.Run(c.name, func(t *testing.T) {
units := []schema.ChunkDoc{
- {Text: c.prev, TKNums: intPtr(10), CKType: "text"},
- {Text: c.cur, TKNums: intPtr(4), CKType: "text"},
+ {Text: c.prev, TKNums: intPtr(10)},
+ {Text: c.cur, TKNums: intPtr(4)},
}
got := mergeUnits(units, c.target, c.overlap, "\n")
if len(got) < 2 {
diff --git a/internal/ingestion/component/chunker/token_pdfpos_test.go b/internal/ingestion/component/chunker/token_pdfpos_test.go
index 9bde5d471b..f6818c6133 100644
--- a/internal/ingestion/component/chunker/token_pdfpos_test.go
+++ b/internal/ingestion/component/chunker/token_pdfpos_test.go
@@ -37,8 +37,8 @@ func TestMergeByTokenSizeFromJSON_ExtendsPDFPositions(t *testing.T) {
posB := json.RawMessage(`[[2,15,25,35,45]]`)
items := [][]schema.ChunkDoc{
{
- {Text: "alpha", DocType: "text", CKType: "text", TKNums: intPtr(5), PDFPositions: posA},
- {Text: "beta", DocType: "text", CKType: "text", TKNums: intPtr(5), PDFPositions: posB},
+ {Text: "alpha", DocType: "text", TKNums: intPtr(5), PDFPositions: posA},
+ {Text: "beta", DocType: "text", TKNums: intPtr(5), PDFPositions: posB},
},
}
got := mergeByTokenSizeFromJSON(items, 128, 0)
@@ -62,8 +62,8 @@ func TestMergeByTokenSizeFromJSON_ExtendsPositions(t *testing.T) {
posB := json.RawMessage(`[[4,5,6]]`)
items := [][]schema.ChunkDoc{
{
- {Text: "a", DocType: "text", CKType: "text", TKNums: intPtr(5), Positions: posA},
- {Text: "b", DocType: "text", CKType: "text", TKNums: intPtr(5), Positions: posB},
+ {Text: "a", DocType: "text", TKNums: intPtr(5), Positions: posA},
+ {Text: "b", DocType: "text", TKNums: intPtr(5), Positions: posB},
},
}
got := mergeByTokenSizeFromJSON(items, 128, 0)
@@ -102,8 +102,8 @@ func TestMergeByTokenSizeFromJSON_PositionsDecodeToMatrix(t *testing.T) {
posB := json.RawMessage(`[[6,7,8,9,10]]`)
items := [][]schema.ChunkDoc{
{
- {Text: "a", DocType: "text", CKType: "text", TKNums: intPtr(5), Positions: posA},
- {Text: "b", DocType: "text", CKType: "text", TKNums: intPtr(5), Positions: posB},
+ {Text: "a", DocType: "text", TKNums: intPtr(5), Positions: posA},
+ {Text: "b", DocType: "text", TKNums: intPtr(5), Positions: posB},
},
}
got := mergeByTokenSizeFromJSON(items, 128, 0)
@@ -144,9 +144,9 @@ func TestMergeByTokenSizeFromJSON_OverlapPrefixCarriesPrevPositions(t *testing.T
// first starts a fresh chunk carrying the WHOLE previous chunk as overlap.
items := [][]schema.ChunkDoc{
{
- {Text: "alpha", DocType: "text", CKType: "text", TKNums: intPtr(5), PDFPositions: posA},
- {Text: "beta", DocType: "text", CKType: "text", TKNums: intPtr(5), PDFPositions: posB},
- {Text: "gamma", DocType: "text", CKType: "text", TKNums: intPtr(5), PDFPositions: posC},
+ {Text: "alpha", DocType: "text", TKNums: intPtr(5), PDFPositions: posA},
+ {Text: "beta", DocType: "text", TKNums: intPtr(5), PDFPositions: posB},
+ {Text: "gamma", DocType: "text", TKNums: intPtr(5), PDFPositions: posC},
},
}
got := mergeByTokenSizeFromJSON(items, 20, 100)
@@ -208,12 +208,12 @@ func TestMergeByTokenSizeFromJSON_PartialOverlapPrefixCarriesOnlyTailPositions(t
// actual-count check agrees with the declared TKNums.)
items := [][]schema.ChunkDoc{
{
- {Text: "a", DocType: "text", CKType: "text", TKNums: intPtr(1), PDFPositions: posA},
- {Text: "b", DocType: "text", CKType: "text", TKNums: intPtr(1), PDFPositions: posB},
- {Text: "c", DocType: "text", CKType: "text", TKNums: intPtr(1), PDFPositions: posC},
- {Text: "d", DocType: "text", CKType: "text", TKNums: intPtr(1), PDFPositions: posD},
- {Text: "e", DocType: "text", CKType: "text", TKNums: intPtr(1), PDFPositions: posE},
- {Text: "f", DocType: "text", CKType: "text", TKNums: intPtr(1), PDFPositions: posF},
+ {Text: "a", DocType: "text", TKNums: intPtr(1), PDFPositions: posA},
+ {Text: "b", DocType: "text", TKNums: intPtr(1), PDFPositions: posB},
+ {Text: "c", DocType: "text", TKNums: intPtr(1), PDFPositions: posC},
+ {Text: "d", DocType: "text", TKNums: intPtr(1), PDFPositions: posD},
+ {Text: "e", DocType: "text", TKNums: intPtr(1), PDFPositions: posE},
+ {Text: "f", DocType: "text", TKNums: intPtr(1), PDFPositions: posF},
},
}
got := mergeByTokenSizeFromJSON(items, 5, 20)
@@ -254,7 +254,6 @@ func TestChunkFromItem_SlicesPositionsAcrossDelimiterPieces(t *testing.T) {
it := schema.ChunkDoc{
Text: "AAAA\nBBBB",
DocType: "text",
- CKType: "text",
PDFPositions: json.RawMessage(`[[1,0,200,0,40]]`),
}
got := chunkFromItem(it, compileDelimPattern([]string{"`\n`"}), false)
@@ -294,7 +293,6 @@ func TestSplitOversizedText_SlicedPositionsTrackPieceTextShare(t *testing.T) {
return schema.ChunkDoc{
Text: text,
DocType: "text",
- CKType: "text",
TKNums: intPtr(tokenizeStr(text)),
PDFPositions: json.RawMessage(`[[1,10,200,50,80]]`),
}
@@ -358,7 +356,6 @@ func TestChunkFromItem_SlicesPositionsIgnoresPositionTags(t *testing.T) {
it := schema.ChunkDoc{
Text: "AAAA@@1\t0\t200\t0\t40##\nCCCC",
DocType: "text",
- CKType: "text",
PDFPositions: json.RawMessage(`[[1,0,200,0,40]]`),
}
got := chunkFromItem(it, compileDelimPattern([]string{"`\n`"}), false)
@@ -392,7 +389,6 @@ func TestSplitOversizedText_SlicesPositionsIgnoreTags(t *testing.T) {
ck := schema.ChunkDoc{
Text: body,
DocType: "text",
- CKType: "text",
TKNums: intPtr(tokenizeStr(body)),
PDFPositions: json.RawMessage(`[[1,0,200,0,40]]`),
}
@@ -461,7 +457,6 @@ func TestMergeByTokenSizeFromJSON_ChainedOverlapNoHeadAccumulation(t *testing.T)
row = append(row, schema.ChunkDoc{
Text: string(rune('a' + i)),
DocType: "text",
- CKType: "text",
TKNums: intPtr(1),
PDFPositions: pos(i + 1),
})
@@ -601,7 +596,6 @@ func TestMergeByTokenSizeFromJSON_ChainedOverlapPerChunkConvergence(t *testing.T
row = append(row, schema.ChunkDoc{
Text: string(rune('a' + i)),
DocType: "text",
- CKType: "text",
TKNums: intPtr(1),
PDFPositions: pos(i + 1),
})
diff --git a/internal/ingestion/component/chunker/token_strict_cap_test.go b/internal/ingestion/component/chunker/token_strict_cap_test.go
index 106b458155..de20ef6af2 100644
--- a/internal/ingestion/component/chunker/token_strict_cap_test.go
+++ b/internal/ingestion/component/chunker/token_strict_cap_test.go
@@ -37,7 +37,7 @@ func TestMergeByTokenSizeFromJSON_HardCapNoOvershoot(t *testing.T) {
for i := 0; i < 8; i++ {
text := strings.TrimSpace(strings.Repeat("w ", 25))
sections = append(sections, schema.ChunkDoc{
- Text: text, DocType: "text", CKType: "text", TKNums: intPtr(tokenizeStr(text)),
+ Text: text, DocType: "text", TKNums: intPtr(tokenizeStr(text)),
})
}
got := mergeByTokenSizeFromJSON([][]schema.ChunkDoc{sections}, budget, 0)
@@ -60,7 +60,7 @@ func TestMergeByTokenSizeFromJSON_OverlapStrictCap(t *testing.T) {
for i := 0; i < 20; i++ {
text := strings.TrimSpace(strings.Repeat("w ", 10))
sections = append(sections, schema.ChunkDoc{
- Text: text, DocType: "text", CKType: "text", TKNums: intPtr(tokenizeStr(text)),
+ Text: text, DocType: "text", TKNums: intPtr(tokenizeStr(text)),
})
}
got := mergeByTokenSizeFromJSON([][]schema.ChunkDoc{sections}, budget, 20)
@@ -78,7 +78,7 @@ func TestMergeByTokenSizeFromJSON_OversizedUnitSplit(t *testing.T) {
const budget = 30
long := strings.TrimSpace(strings.Repeat("word ", 100))
items := [][]schema.ChunkDoc{{
- {Text: long, DocType: "text", CKType: "text", TKNums: intPtr(tokenizeStr(long))},
+ {Text: long, DocType: "text", TKNums: intPtr(tokenizeStr(long))},
}}
got := mergeByTokenSizeFromJSON(items, budget, 0)
merged := got[0]
@@ -235,8 +235,8 @@ func TestInvokeJSONPayload_HardCapEndToEnd(t *testing.T) {
// standing alone.
func TestMergeUnits_UnderCapBoundaryDecision(t *testing.T) {
units := []schema.ChunkDoc{
- {Text: "prev text", TKNums: intPtr(10), CKType: "text"},
- {Text: "incoming text", TKNums: intPtr(10), CKType: "text"},
+ {Text: "prev text", TKNums: intPtr(10)},
+ {Text: "incoming text", TKNums: intPtr(10)},
}
// prev+incoming == target: merge into one chunk.
got := mergeUnits(units, 20, 0, "\n")
@@ -255,8 +255,8 @@ func TestMergeUnits_UnderCapBoundaryDecision(t *testing.T) {
// An oversized incoming unit is expanded (split) instead of standing whole.
big := []schema.ChunkDoc{
- {Text: "prev text", TKNums: intPtr(10), CKType: "text"},
- {Text: strings.Repeat("word ", 100), TKNums: intPtr(100), CKType: "text"},
+ {Text: "prev text", TKNums: intPtr(10)},
+ {Text: strings.Repeat("word ", 100), TKNums: intPtr(100)},
}
bigMerged := mergeUnits(big, 30, 0, "\n")
if len(bigMerged) < 2 {
diff --git a/internal/ingestion/component/chunker/token_test.go b/internal/ingestion/component/chunker/token_test.go
index 27e21e1238..a370a29de3 100644
--- a/internal/ingestion/component/chunker/token_test.go
+++ b/internal/ingestion/component/chunker/token_test.go
@@ -97,8 +97,8 @@ func TestTokenChunkerPreservesSpreadsheetSegmentBoundaries(t *testing.T) {
t.Fatalf("segment chunks = %#v", chunks)
}
for i, chunk := range chunks {
- if chunk["ck_type"] != "table" {
- t.Errorf("chunk[%d] ck_type = %v, want table", i, chunk["ck_type"])
+ if chunk["doc_type_kwd"] != "table" {
+ t.Errorf("chunk[%d] doc_type_kwd = %v, want table", i, chunk["doc_type_kwd"])
}
}
}
@@ -405,9 +405,8 @@ func TestTokenChunker_InvokeJSONPayload_KeepsNonTextStandalone(t *testing.T) {
if got != wantTypes[i] {
t.Errorf("chunk %d: doc_type_kwd = %q, want %q (full chunk: %+v)", i, got, wantTypes[i], ch)
}
- ckType, _ := ch["ck_type"].(string)
- if ckType != wantTypes[i] {
- t.Errorf("chunk %d: ck_type = %q, want %q (derived from doc_type_kwd)", i, ckType, wantTypes[i])
+ if _, ok := ch["ck_type"]; ok {
+ t.Errorf("chunk %d must not emit the retired ck_type: %+v", i, ch)
}
}
// The two text segments on either side of the table/image must remain
@@ -473,7 +472,7 @@ func TestTokenChunkerMediaContextSpansUpstreamItems(t *testing.T) {
chunks := outputChunks(t, out)
var media map[string]any
for _, ck := range chunks {
- if ck["ck_type"] == tc.mediaType {
+ if ck["doc_type_kwd"] == tc.mediaType {
media = ck
break
}
@@ -548,7 +547,7 @@ func TestTokenChunkerDelimiterWindowUsesChildTokenCounts(t *testing.T) {
}
var media map[string]any
for _, ck := range outputChunks(t, out) {
- if ck["ck_type"] == "table" {
+ if ck["doc_type_kwd"] == "table" {
media = ck
}
}
@@ -569,7 +568,6 @@ func TestMaterializeMediaContextKeepsTokenCountInSync(t *testing.T) {
ck := schema.ChunkDoc{
Text: body,
DocType: "table",
- CKType: "table",
TKNums: intPtr(tokenizeStr(body)),
ContextAbove: "above",
ContextBelow: "below",
@@ -611,7 +609,7 @@ func TestTokenChunkerWindowTruncatesOnSentenceBoundary(t *testing.T) {
}
var media map[string]any
for _, ck := range outputChunks(t, out) {
- if ck["ck_type"] == "table" {
+ if ck["doc_type_kwd"] == "table" {
media = ck
}
}
@@ -712,8 +710,8 @@ func TestTokenChunker_SpreadsheetTablePreservesSelectionRange(t *testing.T) {
if len(chunks) != 1 {
t.Fatalf("chunks = %d, want one unsplit table item", len(chunks))
}
- if got := chunks[0]["ck_type"]; got != "table" {
- t.Errorf("ck_type = %v, want table", got)
+ if got := chunks[0]["doc_type_kwd"]; got != "table" {
+ t.Errorf("doc_type_kwd = %v, want table", got)
}
if got := chunks[0]["positions"]; !reflect.DeepEqual(got, wantPositions) {
t.Errorf("positions = %#v, want unchanged %#v", got, wantPositions)
@@ -1103,7 +1101,6 @@ func TestSplitByChildrenRecomputesTokenCounts(t *testing.T) {
children := splitByChildren([]schema.ChunkDoc{{
Text: parentText,
DocType: "text",
- CKType: "text",
TKNums: intPtr(tokenizeStr(parentText)),
}}, regexp.MustCompile(`\. `), true)
@@ -1125,7 +1122,6 @@ func TestApplyChildrenDelimText_RecomputesTokenCounts(t *testing.T) {
out := applyChildrenDelimText([]schema.ChunkDoc{{
Text: parentText,
DocType: "text",
- CKType: "text",
TKNums: intPtr(tokenizeStr(parentText)),
}}, regexp.MustCompile(`\. `), true)
diff --git a/internal/ingestion/component/knowledge_compiler/component_test.go b/internal/ingestion/component/knowledge_compiler/component_test.go
index bc27a695a4..724fb2b0ad 100644
--- a/internal/ingestion/component/knowledge_compiler/component_test.go
+++ b/internal/ingestion/component/knowledge_compiler/component_test.go
@@ -1415,7 +1415,7 @@ func TestOverlayTemplateConfigPlanPrecedence(t *testing.T) {
// TestKnowledgeCompiler_BuildInputsAcceptsMapSliceChunks locks the chunk-carrier
// contract: the upstream chunker hands chunks over as []map[string]any (see the
-// chunk map shape {"id","text","ck_type","doc_type_kwd","tk_nums"} observed from
+// chunk map shape {"id","text","doc_type_kwd","tk_nums"} observed from
// the running pipeline), not as []any of map. buildInputs must accept both
// shapes; a strict []any assertion alone would silently drop every chunk and
// leave the knowledge compiler with empty input (compiling nothing yet still
@@ -1423,8 +1423,8 @@ func TestOverlayTemplateConfigPlanPrecedence(t *testing.T) {
func TestKnowledgeCompiler_BuildInputsAcceptsMapSliceChunks(t *testing.T) {
in, err := buildInputs(map[string]any{
"chunks": []map[string]any{
- {"id": "c1", "text": "《三国演义》", "ck_type": "text"},
- {"id": "c2", "text": "滚滚长江东逝水", "ck_type": "text"},
+ {"id": "c1", "text": "《三国演义》"},
+ {"id": "c2", "text": "滚滚长江东逝水"},
},
}, common.Param{})
if err != nil {
diff --git a/internal/ingestion/component/parser_dispatch_test.go b/internal/ingestion/component/parser_dispatch_test.go
index 8cbfbf7e41..203ee4e489 100644
--- a/internal/ingestion/component/parser_dispatch_test.go
+++ b/internal/ingestion/component/parser_dispatch_test.go
@@ -133,8 +133,8 @@ func TestBuildParserOutputsNormalizesTextFormatsToJSON(t *testing.T) {
if len(items) != 2 {
t.Fatalf("markdown json item count = %d, want 2: %#v", len(items), items)
}
- if got := items[0]["ck_type"]; got != "heading" {
- t.Errorf("markdown item[0].ck_type = %v, want heading", got)
+ if got := items[0]["layout"]; got != "heading" {
+ t.Errorf("markdown item[0].layout = %v, want heading", got)
}
if got := items[0]["text"]; got != "# Title" {
t.Errorf("markdown item[0].text = %v, want # Title", got)
diff --git a/internal/ingestion/component/pdf_vision_dispatch.go b/internal/ingestion/component/pdf_vision_dispatch.go
index a607da0a2a..3ddb05df39 100644
--- a/internal/ingestion/component/pdf_vision_dispatch.go
+++ b/internal/ingestion/component/pdf_vision_dispatch.go
@@ -883,8 +883,19 @@ func mineruContentItemToJSON(item map[string]any, images map[string]*zip.File) m
text := mineruItemText(item, typ)
imageURI := ""
docType := "text"
- layout := typ
+ // External MinerU labels are mapped explicitly. Unknown labels must not
+ // acquire parser block-kind semantics implicitly; list/code are real
+ // structural types and stay structural.
+ layout := "text"
switch typ {
+ case "title":
+ layout = "title"
+ case "list":
+ layout = "list"
+ case "code":
+ layout = "code"
+ case "equation":
+ layout = "equation"
case "image", "chart":
docType = "image"
layout = "figure"
diff --git a/internal/ingestion/component/pdf_vision_dispatch_test.go b/internal/ingestion/component/pdf_vision_dispatch_test.go
index faf45f7ad2..df652a8839 100644
--- a/internal/ingestion/component/pdf_vision_dispatch_test.go
+++ b/internal/ingestion/component/pdf_vision_dispatch_test.go
@@ -217,6 +217,36 @@ func TestDispatchPaddleOCRPDFRejectsMissingBlocks(t *testing.T) {
}
}
+func TestMinerUContentItemToJSONExplicitLayoutMapping(t *testing.T) {
+ for _, tc := range []struct {
+ name string
+ item map[string]any
+ wantLayout string
+ wantDocType string
+ }{
+ {"list", map[string]any{"type": "list", "list_items": []any{"a", "b"}}, "list", "text"},
+ {"code", map[string]any{"type": "code", "code_body": "x"}, "code", "text"},
+ {"title", map[string]any{"type": "title", "text": "T"}, "title", "text"},
+ {"equation", map[string]any{"type": "equation", "text": "x=1"}, "equation", "text"},
+ {"unknown", map[string]any{"type": "footnote", "text": "x"}, "text", "text"},
+ {"image", map[string]any{"type": "image", "image_caption": []any{"cap"}}, "figure", "image"},
+ {"table", map[string]any{"type": "table", "table_body": "<table></table>"}, "table", "table"},
+ } {
+ t.Run(tc.name, func(t *testing.T) {
+ got := mineruContentItemToJSON(tc.item, nil)
+ if got == nil {
+ t.Fatal("item was dropped")
+ }
+ if got["layout"] != tc.wantLayout {
+ t.Errorf("layout = %v, want %q", got["layout"], tc.wantLayout)
+ }
+ if got["doc_type_kwd"] != tc.wantDocType {
+ t.Errorf("doc_type_kwd = %v, want %q", got["doc_type_kwd"], tc.wantDocType)
+ }
+ })
+ }
+}
+
func TestMinerUExtractJSONItemsKeepsImageAndPositions(t *testing.T) {
var buf bytes.Buffer
zw := zip.NewWriter(&buf)
diff --git a/internal/ingestion/component/schema/chunk_docitem_test.go b/internal/ingestion/component/schema/chunk_docitem_test.go
index fcb37dded4..a678bc720b 100644
--- a/internal/ingestion/component/schema/chunk_docitem_test.go
+++ b/internal/ingestion/component/schema/chunk_docitem_test.go
@@ -40,13 +40,13 @@ func TestChunkDocsFromDocItem_ParallelsMapPath(t *testing.T) {
{
name: "text with extra passthrough",
items: []parser.DocItem{
- parser.TextItem{Text: "hello", CKType: "heading", Extra: map[string]any{"layout": "1", "page_number": 3}},
+ parser.TextItem{Text: "hello", Extra: map[string]any{"layout": "1", "page_number": 3}},
},
},
{
name: "image with extra passthrough",
items: []parser.DocItem{
- parser.ImageItem{Text: "alt", Image: "data:image/png;base64,xxx", CKType: "image", Extra: map[string]any{"page_number": 2}},
+ parser.ImageItem{Text: "alt", Image: "data:image/png;base64,xxx", Extra: map[string]any{"page_number": 2}},
},
},
{
@@ -66,7 +66,7 @@ func TestChunkDocsFromDocItem_ParallelsMapPath(t *testing.T) {
items: []parser.DocItem{
parser.TextItem{Text: "a"},
parser.RawItem{M: map[string]any{"text": "b", "doc_type_kwd": "text"}},
- parser.ImageItem{Image: "i", CKType: "image"},
+ parser.ImageItem{Image: "i"},
},
},
// A canonical key present in Extra must NOT shadow the typed field when
@@ -83,7 +83,7 @@ func TestChunkDocsFromDocItem_ParallelsMapPath(t *testing.T) {
{
name: "empty image text must not be shadowed by extra text",
items: []parser.DocItem{
- parser.ImageItem{Text: "", Image: "i", CKType: "image", Extra: map[string]any{"text": "stale"}},
+ parser.ImageItem{Text: "", Image: "i", Extra: map[string]any{"text": "stale"}},
},
},
}
@@ -149,7 +149,7 @@ func TestChunkDocsFromDocItem_SkipsNilElement(t *testing.T) {
items := []parser.DocItem{
parser.TextItem{Text: "a"},
nil,
- parser.ImageItem{Image: "i", CKType: "image"},
+ parser.ImageItem{Image: "i"},
}
got, found, err := ChunkDocsFromDocItem(items)
if err != nil {
diff --git a/internal/ingestion/component/schema/chunk_types.go b/internal/ingestion/component/schema/chunk_types.go
index dce014701a..cf2de2738b 100644
--- a/internal/ingestion/component/schema/chunk_types.go
+++ b/internal/ingestion/component/schema/chunk_types.go
@@ -106,7 +106,6 @@ func (m ChunkerFileMeta) MarshalJSON() ([]byte, error) {
type ChunkDoc struct {
Text string `json:"text,omitempty"`
DocType string `json:"doc_type_kwd,omitempty"`
- CKType string `json:"ck_type,omitempty"`
TKNums *int `json:"tk_nums,omitempty"`
Mom string `json:"mom,omitempty"`
ImgID string `json:"img_id,omitempty"`
@@ -156,6 +155,8 @@ func (d *ChunkDoc) UnmarshalJSON(data []byte) error {
// tagger emits tag_feas and the vocabulary comes from a tag source file,
// so no chunk carries tag_kwd. Deleted rather than left in raw so it
// cannot fall through to Extra and be re-emitted by MarshalJSON.
+ // ck_type is a retired parser key: it has no field, but must stay in this
+ // list so an old payload cannot leak it back through Extra.
for _, key := range []string{
"text", "doc_type_kwd", "mom", "img_id",
"ck_type", "tk_nums", "layout", "layout_type", "layoutno", "image",
@@ -184,6 +185,10 @@ func (d ChunkDoc) MarshalJSON() ([]byte, error) {
return nil, err
}
for k, v := range d.Extra {
+ // ck_type is retired; a stray Extra key must not resurrect it.
+ if k == "ck_type" {
+ continue
+ }
if _, exists := raw[k]; !exists {
raw[k] = v
}
@@ -294,6 +299,9 @@ func (d ChunkDoc) ToMap() map[string]any {
return map[string]any{}
}
for k, raw := range d.Extra {
+ if k == "ck_type" {
+ continue
+ }
out[k] = decodeExtraValue(raw)
}
// Every pre-index chunk has one canonical text field. Keep the key even
diff --git a/internal/ingestion/component/schema/schema_test.go b/internal/ingestion/component/schema/schema_test.go
index dd53b20ced..862f438d81 100644
--- a/internal/ingestion/component/schema/schema_test.go
+++ b/internal/ingestion/component/schema/schema_test.go
@@ -241,7 +241,6 @@ func TestChunkDocSpreadsheetFieldsRoundTrip(t *testing.T) {
original := ChunkDoc{
Text: "<table><tr><th>ID</th></tr></table>",
DocType: "table",
- CKType: "table",
Sheet: "Orders",
SheetIndex: &sheetIndex,
}
@@ -278,6 +277,32 @@ func TestChunkDocLegacyRowIRKeysPassThrough(t *testing.T) {
t.Errorf("legacy key %q lost in round trip: %#v", key, out)
}
}
+ if _, ok := out["ck_type"]; ok {
+ t.Errorf("retired ck_type must not survive the round trip: %#v", out)
+ }
+}
+
+// A programmatic Extra entry must not resurrect the retired ck_type key on
+// either projection (MarshalJSON or ToMap).
+func TestChunkDocRetiredCKTypeCannotBeResurrectedViaExtra(t *testing.T) {
+ d := ChunkDoc{
+ Text: "body",
+ Extra: map[string]json.RawMessage{"ck_type": json.RawMessage(`"heading"`)},
+ }
+ if _, ok := d.ToMap()["ck_type"]; ok {
+ t.Fatalf("ToMap resurrected ck_type: %#v", d.ToMap())
+ }
+ data, err := json.Marshal(d)
+ if err != nil {
+ t.Fatalf("marshal: %v", err)
+ }
+ var raw map[string]any
+ if err := json.Unmarshal(data, &raw); err != nil {
+ t.Fatalf("unmarshal: %v", err)
+ }
+ if _, ok := raw["ck_type"]; ok {
+ t.Fatalf("MarshalJSON resurrected ck_type: %s", data)
+ }
}
func TestChunkDocVisualParentMetadataRoundTrip(t *testing.T) {
diff --git a/internal/ingestion/component/tokenizer_unit_test.go b/internal/ingestion/component/tokenizer_unit_test.go
index 2fcbd4b473..2db6eeb52b 100644
--- a/internal/ingestion/component/tokenizer_unit_test.go
+++ b/internal/ingestion/component/tokenizer_unit_test.go
@@ -436,7 +436,7 @@ func TestValidateTokenizerOutputs_SymbolOnlyContentLtksIsEmptyFails(t *testing.T
func TestChunkDocsToMaps_PreservesPDFPositions(t *testing.T) {
pos := json.RawMessage(`[[1,10,20,30,40],[2,15,25,35,45]]`)
chunks := []schema.ChunkDoc{
- {Text: "PDF paragraph", DocType: "text", CKType: "text",
+ {Text: "PDF paragraph", DocType: "text",
Positions: pos, PDFPositions: pos},
}
maps := schema.ChunkDocsToMaps(chunks)
diff --git a/internal/ingestion/task/indexdoc/process.go b/internal/ingestion/task/indexdoc/process.go
index 78684dedbb..1415fe7476 100644
--- a/internal/ingestion/task/indexdoc/process.go
+++ b/internal/ingestion/task/indexdoc/process.go
@@ -144,16 +144,19 @@ func cleanupConsumedChunkFields(ck map[string]any) {
}
// pipelineOnlyFields are the parser/chunker BOOKKEEPING keys: each one is
-// consumed inside the pipeline (ck_type drives chunk merging and the image-crop
-// decision, tk_nums carries the chunker's token count into the Tokenizer,
-// layout*/image/context_* describe the media block, and page_number/table_id/
-// sheet/headers/cells describe the table or spreadsheet block) and NONE of them
-// is a chunk-store column.
+// consumed inside the pipeline (layout drives block-kind decisions, tk_nums
+// carries the chunker's token count into the Tokenizer, layout*/image/context_*
+// describe the media block, and page_number/table_id/sheet/headers/cells
+// describe the table or spreadsheet block) and NONE of them is a chunk-store
+// column.
//
// The Python index doc carries none of them — its chunk builder emits only the
// persist fields — but Go's chunker hands them on, and the write boundary is
// where they stop: Elasticsearch and Vastbase would add a field for each key,
// and Infinity would drop each key at insert with a warning.
+//
+// ck_type is retired but stays here as a tombstone: a legacy producer can still
+// hand it on, and no engine has a column for it.
var pipelineOnlyFields = []string{
"ck_type", "tk_nums", "layout", "layout_type", "layoutno", "image",
"context_above", "context_below", "page_number", "_preserve_order",
diff --git a/internal/ingestion/task/indexdoc/process_test.go b/internal/ingestion/task/indexdoc/process_test.go
index ddedcb0bf2..31635a7cf8 100644
--- a/internal/ingestion/task/indexdoc/process_test.go
+++ b/internal/ingestion/task/indexdoc/process_test.go
@@ -664,9 +664,9 @@ func TestProcessChunkPositions_ChunkerRoundTrip(t *testing.T) {
// normalizes it to [][]float64 at the chunker boundary too, so the parser
// container type never reaches the indexer.
pdfItem := parser.ImageItem{
- Text: "fig",
- Image: "",
- CKType: "image",
+ Text: "fig",
+ Image: "",
+
Positions: parser.Positions{
parser.AbsoluteBox{PageNumber: 1, Left: 100, Right: 50, Top: 200, Bottom: 150},
},
diff --git a/internal/parser/parser/csv_parser_test.go b/internal/parser/parser/csv_parser_test.go
index 24fa1be83f..33fd994dec 100644
--- a/internal/parser/parser/csv_parser_test.go
+++ b/internal/parser/parser/csv_parser_test.go
@@ -27,8 +27,8 @@ func TestCSVParser_EmitsSpreadsheetRows(t *testing.T) {
t.Fatalf("len(res.JSON) = %d, want one HTML table item", len(res.JSON))
}
item := res.JSON[0].ToMap()
- if item["ck_type"] != "table" || item["doc_type_kwd"] != "table" {
- t.Fatalf("item types = %v/%v, want table/table", item["ck_type"], item["doc_type_kwd"])
+ if item["doc_type_kwd"] != "table" {
+ t.Fatalf("item doc_type_kwd = %v, want table", item["doc_type_kwd"])
}
wantText := "<table><caption>Data</caption>\n" +
"<tr><th>Name</th><th>Age</th><th>City</th></tr>\n" +
@@ -91,7 +91,7 @@ func TestCSVParserHTML4ExcelIsIgnored(t *testing.T) {
if res.Err != nil {
t.Fatalf("ParseWithResult failed: %v", res.Err)
}
- if len(res.JSON) != 1 || res.JSON[0].ToMap()["ck_type"] != "table" {
+ if len(res.JSON) != 1 || res.JSON[0].ToMap()["doc_type_kwd"] != "table" {
t.Fatalf("items = %#v, want one table item", res.JSON)
}
if !reflect.DeepEqual(res.JSON, plainRes.JSON) {
@@ -117,8 +117,8 @@ func TestCSVParser_ConfigureOutputFormatJSON(t *testing.T) {
if len(res.JSON) != 1 {
t.Fatalf("len(res.JSON) = %d, want one HTML table item", len(res.JSON))
}
- if res.JSON[0].ToMap()["ck_type"] != "table" {
- t.Fatalf("ck_type = %v, want table", res.JSON[0].ToMap()["ck_type"])
+ if res.JSON[0].ToMap()["doc_type_kwd"] != "table" {
+ t.Fatalf("doc_type_kwd = %v, want table", res.JSON[0].ToMap()["doc_type_kwd"])
}
if positions := res.JSON[0].ToMap()["positions"].([][]float64); len(positions) != 2 {
t.Fatalf("len(positions) = %d, want one tuple per <tr>", len(positions))
diff --git a/internal/parser/parser/doc_item.go b/internal/parser/parser/doc_item.go
index 9e26ebe55e..fa7adf08a3 100644
--- a/internal/parser/parser/doc_item.go
+++ b/internal/parser/parser/doc_item.go
@@ -39,13 +39,11 @@ type DocItem interface {
ToMap() map[string]any
}
-// TextItem is a body/heading/list/code span. CKType optionally carries the
-// legacy ck_type discriminator (heading/list/code/...); it is omitted from
-// ToMap when empty. Extra carries layout/page_number/level/... passthrough.
+// TextItem is a body/heading/list/code span. Extra carries
+// layout/page_number/level/... passthrough.
type TextItem struct {
- Text string
- CKType string
- Extra map[string]any
+ Text string
+ Extra map[string]any
}
func (t TextItem) ToMap() map[string]any {
@@ -53,20 +51,14 @@ func (t TextItem) ToMap() map[string]any {
"text": t.Text,
DocTypeKey: DocTypeText,
}
- if t.CKType != "" {
- m["ck_type"] = t.CKType
- }
return mergeExtra(m, t.Extra)
}
-// TableItem carries a parsed TableData. CKType is the legacy ck_type
-// discriminator (always "table" today, mirroring NewTableDataJSONItem); it is
-// emitted for pipeline compatibility and will be dropped in a later pass.
+// TableItem carries a parsed TableData.
type TableItem struct {
Table *entity.TableData
Sheet string
Positions Positions
- CKType string
Extra map[string]any
}
@@ -74,7 +66,6 @@ func (t TableItem) ToMap() map[string]any {
m := map[string]any{
"table": t.Table,
DocTypeKey: DocTypeTable,
- "ck_type": firstNonEmptyStr(t.CKType, DocTypeTable),
}
if t.Sheet != "" {
m["sheet"] = t.Sheet
@@ -86,15 +77,12 @@ func (t TableItem) ToMap() map[string]any {
}
// ImageItem is an embedded image. Text is the alt text, Image the data
-// URL/base64 payload (possibly empty when omitted). CKType carries the legacy
-// ck_type discriminator ("image" for xlsx/pptx/html/markdown, empty for docx);
-// it is omitted from ToMap when empty.
+// URL/base64 payload (possibly empty when omitted).
type ImageItem struct {
Text string
Image string
Sheet string
Positions Positions
- CKType string
Extra map[string]any
}
@@ -104,9 +92,6 @@ func (i ImageItem) ToMap() map[string]any {
"image": i.Image,
DocTypeKey: DocTypeImage,
}
- if i.CKType != "" {
- m["ck_type"] = i.CKType
- }
if i.Sheet != "" {
m["sheet"] = i.Sheet
}
@@ -188,9 +173,8 @@ func DocItemFromMap(m map[string]any) DocItem {
func textItemFromMap(m map[string]any) DocItem {
return TextItem{
- Text: stringField(m, "text"),
- CKType: stringField(m, "ck_type"),
- Extra: extraFromMap(m, "text", DocTypeKey, "ck_type"),
+ Text: stringField(m, "text"),
+ Extra: extraFromMap(m, "text", DocTypeKey, "ck_type"),
}
}
@@ -207,7 +191,6 @@ func imageItemFromMap(m map[string]any) DocItem {
return ImageItem{
Text: stringField(m, "text"),
Image: stringField(m, "image"),
- CKType: stringField(m, "ck_type"),
Sheet: stringField(m, "sheet"),
Positions: pos,
Extra: extraFromMap(m, skip...),
@@ -233,7 +216,6 @@ func tableItemFromMap(m map[string]any) DocItem {
Table: td,
Sheet: stringField(m, "sheet"),
Positions: pos,
- CKType: stringField(m, "ck_type"),
Extra: extraFromMap(m, skip...),
}
}
@@ -287,6 +269,10 @@ func ItemsToMaps(items []DocItem) []map[string]any {
// present, so a stray Extra key can never shadow text/table/image/doc_type_kwd.
func mergeExtra(m, extra map[string]any) map[string]any {
for k, v := range extra {
+ // ck_type is retired; a stray Extra key must not resurrect it.
+ if k == "ck_type" {
+ continue
+ }
if _, ok := m[k]; !ok {
m[k] = v
}
@@ -294,15 +280,6 @@ func mergeExtra(m, extra map[string]any) map[string]any {
return m
}
-func firstNonEmptyStr(values ...string) string {
- for _, v := range values {
- if v != "" {
- return v
- }
- }
- return ""
-}
-
// ---------------------------------------------------------------------------
// Position: strong-typed replacement for the loosely-typed `positions` wire
// field. The single wire shape is a 5-tuple array, but it carries TWO
diff --git a/internal/parser/parser/doc_item_test.go b/internal/parser/parser/doc_item_test.go
index 500c405942..59689f994f 100644
--- a/internal/parser/parser/doc_item_test.go
+++ b/internal/parser/parser/doc_item_test.go
@@ -34,18 +34,10 @@ func TestTextItem_ToMap(t *testing.T) {
mustMap(t, map[string]any{"text": "", DocTypeKey: DocTypeText},
TextItem{}.ToMap())
})
- t.Run("with text and no ck_type", func(t *testing.T) {
+ t.Run("with text", func(t *testing.T) {
mustMap(t, map[string]any{"text": "body", DocTypeKey: DocTypeText},
TextItem{Text: "body"}.ToMap())
})
- t.Run("ck_type emitted only when non-empty", func(t *testing.T) {
- mustMap(t, map[string]any{"text": "h", DocTypeKey: DocTypeText, "ck_type": "heading"},
- TextItem{Text: "h", CKType: "heading"}.ToMap())
- // empty CKType must NOT appear as a key.
- if _, ok := (TextItem{Text: "h"}).ToMap()["ck_type"]; ok {
- t.Error("ck_type present despite empty CKType")
- }
- })
t.Run("extra passthrough", func(t *testing.T) {
mustMap(t, map[string]any{
"text": "x",
@@ -60,21 +52,19 @@ func TestTextItem_ToMap(t *testing.T) {
}
func TestTextItem_ExtraDoesNotClobberCanonical(t *testing.T) {
- // An attacker/stray Extra key must never shadow the typed fields.
+ // An attacker/stray Extra key must never shadow the typed fields, and a
+ // retired ck_type must not be resurrected through Extra.
got := TextItem{
- Text: "real",
- CKType: "heading",
+ Text: "real",
Extra: map[string]any{
"text": "evil",
DocTypeKey: "evil",
- "ck_type": "evil",
"layout": "kept", // not a canonical key -> passes through
},
}.ToMap()
mustMap(t, map[string]any{
"text": "real",
DocTypeKey: DocTypeText,
- "ck_type": "heading",
"layout": "kept",
}, got)
}
@@ -84,25 +74,15 @@ func TestTableItem_ToMap(t *testing.T) {
mustMap(t, map[string]any{
"table": (*entity.TableData)(nil),
DocTypeKey: DocTypeTable,
- "ck_type": DocTypeTable,
}, TableItem{}.ToMap())
})
- t.Run("ck_type falls back to table when empty", func(t *testing.T) {
- mustMap(t, map[string]any{
- "table": (*entity.TableData)(nil),
- DocTypeKey: DocTypeTable,
- "ck_type": DocTypeTable,
- }, TableItem{CKType: ""}.ToMap())
- })
- t.Run("ck_type explicit wins", func(t *testing.T) {
+ t.Run("sheet and positions", func(t *testing.T) {
mustMap(t, map[string]any{
"table": (*entity.TableData)(nil),
DocTypeKey: DocTypeTable,
- "ck_type": "table",
"sheet": "Sheet1",
"positions": [][]float64{{0, 1, 1, 2, 2}},
}, TableItem{
- CKType: "table",
Sheet: "Sheet1",
Positions: SpreadsheetPositions([][]float64{{0, 1, 1, 2, 2}}),
}.ToMap())
@@ -119,30 +99,25 @@ func TestTableItem_ToMap(t *testing.T) {
}
func TestImageItem_ToMap(t *testing.T) {
- t.Run("docx-like: no ck_type, empty image", func(t *testing.T) {
- // docx image items historically set doc_type_kwd:"image" with no
- // ck_type and an empty image when over budget.
+ t.Run("docx-like: empty image", func(t *testing.T) {
+ // docx image items historically set doc_type_kwd:"image" with an empty
+ // image when over budget.
mustMap(t, map[string]any{
"text": "",
"image": "",
DocTypeKey: DocTypeImage,
}, ImageItem{}.ToMap())
- if _, ok := (ImageItem{}).ToMap()["ck_type"]; ok {
- t.Error("ck_type present despite empty CKType")
- }
})
- t.Run("xlsx-like: ck_type image + sheet + positions", func(t *testing.T) {
+ t.Run("xlsx-like: sheet + positions", func(t *testing.T) {
mustMap(t, map[string]any{
"text": "alt",
"image": "data:image/png;base64,xxx",
DocTypeKey: DocTypeImage,
- "ck_type": "image",
"sheet": "Sheet2",
"positions": [][]float64{{1, 0, 0, 3, 4}},
}, ImageItem{
Text: "alt",
Image: "data:image/png;base64,xxx",
- CKType: "image",
Sheet: "Sheet2",
Positions: SpreadsheetPositions([][]float64{{1, 0, 0, 3, 4}}),
}.ToMap())
@@ -166,7 +141,6 @@ func TestRawItem_Passthrough(t *testing.T) {
in := map[string]any{
"text": "hello",
DocTypeKey: DocTypeText,
- "ck_type": "heading",
"layout": "title",
"arbitrary": []int{1, 2, 3},
"nested": map[string]any{"k": "v"},
@@ -222,7 +196,6 @@ func TestConstructorWireShapeRegression(t *testing.T) {
mustMap(t, map[string]any{
"table": td,
DocTypeKey: DocTypeTable,
- "ck_type": DocTypeTable,
"sheet": "S",
"positions": [][]float64{{0, 0, 0, 1, 1}},
}, NewTableDataJSONItem(td, "S", SpreadsheetPositions([][]float64{{0, 0, 0, 1, 1}})).ToMap())
@@ -253,7 +226,6 @@ func TestDocItemFromMap_WireShapePreserved(t *testing.T) {
textMap := map[string]any{
"text": "hello",
DocTypeKey: DocTypeText,
- "ck_type": "heading",
"layout": "1",
"page_number": 3,
}
@@ -261,13 +233,11 @@ func TestDocItemFromMap_WireShapePreserved(t *testing.T) {
"text": "alt",
"image": "data:image/png;base64,xxx",
DocTypeKey: DocTypeImage,
- "ck_type": "image",
"page_number": 2,
}
tableMap := map[string]any{
"table": &entity.TableData{},
DocTypeKey: DocTypeTable,
- "ck_type": DocTypeTable,
"sheet": "s",
"positions": [][]float64{{1, 2}},
"sheet_index": 0,
@@ -281,13 +251,11 @@ func TestDocItemFromMap_WireShapePreserved(t *testing.T) {
"text": "alt",
"image": "data:image/png;base64,xxx",
DocTypeKey: DocTypeImage,
- "ck_type": "image",
"positions": [][]any{{1, 2, 3, 4}},
}
tableMapAny := map[string]any{
"table": &entity.TableData{},
DocTypeKey: DocTypeTable,
- "ck_type": DocTypeTable,
"positions": [][]any{{1, 2, 3, 4}},
}
// A table whose "table" value is a JSON-decoded map[string]any (not a typed
@@ -300,7 +268,6 @@ func TestDocItemFromMap_WireShapePreserved(t *testing.T) {
tableMapJSONDecoded := map[string]any{
"table": map[string]any{"grid": []any{}, "header_rows": float64(1)},
DocTypeKey: DocTypeTable,
- "ck_type": DocTypeTable,
}
for _, m := range []map[string]any{textMap, imageMap, tableMap, opaqueMap, imageMapAny, tableMapAny, tableMapJSONDecoded} {
@@ -325,11 +292,29 @@ func TestDocItemFromMap_WireShapePreserved(t *testing.T) {
}
}
+// A legacy payload may still carry the retired ck_type key. Typed conversion
+// must drop it rather than route it through Extra and re-emit it.
+func TestDocItemFromMap_DropsRetiredCKType(t *testing.T) {
+ legacy := map[string]any{
+ "text": "h",
+ DocTypeKey: DocTypeText,
+ "ck_type": "heading",
+ "layout": "heading",
+ }
+ got := DocItemFromMap(legacy).ToMap()
+ if _, ok := got["ck_type"]; ok {
+ t.Fatalf("retired ck_type leaked through ToMap: %#v", got)
+ }
+ if got["layout"] != "heading" {
+ t.Fatalf("layout lost during conversion: %#v", got)
+ }
+}
+
// DocItemsFromMaps maps a []map[string]any to typed DocItems the same way
// DocItemFromMap does, preserving order and skipping nothing.
func TestMapsToItems_OrderAndType(t *testing.T) {
maps := []map[string]any{
- {"text": "a", DocTypeKey: DocTypeText, "ck_type": "text"},
+ {"text": "a", DocTypeKey: DocTypeText},
{"foo": "bar"},
}
items := MapsToItems(maps)
diff --git a/internal/parser/parser/docx_ir.go b/internal/parser/parser/docx_ir.go
index 77ab144e32..3bb904be0a 100644
--- a/internal/parser/parser/docx_ir.go
+++ b/internal/parser/parser/docx_ir.go
@@ -599,7 +599,7 @@ func appendDOCXParagraphSections(sections []map[string]any, el docxIRElement, he
"doc_type_kwd": "text",
}
if heading {
- item["ck_type"] = "heading"
+ item["layout"] = "heading"
}
sections = append(sections, item)
}
diff --git a/internal/parser/parser/docx_ir_test.go b/internal/parser/parser/docx_ir_test.go
index 23037c6807..d26229c900 100644
--- a/internal/parser/parser/docx_ir_test.go
+++ b/internal/parser/parser/docx_ir_test.go
@@ -29,7 +29,7 @@ func TestBuildDOCXJSONSections_FromJSON(t *testing.T) {
got := buildDOCXJSONSections(irJSON, newEmbeddedMediaBudget())
want := []map[string]any{
- {"text": "Title", "image": nil, "doc_type_kwd": "text", "ck_type": "heading"},
+ {"text": "Title", "image": nil, "doc_type_kwd": "text", "layout": "heading"},
{"text": "Hello", "image": nil, "doc_type_kwd": "text"},
{"text": "", "image": "aGVsbG8=", "doc_type_kwd": "image"},
// The table sits under heading "Title", so it carries the
diff --git a/internal/parser/parser/docx_parser_test.go b/internal/parser/parser/docx_parser_test.go
index fe4de3724e..d724bf05aa 100644
--- a/internal/parser/parser/docx_parser_test.go
+++ b/internal/parser/parser/docx_parser_test.go
@@ -59,8 +59,8 @@ func TestBuildDOCXJSONSections_Headings(t *testing.T) {
if got, ok := item["doc_type_kwd"].(string); !ok || got != "text" {
t.Errorf("doc_type_kwd = %q, want %q", got, "text")
}
- if got, ok := item["ck_type"].(string); !ok || got != "heading" {
- t.Errorf("ck_type = %q, want %q", got, "heading")
+ if got, ok := item["layout"].(string); !ok || got != "heading" {
+ t.Errorf("layout = %q, want %q", got, "heading")
}
}
@@ -147,8 +147,8 @@ func TestBuildDOCXJSONSections_MixedContent(t *testing.T) {
if got, _ := sections[3]["doc_type_kwd"].(string); got != "text" {
t.Errorf("item[3].doc_type_kwd = %q, want %q", got, "text")
}
- if got, _ := sections[3]["ck_type"].(string); got != "heading" {
- t.Errorf("item[3].ck_type = %q, want %q", got, "heading")
+ if got, _ := sections[3]["layout"].(string); got != "heading" {
+ t.Errorf("item[3].layout = %q, want %q", got, "heading")
}
}
diff --git a/internal/parser/parser/html_parser.go b/internal/parser/parser/html_parser.go
index 375d61eb0f..a9ebf37524 100644
--- a/internal/parser/parser/html_parser.go
+++ b/internal/parser/parser/html_parser.go
@@ -168,17 +168,17 @@ func walkHTMLBlocksWithState(root *html.Node, out *[]map[string]any, state *html
// Emit the <table> as a SINGLE structured doc_type_kwd:"table"
// item, in document order. Keeping the full <table>…</table>
// markup (not flattened) preserves row/column structure for
- // embedding, retrieval, and LLM rendering, and doc_type_kwd/
- // ck_type drives downstream table handling (discrete chunk +
+ // embedding, retrieval, and LLM rendering, and doc_type_kwd
+ // drives downstream table handling (discrete chunk +
// table context). We emit ONLY this item — no duplicate
// doc_type_kwd:"text" copy — so the table is embedded once and
// its markup does not pollute neighbouring prose chunks.
emitHTMLTable(child, out, state)
continue
}
- ckType := htmlTagToCkType(tag)
+ layout := htmlTagToLayout(tag)
trim := tag != "pre" && tag != "textarea"
- htmlLeafTextWithState(child, out, ckType, trim, state)
+ htmlLeafTextWithState(child, out, layout, trim, state)
}
}
@@ -220,24 +220,26 @@ func isHTMLInlineElement(n *html.Node) bool {
return false
}
-func appendHTMLTextItem(out *[]map[string]any, text, ckType string, trim bool) {
+func appendHTMLTextItem(out *[]map[string]any, text, layout string, trim bool) {
if trim {
text = strings.TrimSpace(text)
}
if text == "" {
return
}
- *out = append(*out, map[string]any{
+ item := map[string]any{
"text": text,
"doc_type_kwd": "text",
- "ck_type": ckType,
- })
+ }
+ if layout != "" {
+ item["layout"] = layout
+ }
+ *out = append(*out, item)
}
-// htmlTagToCkType maps HTML block tags to the python `ck_type`
-// vocabulary used downstream by TitleChunker and similar
-// components. Tags not in the map fall back to "text".
-func htmlTagToCkType(tag string) string {
+// htmlTagToLayout maps an HTML block tag to its structural layout value.
+// Media tags are handled by their own emitters, so they have no entry here.
+func htmlTagToLayout(tag string) string {
switch tag {
case "h1", "h2", "h3", "h4", "h5", "h6":
return "heading"
@@ -247,14 +249,10 @@ func htmlTagToCkType(tag string) string {
return "list"
case "pre", "code":
return "code"
- case "table", "tr", "td", "th":
- return "table"
case "blockquote":
return "quote"
- case "img":
- return "image"
}
- return "text"
+ return ""
}
// leafWriter accumulates the visible text of an HTML subtree while applying
@@ -323,27 +321,28 @@ func (w *leafWriter) hardBreak() {
// subtrees are skipped. Whitespace is folded per CSS rules (so
// "<h1>Hello world</h1>" becomes "Hello world" and "<br>" survives as a
// real line break), while <pre>/<textarea> keep their source formatting
-// verbatim. ckType is the block's ck_type (from htmlTagToCkType) applied to the
-// accumulated prose item; trim controls whether trailing/leading whitespace is
-// collapsed (false for <pre>/<textarea>, which must stay verbatim). Any <table>
+// verbatim. layout is the block's structural layout value (from
+// htmlTagToLayout) applied to the accumulated prose item; trim controls
+// whether trailing/leading whitespace is collapsed (false for
+// <pre>/<textarea>, which must stay verbatim). Any <table>
// encountered in the subtree is emitted as a single structured
// doc_type_kwd:"table" item at its document position (see walkHTMLLeaf's
// "table" case) — the prose around it is flushed as ordinary text items, so
// the table is never relocated to the end and never duplicated.
-func htmlLeafTextWithState(n *html.Node, out *[]map[string]any, ckType string, trim bool, state *htmlWalkState) {
+func htmlLeafTextWithState(n *html.Node, out *[]map[string]any, layout string, trim bool, state *htmlWalkState) {
var b bytes.Buffer
w := &leafWriter{b: &b}
- walkHTMLLeaf(n, w, out, ckType, trim, state)
- flushLeafText(w, out, ckType, trim)
+ walkHTMLLeaf(n, w, out, layout, trim, state)
+ flushLeafText(w, out, layout, trim)
}
// flushLeafText emits any text accumulated in w as a doc_type_kwd:"text" item
-// (dropping empty output) and resets the writer. ckType/trim mirror
+// (dropping empty output) and resets the writer. layout/trim mirror
// appendHTMLTextItem. It is called at block boundaries and at <table> elements
// so tables are emitted in their original document position rather than being
// relocated. Prose with no specific block tag (e.g. text accumulated just
-// before a nested table) is flushed as ck_type "text".
-func flushLeafText(w *leafWriter, out *[]map[string]any, ckType string, trim bool) {
+// before a nested table) is flushed with no layout.
+func flushLeafText(w *leafWriter, out *[]map[string]any, layout string, trim bool) {
text := w.b.String()
if trim {
text = strings.TrimSpace(text)
@@ -351,14 +350,14 @@ func flushLeafText(w *leafWriter, out *[]map[string]any, ckType string, trim boo
if text == "" {
return
}
- appendHTMLTextItem(out, text, ckType, false)
+ appendHTMLTextItem(out, text, layout, false)
w.b.Reset()
w.lastSpace = false
w.lineStart = true
w.endsNL = false
}
-func walkHTMLLeaf(n *html.Node, w *leafWriter, out *[]map[string]any, ckType string, trim bool, state *htmlWalkState) {
+func walkHTMLLeaf(n *html.Node, w *leafWriter, out *[]map[string]any, layout string, trim bool, state *htmlWalkState) {
switch n.Type {
case html.TextNode:
w.writeText(n.Data)
@@ -378,7 +377,7 @@ func walkHTMLLeaf(n *html.Node, w *leafWriter, out *[]map[string]any, ckType str
}
return
}
- flushLeafText(w, out, ckType, trim)
+ flushLeafText(w, out, layout, trim)
appendHTMLImageItem(out, state, src, htmlAttribute(n, "alt"), "", 0, 0)
return
}
@@ -386,7 +385,7 @@ func walkHTMLLeaf(n *html.Node, w *leafWriter, out *[]map[string]any, ckType str
// Verbatim: no folding, no injected block breaks.
w.pre = true
for child := n.FirstChild; child != nil; child = child.NextSibling {
- walkHTMLLeaf(child, w, out, ckType, trim, state)
+ walkHTMLLeaf(child, w, out, layout, trim, state)
}
w.pre = false
return
@@ -412,7 +411,7 @@ func walkHTMLLeaf(n *html.Node, w *leafWriter, out *[]map[string]any, ckType str
w.hardBreak()
}
for child := n.FirstChild; child != nil; child = child.NextSibling {
- walkHTMLLeaf(child, w, out, ckType, trim, state)
+ walkHTMLLeaf(child, w, out, layout, trim, state)
}
if !w.pre && isBlockTag(n.Data) && w.b.Len() > 0 && !w.endsNL {
w.hardBreak()
@@ -443,7 +442,6 @@ func emitHTMLTable(n *html.Node, out *[]map[string]any, state *htmlWalkState) {
td, _ := tableutil.ParseTableHTML(markup)
table := map[string]any{
"doc_type_kwd": "table",
- "ck_type": "table",
"table": td,
}
tableID := ""
@@ -521,7 +519,6 @@ func appendHTMLImageItem(out *[]map[string]any, state *htmlWalkState, src, alt,
item := map[string]any{
"text": strings.TrimSpace(alt),
"doc_type_kwd": "image",
- "ck_type": "image",
"media_order": state.mediaOrder,
}
item["image"] = src
diff --git a/internal/parser/parser/html_parser_align_test.go b/internal/parser/parser/html_parser_align_test.go
index 5c0d93a621..dcf5f6610e 100644
--- a/internal/parser/parser/html_parser_align_test.go
+++ b/internal/parser/parser/html_parser_align_test.go
@@ -9,7 +9,7 @@ import (
// TestHTMLParser_TableProducesStructuredItems asserts the HTML table contract:
// a <table> is emitted as exactly ONE structured doc_type_kwd:"table"/
-// ck_type:"table" item, in its original document position (not relocated to
+// layout-free structured table item, in its original document position (not relocated to
// the end), with NO duplicate doc_type_kwd:"text" copy carrying the <table>
// markup. Keeping the full <table>…</table> markup (not flattened) preserves
// row/column structure for embedding/retrieval/LLM rendering, and the single
@@ -61,8 +61,8 @@ func TestHTMLParser_TableProducesStructuredItems(t *testing.T) {
if tableCount != 1 {
t.Fatalf("structured doc_type_kwd:\"table\" item count = %d, want exactly 1", tableCount)
}
- if got, want := res.JSON[tableIdx].ToMap()["ck_type"], "table"; got != want {
- t.Errorf("structured table item ck_type = %v, want %v", got, want)
+ if _, ok := res.JSON[tableIdx].ToMap()["ck_type"]; ok {
+ t.Errorf("retired ck_type present on structured table item: %#v", res.JSON[tableIdx].ToMap())
}
if !strings.Contains(tableText, "<table") ||
!strings.Contains(tableText, "Name") ||
@@ -144,8 +144,8 @@ func TestHTMLParser_NestedTableProducesStructuredItems(t *testing.T) {
if tableCount != 1 {
t.Fatalf("structured doc_type_kwd:\"table\" item count = %d, want exactly 1", tableCount)
}
- if got, want := res.JSON[tableIdx].ToMap()["ck_type"], "table"; got != want {
- t.Errorf("structured nested table item ck_type = %v, want %v", got, want)
+ if _, ok := res.JSON[tableIdx].ToMap()["ck_type"]; ok {
+ t.Errorf("retired ck_type present on structured nested table item: %#v", res.JSON[tableIdx].ToMap())
}
if !strings.Contains(tableText, "<table") ||
!strings.Contains(tableText, "Name") ||
@@ -383,25 +383,24 @@ func TestHTMLParser_NestedListItemsPreserved(t *testing.T) {
}
// The whole top-level <ul> is emitted as ONE text item tagged
- // ck_type:"list" (Go's "one item per block-level element" model). Guard
+ // layout:"list" (Go's "one item per block-level element" model). Guard
// against a regression that would drop the list tag or split it
// unexpectedly. Note: a list nested inside a non-list container
// (div/section/…) is folded into that container's single text item and
- // therefore loses ck_type:"list" (becomes "text") — a fidelity nuance,
- // not a collapse, since downstream chunking reads doc_type_kwd not ck_type.
+ // therefore loses layout:"list" — a fidelity nuance, not a collapse.
var listText string
for _, it := range ItemsToMaps(res.JSON) {
- if it["ck_type"] == "list" {
+ if it["layout"] == "list" {
listText, _ = it["text"].(string)
break
}
}
if listText == "" {
- t.Fatalf("top-level <ul> not tagged ck_type:\"list\"; got items: %#v", res.JSON)
+ t.Fatalf("top-level <ul> not tagged layout:\"list\"; got items: %#v", res.JSON)
}
for _, want := range []string{"Item 1", "Item 2", "Sub A", "Sub B"} {
if !strings.Contains(listText, want) {
- t.Errorf("ck_type:\"list\" item missing %q: %q", want, listText)
+ t.Errorf("layout:\"list\" item missing %q: %q", want, listText)
}
}
}
@@ -413,7 +412,7 @@ func TestHTMLParser_NestedListItemsPreserved(t *testing.T) {
// structure and the LLM / user would see broken code. This mirrors Python's
// RAGFlowHtmlParser, which keeps pre/textarea verbatim (_PRE_TAGS in
// deepdoc/parser/html_parser.py). Both blocks keep doc_type_kwd:"text"; <pre>
-// is tagged ck_type:"code" while <textarea> falls back to ck_type:"text"
+// is tagged layout:"code" while <textarea> has no structural layout
// (htmlTagToCkType has no textarea case) — a labeling nuance, not a content
// divergence. Note: the single leading newline immediately after <pre>/<textarea>
// is dropped by the HTML parser per spec (pre/textarea eat one leading newline);
@@ -453,8 +452,8 @@ SELECT * FROM t
if preItem == nil {
t.Fatalf("no <pre> item found; got items: %#v", res.JSON)
}
- if preItem["ck_type"] != "code" {
- t.Errorf("<pre> item ck_type = %v, want code", preItem["ck_type"])
+ if preItem["layout"] != "code" {
+ t.Errorf("<pre> item layout = %v, want code", preItem["layout"])
}
preText, _ := preItem["text"].(string)
// 4-space and 8-space indents must survive verbatim (a folded path would
diff --git a/internal/parser/parser/markdown_parser.go b/internal/parser/parser/markdown_parser.go
index 2a20b9df7d..e16c41984a 100644
--- a/internal/parser/parser/markdown_parser.go
+++ b/internal/parser/parser/markdown_parser.go
@@ -386,27 +386,26 @@ func markdownTableCells(line string) []string {
//
// Tables: a GFM/HTML table is rendered inline as a single <table> HTML
// block by renderMarkdownTablesInlineText and kept as one HTML block.
-// It is emitted as ONE structured item (doc_type_kwd:"table",
-// ck_type:"table") in its original document position — there is no
+// It is emitted as ONE structured item (doc_type_kwd:"table")
+// in its original document position — there is no
// duplicate doc_type_kwd:"text" copy. The downstream chunker consumes
// doc_type_kwd:"table" to keep the table whole and attach table context
// to neighbouring chunks (chunker/token.go). Non-table HTML blocks
-// (<div>, <style>, …) are emitted as ordinary text with no ck_type.
+// (<div>, <style>, …) are emitted as ordinary text with no layout.
func walkMarkdownBlocksWithImages(ctx context.Context, doc ast.Node, out *[]map[string]any, flatten, fetchRemoteImages bool, budget *embeddedMediaBudget) int {
unresolvedImages := 0
for _, child := range doc.GetChildren() {
- var ckType string
+ var layout string
var docTypeKwd string
var txt string
switch n := child.(type) {
case *ast.Heading:
txt = headingText(n)
- ckType = "heading"
+ layout = "heading"
docTypeKwd = "text"
case *ast.Paragraph:
txt = leafText(n)
- ckType = "text"
docTypeKwd = "text"
case *ast.List:
// Python's _markdown keeps one item per list entry, marker included
@@ -420,21 +419,21 @@ func walkMarkdownBlocksWithImages(ctx context.Context, doc ast.Node, out *[]map[
*out = append(*out, map[string]any{
"text": entry,
"doc_type_kwd": "text",
- "ck_type": "list",
+ "layout": "list",
})
}
continue
case *ast.CodeBlock:
txt = leafText(n)
- ckType = "code"
+ layout = "code"
docTypeKwd = "text"
case *ast.HTMLBlock:
// An HTML block is either an inlined table (kept as a single
// HTML block thanks to the blank lines renderMarkdownTablesInline
// wraps around it) or a plain HTML block such as <div>/<style>.
// Only a table is emitted as a structured table item; everything
- // else is treated as ordinary text (no ck_type). We emit exactly
- // ONE item (doc_type_kwd:"table"/ck_type:"table") in document
+ // else is treated as ordinary text (no layout). We emit exactly
+ // ONE item (doc_type_kwd:"table") in document
// order — no duplicate doc_type_kwd:"text" copy — so the table is
// embedded once and its markup does not pollute prose chunks.
txt = leafText(n)
@@ -447,13 +446,12 @@ func walkMarkdownBlocksWithImages(ctx context.Context, doc ast.Node, out *[]map[
td, _ := tableutil.ParseTableHTML(txt)
table := map[string]any{
"doc_type_kwd": "table",
- "ck_type": "table",
"table": td,
}
*out = append(*out, table)
continue
}
- // Non-table HTML block: ordinary text, no ck_type.
+ // Non-table HTML block: ordinary text, no layout.
docTypeKwd = "text"
default:
txt = leafText(n)
@@ -467,8 +465,8 @@ func walkMarkdownBlocksWithImages(ctx context.Context, doc ast.Node, out *[]map[
"text": txt,
"doc_type_kwd": docTypeKwd,
}
- if ckType != "" {
- item["ck_type"] = ckType
+ if layout != "" {
+ item["layout"] = layout
}
// Resolve Markdown images from the AST node of THIS block only, so
@@ -719,7 +717,7 @@ func resolveAndValidateHost(ctx context.Context, host string) (net.IP, error) {
// headingText returns a canonical ATX heading (marker plus inline text).
// Keeping the marker in the parser payload preserves the Markdown structure
-// for GeneralChunker and downstream LLM prompts; ck_type alone does not carry
+// for GeneralChunker and downstream LLM prompts; layout alone does not carry
// the heading level.
func headingText(h *ast.Heading) string {
var buf bytes.Buffer
diff --git a/internal/parser/parser/markdown_parser_test.go b/internal/parser/parser/markdown_parser_test.go
index 27689d4b3d..b63b101ea5 100644
--- a/internal/parser/parser/markdown_parser_test.go
+++ b/internal/parser/parser/markdown_parser_test.go
@@ -34,8 +34,8 @@ func TestMarkdownParser_ParseWithResult_Basic(t *testing.T) {
if got, _ := res.JSON[0].ToMap()["text"].(string); got != "# Hello" {
t.Fatalf("first item text = %q, want %q", got, "# Hello")
}
- if got, _ := res.JSON[0].ToMap()["ck_type"].(string); got != "heading" {
- t.Fatalf("first item ck_type = %q, want %q", got, "heading")
+ if got, _ := res.JSON[0].ToMap()["layout"].(string); got != "heading" {
+ t.Fatalf("first item layout = %q, want %q", got, "heading")
}
}
@@ -52,8 +52,8 @@ func TestMarkdownParser_ParseWithResult_UTF8BOM(t *testing.T) {
if got, _ := res.JSON[0].ToMap()["text"].(string); got != "# Hello" {
t.Fatalf("first item text = %q, want %q", got, "# Hello")
}
- if got, _ := res.JSON[0].ToMap()["ck_type"].(string); got != "heading" {
- t.Fatalf("first item ck_type = %q, want %q", got, "heading")
+ if got, _ := res.JSON[0].ToMap()["layout"].(string); got != "heading" {
+ t.Fatalf("first item layout = %q, want %q", got, "heading")
}
}
@@ -79,7 +79,7 @@ func TestMarkdownParser_ListItemsKeepWordBoundaries(t *testing.T) {
}
var lists []string
for _, item := range ItemsToMaps(res.JSON) {
- if ck, _ := item["ck_type"].(string); ck == "list" {
+ if ck, _ := item["layout"].(string); ck == "list" {
text, _ := item["text"].(string)
lists = append(lists, text)
}
@@ -141,7 +141,7 @@ func TestMarkdownParser_OrderedListKeepsNumbers(t *testing.T) {
}
var lists []string
for _, item := range ItemsToMaps(res.JSON) {
- if ck, _ := item["ck_type"].(string); ck == "list" {
+ if ck, _ := item["layout"].(string); ck == "list" {
text, _ := item["text"].(string)
lists = append(lists, text)
}
@@ -273,8 +273,8 @@ func TestMarkdownParser_ParseWithResult_RendersTableInline(t *testing.T) {
switch kd {
case "table":
tableCount++
- if ck, _ := item["ck_type"].(string); ck != "table" {
- t.Errorf("table item ck_type = %q, want \"table\"", ck)
+ if _, ok := item["ck_type"]; ok {
+ t.Errorf("retired ck_type present on table item: %#v", item)
}
td, ok := item["table"].(*entity.TableData)
if !ok || td == nil {
@@ -642,8 +642,8 @@ func TestMarkdownParser_TableNotCollapsed(t *testing.T) {
if _, ok := item["table"].(*entity.TableData); !ok {
t.Fatalf("table item missing *entity.TableData: %#v", item)
}
- if ck, _ := item["ck_type"].(string); ck != "table" {
- t.Fatalf("table item ck_type = %q, want \"table\"", ck)
+ if _, ok := item["ck_type"]; ok {
+ t.Fatalf("retired ck_type present on table item: %#v", item)
}
}
}
@@ -707,8 +707,8 @@ func TestMarkdownParser_TableWithSurroundingText(t *testing.T) {
if _, ok := item["table"].(*entity.TableData); !ok {
t.Fatalf("table item missing *entity.TableData: %#v", item)
}
- if ck, _ := item["ck_type"].(string); ck != "table" {
- t.Fatalf("table item ck_type = %q, want \"table\"", ck)
+ if _, ok := item["ck_type"]; ok {
+ t.Fatalf("retired ck_type present on table item: %#v", item)
}
}
}
@@ -740,8 +740,8 @@ func TestMarkdownParser_TableWithSurroundingText(t *testing.T) {
}
// TestMarkdownParser_NonTableHTMLBlockNotTable guards against the blanket
-// ck_type:"table" bug: a raw HTML block that is NOT a <table> (e.g. <div>,
-// <style>) must be emitted as ordinary text with no ck_type, so downstream
+// raw-table bug: a raw HTML block that is NOT a <table> (e.g. <div>,
+// <style>) must be emitted as ordinary text with no layout, so downstream
// consumers (chunker) do not mistake it for a table.
func TestMarkdownParser_NonTableHTMLBlockNotTable(t *testing.T) {
ctx := t.Context()
@@ -754,8 +754,8 @@ func TestMarkdownParser_NonTableHTMLBlockNotTable(t *testing.T) {
for _, item := range ItemsToMaps(res.JSON) {
text, _ := item["text"].(string)
if strings.Contains(text, "<div") || strings.Contains(text, "<style") {
- if ck, ok := item["ck_type"].(string); ok && ck == "table" {
- t.Fatalf("non-table HTML block wrongly tagged ck_type:\"table\": %q", text)
+ if _, ok := item["ck_type"]; ok {
+ t.Fatalf("retired ck_type present on non-table item: %q", text)
}
if kd, _ := item["doc_type_kwd"].(string); kd == "table" {
t.Fatalf("non-table HTML block wrongly tagged doc_type_kwd:\"table\": %q", text)
@@ -827,8 +827,8 @@ func TestMarkdownParser_RawHTMLTableHandled(t *testing.T) {
if _, ok := item["table"].(*entity.TableData); !ok {
t.Fatalf("raw table item missing *entity.TableData: %#v", item)
}
- if ck, _ := item["ck_type"].(string); ck != "table" {
- t.Fatalf("raw table item ck_type = %q, want \"table\"", ck)
+ if _, ok := item["ck_type"]; ok {
+ t.Fatalf("retired ck_type present on raw table item: %#v", item)
}
}
}
diff --git a/internal/parser/parser/office_table_render.go b/internal/parser/parser/office_table_render.go
index 14b9f60119..f91947a3c4 100644
--- a/internal/parser/parser/office_table_render.go
+++ b/internal/parser/parser/office_table_render.go
@@ -96,7 +96,7 @@ func trimCells(row []string) []string {
// Every tuple is a spreadsheet position, [sheet, rowStart, rowEnd, colStart,
// colEnd]. The same `positions` field carries PDF layout boxes
// ([page, left, right, top, bottom]) on PDF items, so a consumer must key the
-// vocabulary off sheet_index / ck_type and never reinterpret one as the other.
+// vocabulary off sheet_index / doc_type_kwd and never reinterpret one as the other.
func buildSheetItems(records [][]string, sheet string, sheetIndex, headerRow int, dataRows []int, images []map[string]any) []map[string]any {
sortImagesByAnchor(images)
if len(records) == 0 {
@@ -356,7 +356,6 @@ cellLoop:
item := map[string]any{
"text": alt,
"doc_type_kwd": "image",
- "ck_type": "image",
"sheet": sheet,
"cell": cell,
"row_start": row,
diff --git a/internal/parser/parser/office_table_render_test.go b/internal/parser/parser/office_table_render_test.go
index 6c04242f8a..b491e30a33 100644
--- a/internal/parser/parser/office_table_render_test.go
+++ b/internal/parser/parser/office_table_render_test.go
@@ -30,8 +30,8 @@ func TestXLSXParserEmitsSegmentedHTMLTable(t *testing.T) {
t.Fatalf("items = %d, want one segmented HTML table item", len(res.JSON))
}
item := res.JSON[0].ToMap()
- if item["ck_type"] != "table" || item["doc_type_kwd"] != "table" {
- t.Fatalf("item types = ck_type:%v doc_type:%v", item["ck_type"], item["doc_type_kwd"])
+ if item["doc_type_kwd"] != "table" {
+ t.Fatalf("item doc_type_kwd = %v, want table", item["doc_type_kwd"])
}
wantText := "<table><caption>Sheet1</caption>\n" +
"<tr><th>ID</th><th>Status</th></tr>\n" +
@@ -81,7 +81,7 @@ func TestXLSXParserHTML4ExcelIsIgnored(t *testing.T) {
if len(res.JSON) != 1 {
t.Fatalf("items = %d, want one atomic table item", len(res.JSON))
}
- if res.JSON[0].ToMap()["ck_type"] != "table" || res.JSON[0].ToMap()["table"] == nil {
+ if res.JSON[0].ToMap()["doc_type_kwd"] != "table" || res.JSON[0].ToMap()["table"] == nil {
t.Fatalf("html4excel item = %#v", res.JSON[0])
}
if !reflect.DeepEqual(res.JSON, plainRes.JSON) {
diff --git a/internal/parser/parser/parse_result.go b/internal/parser/parser/parse_result.go
index 3afa7225da..3398e8997b 100644
--- a/internal/parser/parser/parse_result.go
+++ b/internal/parser/parser/parse_result.go
@@ -117,6 +117,5 @@ func NewTableDataJSONItem(td *entity.TableData, sheet string, positions Position
Table: td,
Sheet: sheet,
Positions: positions,
- CKType: DocTypeTable,
}
}
diff --git a/internal/parser/parser/parse_result_test.go b/internal/parser/parser/parse_result_test.go
index 6f47acd44c..073084b577 100644
--- a/internal/parser/parser/parse_result_test.go
+++ b/internal/parser/parser/parse_result_test.go
@@ -23,7 +23,7 @@ import (
"ragflow/internal/entity"
)
-func TestNewTableDataJSONItem_IncludesDocTypeAndChunkType(t *testing.T) {
+func TestNewTableDataJSONItem_IncludesDocType(t *testing.T) {
td := &entity.TableData{
Grid: [][]entity.TableCell{{{Text: "col"}}, {{Text: "value"}}},
HeaderRows: 1,
@@ -35,9 +35,6 @@ func TestNewTableDataJSONItem_IncludesDocTypeAndChunkType(t *testing.T) {
if got, want := m[DocTypeKey], DocTypeTable; got != want {
t.Fatalf("item[%q] = %v, want %v", DocTypeKey, got, want)
}
- if got, want := m["ck_type"], DocTypeTable; got != want {
- t.Fatalf("item[%q] = %v, want %v", "ck_type", got, want)
- }
got, ok := m["table"].(*entity.TableData)
if !ok {
t.Fatalf("item[%q] = %T, want *entity.TableData", "table", m["table"])
@@ -175,8 +172,8 @@ func TestMarkdownParser_ParseWithResult(t *testing.T) {
if txt, _ := heading["text"].(string); !strings.Contains(txt, "Title") {
t.Errorf("JSON[0].text = %q, want contains 'Title'", txt)
}
- if got, _ := heading["ck_type"].(string); got != "heading" {
- t.Errorf("JSON[0].ck_type = %q, want heading", got)
+ if got, _ := heading["layout"].(string); got != "heading" {
+ t.Errorf("JSON[0].layout = %q, want heading", got)
}
for i, it := range ItemsToMaps(res.JSON) {
if _, ok := it["doc_type_kwd"]; !ok {
diff --git a/internal/parser/parser/parse_with_result_test.go b/internal/parser/parser/parse_with_result_test.go
index 7c51365021..74b88cce62 100644
--- a/internal/parser/parser/parse_with_result_test.go
+++ b/internal/parser/parser/parse_with_result_test.go
@@ -19,7 +19,7 @@
// that did not previously satisfy ParseResultProducer:
//
// - HTMLParser — block-level walker that emits the python-compatible
-// {text, doc_type_kwd, ck_type} shape.
+// {text, doc_type_kwd, layout} shape.
// - TextParser — normalized one-unit output for the text&code family
// (.txt / .py / .js / .java / .c / .cpp / .h / .php / .go / .ts
// / .sh / .cs / .kt / .sql).
@@ -135,7 +135,7 @@ func TestTextParser_ParseWithResult_GBK(t *testing.T) {
// TestHTMLParser_ParseWithResult_BlockSplit pins the HTML walker.
// Three block elements (heading, paragraph, list) yield three
-// items with the python-compatible ck_type vocabulary.
+// items with the structural layout vocabulary.
func TestHTMLParser_ParseWithResult_BlockSplit(t *testing.T) {
ctx := t.Context()
p := NewHTMLParser()
@@ -154,20 +154,20 @@ func TestHTMLParser_ParseWithResult_BlockSplit(t *testing.T) {
if len(res.JSON) != 3 {
t.Fatalf("JSON len = %d, want 3 (h1, p, ul)", len(res.JSON))
}
- if got, want := res.JSON[0].ToMap()["ck_type"], "heading"; got != want {
- t.Errorf("JSON[0].ck_type = %v, want %v", got, want)
+ if got, want := res.JSON[0].ToMap()["layout"], "heading"; got != want {
+ t.Errorf("JSON[0].layout = %v, want %v", got, want)
}
if got, want := res.JSON[0].ToMap()["text"], "Title"; got != want {
t.Errorf("JSON[0].text = %v, want %v", got, want)
}
- if got, want := res.JSON[1].ToMap()["ck_type"], "paragraph"; got != want {
- t.Errorf("JSON[1].ck_type = %v, want %v", got, want)
+ if got, want := res.JSON[1].ToMap()["layout"], "paragraph"; got != want {
+ t.Errorf("JSON[1].layout = %v, want %v", got, want)
}
if got, want := res.JSON[1].ToMap()["text"], "First paragraph."; got != want {
t.Errorf("JSON[1].text = %v, want %v", got, want)
}
- if got, want := res.JSON[2].ToMap()["ck_type"], "list"; got != want {
- t.Errorf("JSON[2].ck_type = %v, want %v", got, want)
+ if got, want := res.JSON[2].ToMap()["layout"], "list"; got != want {
+ t.Errorf("JSON[2].layout = %v, want %v", got, want)
}
if got, want := res.JSON[2].ToMap()["text"], "Item one"; got != want {
t.Errorf("JSON[2].text = %v, want %v", got, want)
@@ -195,7 +195,7 @@ Tail text
}
want := []struct {
text string
- ckType string
+ layout string
}{
{"Intro text", "text"},
{"Title", "heading"},
@@ -213,8 +213,8 @@ Tail text
if got := res.JSON[i].ToMap()["doc_type_kwd"]; got != "text" {
t.Errorf("JSON[%d].doc_type_kwd = %v, want text", i, got)
}
- if got := res.JSON[i].ToMap()["ck_type"]; got != w.ckType {
- t.Errorf("JSON[%d].ck_type = %v, want %v", i, got, w.ckType)
+ if got := res.JSON[i].ToMap()["layout"]; got != w.layout {
+ t.Errorf("JSON[%d].layout = %v, want %v", i, got, w.layout)
}
}
}
diff --git a/internal/parser/parser/pdf_parser_paddleocr.go b/internal/parser/parser/pdf_parser_paddleocr.go
index fd829ecd73..c9d7821527 100644
--- a/internal/parser/parser/pdf_parser_paddleocr.go
+++ b/internal/parser/parser/pdf_parser_paddleocr.go
@@ -11,6 +11,18 @@ import (
"ragflow/internal/utility"
)
+// isStructuralOCRTextLayout reports whether an item already carries a
+// structural block kind produced by markdown normalization. Those values are
+// finer than an OCR block label and must win over it.
+func isStructuralOCRTextLayout(fields map[string]any) bool {
+ layout, _ := fields["layout"].(string)
+ switch layout {
+ case "heading", "paragraph", "list", "code", "quote":
+ return true
+ }
+ return false
+}
+
// PaddleOCRPDFResult projects OCR regions onto the PDF's page geometry and
// emits the structured items consumed by the existing PDF chunk pipeline.
func PaddleOCRPDFResult(ctx context.Context, filename string, data []byte, response *models.OCRFileResponse) ParseResult {
@@ -63,17 +75,22 @@ func PaddleOCRPDFResult(ctx context.Context, filename string, data []byte, respo
label = "caption"
}
for _, item := range parsed.JSON {
+ _, isTextItem := item.(TextItem)
if text, ok := item.(TextItem); ok {
if strings.TrimSpace(text.Text) == "" {
continue
}
- if isTitle && text.CKType == "text" {
+ layout, _ := text.Extra["layout"].(string)
+ if isTitle && layout != "heading" && layout != "list" && layout != "code" {
prefix := "## "
if label == "doc_title" {
prefix = "# "
}
text.Text = prefix + text.Text
- text.CKType = "heading"
+ if text.Extra == nil {
+ text.Extra = map[string]any{}
+ }
+ text.Extra["layout"] = "heading"
item = text
}
}
@@ -81,10 +98,18 @@ func PaddleOCRPDFResult(ctx context.Context, filename string, data []byte, respo
// Page identity is retained even when the block has no usable geometry.
fields["page_number"] = page.Index + 1
fields["_preserve_order"] = true
- if isTitle {
- fields["layout"] = "title"
+ // Markdown normalization already asserted a structural layout
+ // (heading/list/code). The OCR block label is coarser and must
+ // not overwrite it; only fill an absent layout.
+ if isTitle && isTextItem && !isStructuralOCRTextLayout(fields) {
+ // The asserted-title marker the Title chunker reads without
+ // the notTitle heuristic guard. PDF model labels that merely
+ // look title-like stay on "title".
+ fields["layout"] = "heading"
} else if label != "" {
- fields["layout"] = label
+ if existing, _ := fields["layout"].(string); existing == "" {
+ fields["layout"] = label
+ }
}
if len(position) > 0 {
matrix := [][]any{position}
diff --git a/internal/parser/parser/pdf_parser_paddleocr_test.go b/internal/parser/parser/pdf_parser_paddleocr_test.go
index 0530e041bc..caf21ab35c 100644
--- a/internal/parser/parser/pdf_parser_paddleocr_test.go
+++ b/internal/parser/parser/pdf_parser_paddleocr_test.go
@@ -100,12 +100,12 @@ func TestPaddleOCRTitleLabels(t *testing.T) {
if err != nil {
t.Fatal(err)
}
- for _, tc := range []struct{ label, text, want, ckType string }{
+ for _, tc := range []struct{ label, text, want, layout string }{
{"doc_title", "Document title", "# Document title", "heading"},
{"paragraph_title", "Section title", "## Section title", "heading"},
{"doc_title", "### Existing title", "### Existing title", "heading"},
- {"figure_title", "Figure caption", "Figure caption", "text"},
- {"table_title", "Table caption", "Table caption", "text"},
+ {"figure_title", "Figure caption", "Figure caption", "caption"},
+ {"table_title", "Table caption", "Table caption", "caption"},
} {
t.Run(tc.label+tc.text, func(t *testing.T) {
response := &models.OCRFileResponse{Pages: []models.OCRPage{{Index: 0, Blocks: []models.OCRBlock{{Text: tc.text, Label: tc.label}}}}}
@@ -114,7 +114,7 @@ func TestPaddleOCRTitleLabels(t *testing.T) {
t.Fatal(result.Err)
}
item := result.JSON[0].ToMap()
- if item["ck_type"] != tc.ckType || item["text"] != tc.want {
+ if item["layout"] != tc.layout || item["text"] != tc.want {
t.Fatalf("title semantics: %v", item)
}
formatted := pdfItemsToResult("title.pdf", ItemsToMaps(result.JSON), "markdown", 1)
@@ -125,6 +125,42 @@ func TestPaddleOCRTitleLabels(t *testing.T) {
}
}
+func TestPaddleOCRBlocksPreserveMarkdownLayout(t *testing.T) {
+ data, err := os.ReadFile("testdata/paddleocr_pages.pdf")
+ if err != nil {
+ t.Fatal(err)
+ }
+ for _, tc := range []struct{ name, label, text, wantLayout string }{
+ {"text heading", "text", "# Section", "heading"},
+ {"text code", "text", "```\ncode line\n```", "code"},
+ {"text list", "text", "- item one", "list"},
+ {"title heading", "paragraph_title", "# Section", "heading"},
+ {"title code", "paragraph_title", "```\ncode line\n```", "code"},
+ {"title list", "paragraph_title", "- item one", "list"},
+ } {
+ t.Run(tc.name, func(t *testing.T) {
+ // The layout model label is coarser than the Markdown
+ // normalization. It must not overwrite a structural block kind,
+ // including inside a title-labelled block.
+ response := &models.OCRFileResponse{Pages: []models.OCRPage{{Index: 0, Blocks: []models.OCRBlock{{Text: tc.text, Label: tc.label}}}}}
+ result := PaddleOCRPDFResult(t.Context(), "layout.pdf", data, response)
+ if result.Err != nil {
+ t.Fatal(result.Err)
+ }
+ if len(result.JSON) == 0 {
+ t.Fatal("no items emitted")
+ }
+ item := result.JSON[0].ToMap()
+ if item["layout"] != tc.wantLayout {
+ t.Fatalf("layout = %v, want %q (markdown structural layout must win over the OCR label): %v", item["layout"], tc.wantLayout, item)
+ }
+ if text, _ := item["text"].(string); strings.HasPrefix(text, "## ") {
+ t.Errorf("structural item must not be promoted with a heading prefix: %q", text)
+ }
+ })
+ }
+}
+
func TestPaddleOCRPDFResultRejectsRenderedEmptyText(t *testing.T) {
for _, label := range []string{"text", "doc_title", "paragraph_title"} {
t.Run(label, func(t *testing.T) {
diff --git a/internal/parser/parser/pptx_ir.go b/internal/parser/parser/pptx_ir.go
index 35351c9bdf..81a20ae2db 100644
--- a/internal/parser/parser/pptx_ir.go
+++ b/internal/parser/parser/pptx_ir.go
@@ -69,7 +69,6 @@ func buildPPTXJSONSections(irJSON string, budget *embeddedMediaBudget) ([]map[st
"text": "",
"image": nil,
"doc_type_kwd": "image",
- "ck_type": "image",
"slide_number": i + 1,
"media_order": mediaOrder,
}
diff --git a/internal/parser/parser/xls_parser_test.go b/internal/parser/parser/xls_parser_test.go
index 95224ff5b2..0a9a2e0bbd 100644
--- a/internal/parser/parser/xls_parser_test.go
+++ b/internal/parser/parser/xls_parser_test.go
@@ -225,8 +225,8 @@ func TestXLSParser_SpreadsheetJSONOutput(t *testing.T) {
t.Fatalf("res.JSON should have structured table items")
}
item := res.JSON[0].ToMap()
- if item["ck_type"] != "table" {
- t.Errorf("table item ck_type = %v, want table", item["ck_type"])
+ if item["doc_type_kwd"] != "table" {
+ t.Errorf("table item doc_type_kwd = %v, want table", item["doc_type_kwd"])
}
text := tableItemHTML(t, item)
if !strings.Contains(text, "<tr><th>Header1</th><th>Header2</th></tr>") ||
@@ -271,8 +271,8 @@ func TestXLSXParser_JSONOutput(t *testing.T) {
if len(res.JSON) != 1 {
t.Fatalf("res.JSON = %#v, want one segmented HTML table item", res.JSON)
}
- if res.JSON[0].ToMap()["ck_type"] != "table" {
- t.Errorf("item ck_type = %v, want table", res.JSON[0].ToMap()["ck_type"])
+ if res.JSON[0].ToMap()["doc_type_kwd"] != "table" {
+ t.Errorf("item doc_type_kwd = %v, want table", res.JSON[0].ToMap()["doc_type_kwd"])
}
// Even if configured to legacy "html", format is unified to "json"
diff --git a/internal/parser/parser/xls_tcadp_test.go b/internal/parser/parser/xls_tcadp_test.go
index 7768be6cc6..84a6fe4973 100644
--- a/internal/parser/parser/xls_tcadp_test.go
+++ b/internal/parser/parser/xls_tcadp_test.go
@@ -228,7 +228,7 @@ func TestCSVParser_ParseWithResult_DefaultCSVBehavior(t *testing.T) {
if len(res.JSON) != 1 {
t.Fatalf("JSON items = %d, want one HTML table item", len(res.JSON))
}
- if res.JSON[0].ToMap()["ck_type"] != "table" {
+ if res.JSON[0].ToMap()["doc_type_kwd"] != "table" {
t.Fatalf("JSON item = %#v; want a table item", res.JSON[0])
}
}
diff --git a/internal/parser/parser/xlsx_parse_method_test.go b/internal/parser/parser/xlsx_parse_method_test.go
index 43ea5bf7a4..643728a757 100644
--- a/internal/parser/parser/xlsx_parse_method_test.go
+++ b/internal/parser/parser/xlsx_parse_method_test.go
@@ -177,7 +177,7 @@ func TestCSVParser_DeepDocParseMethod(t *testing.T) {
if len(res.JSON) != 1 {
t.Fatalf("JSON items = %d, want one HTML table item", len(res.JSON))
}
- if res.JSON[0].ToMap()["ck_type"] != "table" {
+ if res.JSON[0].ToMap()["doc_type_kwd"] != "table" {
t.Fatalf("JSON item = %#v, want a table item", res.JSON[0])
}
}
diff --git a/internal/service/file/file_content_test.go b/internal/service/file/file_content_test.go
index b50ea119e9..e72bf1d1dc 100644
--- a/internal/service/file/file_content_test.go
+++ b/internal/service/file/file_content_test.go
@@ -83,7 +83,7 @@ func TestParseResultText_SpreadsheetSegmentsAreTheirOwnMarkup(t *testing.T) {
result, err := parseResultText(parser.ParseResult{
OutputFormat: "json",
JSON: parser.MapsToItems([]map[string]any{
- {"text": segment, "doc_type_kwd": "table", "ck_type": "table", "sheet": "Data", "sheet_index": 1},
+ {"text": segment, "doc_type_kwd": "table", "sheet": "Data", "sheet_index": 1},
}),
})
if err != nil {
diff --git a/tools/mdprobe/main.go b/tools/mdprobe/main.go
index c9e12088f3..3703173a78 100644
--- a/tools/mdprobe/main.go
+++ b/tools/mdprobe/main.go
@@ -50,9 +50,9 @@ func main() {
m := item.ToMap()
text, _ := m["text"].(string)
total += len(text)
- ck := fmt.Sprint(m["ck_type"])
+ layout := fmt.Sprint(m["layout"])
docType := fmt.Sprint(m["doc_type_kwd"])
- fmt.Printf(" [%3d] %-7s %-6s %6d chars %q\n", i, ck, docType, len(text), truncate(text, 90))
+ fmt.Printf(" [%3d] %-7s %-6s %6d chars %q\n", i, layout, docType, len(text), truncate(text, 90))
}
fmt.Printf("parser emitted %d chars total (file %d)\n", total, len(data))