Skip to content

RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.

refactor(ingestion): remove all ck_type tombstones (#21317)

jay777217aac10a22b69
diff --git a/internal/ingestion/component/chunker/block_kind_contract_cgo_test.go b/internal/ingestion/component/chunker/block_kind_contract_cgo_test.go
index 66e657d230..c72cdd72ff 100644
--- a/internal/ingestion/component/chunker/block_kind_contract_cgo_test.go
+++ b/internal/ingestion/component/chunker/block_kind_contract_cgo_test.go
@@ -24,20 +24,16 @@ import (
// TestCropTitleChunksNeverEmitsChunkStructure pins that the Title-family crop
// path keeps the output shape identical whether or not a PDF engine is
-// available: retired ck_type and structural layout never leave the component.
+// available: structural layout never leaves the component.
func TestCropTitleChunksNeverEmitsChunkStructure(t *testing.T) {
for _, engine := range []deepdoctype.PDFEngine{nil, mockCropEngine{}} {
in := []map[string]any{{
"text": "body",
"doc_type_kwd": "text",
- "ck_type": "heading",
"layout": "heading",
}}
got := cropTitleChunks(t.Context(), engine, in)
for i, ck := range got {
- if kind, ok := ck["ck_type"]; ok {
- t.Errorf("engine=%T chunk[%d] emits retired ck_type %v: %#v", engine, i, kind, ck)
- }
if layout, ok := ck["layout"]; ok {
t.Errorf("engine=%T chunk[%d] leaks structural layout %v: %#v", engine, i, layout, ck)
}
diff --git a/internal/ingestion/component/chunker/block_kind_contract_test.go b/internal/ingestion/component/chunker/block_kind_contract_test.go
index 86f7283fa6..471b9a8c9b 100644
--- a/internal/ingestion/component/chunker/block_kind_contract_test.go
+++ b/internal/ingestion/component/chunker/block_kind_contract_test.go
@@ -20,10 +20,9 @@ import (
"ragflow/internal/ingestion/component/schema"
)
-// TestChunkerOutputDropsRetiredCKType pins the output contract: ck_type is a
-// retired parser key and no chunker may emit it, even when the input payload
-// still carries a legacy ck_type.
-func TestChunkerOutputDropsRetiredCKType(t *testing.T) {
+// TestChunkerOutputHasNoBlockKindMetadata pins the output contract: chunker
+// output carries no block-kind metadata, structural or otherwise.
+func TestChunkerOutputHasNoBlockKindMetadata(t *testing.T) {
cases := []struct {
name string
run func(t *testing.T) []map[string]any
@@ -38,8 +37,8 @@ func TestChunkerOutputDropsRetiredCKType(t *testing.T) {
out, err := c.Invoke(t.Context(), nil, map[string]any{
"name": "t.md", "file_type": "md", "output_format": "json",
"json": []map[string]any{
- {"text": "# Heading", "doc_type_kwd": "text", "layout": "heading", "ck_type": "heading"},
- {"text": "body", "doc_type_kwd": "text", "ck_type": "text"},
+ {"text": "# Heading", "doc_type_kwd": "text", "layout": "heading"},
+ {"text": "body", "doc_type_kwd": "text"},
},
})
if err != nil {
@@ -58,8 +57,8 @@ func TestChunkerOutputDropsRetiredCKType(t *testing.T) {
out, err := c.Invoke(t.Context(), nil, map[string]any{
"name": "t.md", "output_format": "json",
"json": []map[string]any{
- {"text": "# Heading", "doc_type_kwd": "text", "layout": "heading", "ck_type": "heading"},
- {"text": "body", "doc_type_kwd": "text", "ck_type": "text"},
+ {"text": "# Heading", "doc_type_kwd": "text", "layout": "heading"},
+ {"text": "body", "doc_type_kwd": "text"},
},
})
if err != nil {
@@ -94,8 +93,8 @@ func TestChunkerOutputDropsRetiredCKType(t *testing.T) {
out, err := c.Invoke(t.Context(), nil, map[string]any{
"name": "t.md",
"chunks": []map[string]any{
- {"text": "# Heading", "doc_type_kwd": "text", "layout": "heading", "ck_type": "heading"},
- {"text": "body", "doc_type_kwd": "text", "ck_type": "text"},
+ {"text": "# Heading", "doc_type_kwd": "text", "layout": "heading"},
+ {"text": "body", "doc_type_kwd": "text"},
},
})
if err != nil {
@@ -113,9 +112,6 @@ func TestChunkerOutputDropsRetiredCKType(t *testing.T) {
t.Fatal("no chunks emitted")
}
for i, ck := range chunks {
- if kind, ok := ck["ck_type"]; ok {
- t.Errorf("chunk[%d] emits retired ck_type %v: %#v", i, kind, ck)
- }
if layout, ok := ck["layout"].(string); ok && isStructuralLayout(layout) {
t.Errorf("chunk[%d] leaks structural layout %q: %#v", i, layout, ck)
}
@@ -149,9 +145,9 @@ func TestChunkOutputBoundaryClearsStructuralLayout(t *testing.T) {
}
}
-// TestTitleReadsLayoutNotRetiredCKType pins that a legacy ck_type alone no
-// longer promotes a line to a heading; only layout does.
-func TestTitleReadsLayoutNotRetiredCKType(t *testing.T) {
+// TestTitleDoesNotPromotePlainTextToHeading pins that heading detection
+// requires parser layout metadata.
+func TestTitleDoesNotPromotePlainTextToHeading(t *testing.T) {
c, err := NewGroupTitleChunker(map[string]any{"levels": [][]string{{`^# `}}})
if err != nil {
t.Fatalf("NewGroupTitleChunker: %v", err)
@@ -159,9 +155,9 @@ func TestTitleReadsLayoutNotRetiredCKType(t *testing.T) {
out, err := c.Invoke(t.Context(), nil, map[string]any{
"name": "t.docx",
"chunks": []map[string]any{
- {"text": "Introduction", "doc_type_kwd": "text", "ck_type": "heading"},
+ {"text": "Introduction", "doc_type_kwd": "text"},
{"text": "Body.", "doc_type_kwd": "text"},
- {"text": "Conclusion", "doc_type_kwd": "text", "ck_type": "heading"},
+ {"text": "Conclusion", "doc_type_kwd": "text"},
{"text": "Final.", "doc_type_kwd": "text"},
},
})
@@ -169,7 +165,7 @@ func TestTitleReadsLayoutNotRetiredCKType(t *testing.T) {
t.Fatalf("Invoke: %v", err)
}
if chunks := outputChunks(t, out); len(chunks) != 1 {
- t.Fatalf("legacy ck_type must not create headings: got %d chunks, want 1", len(chunks))
+ t.Fatalf("plain text created headings: got %d chunks, want 1", len(chunks))
}
}
diff --git a/internal/ingestion/component/chunker/crop_title_test.go b/internal/ingestion/component/chunker/crop_title_test.go
index 8f402ea953..6dce870773 100644
--- a/internal/ingestion/component/chunker/crop_title_test.go
+++ b/internal/ingestion/component/chunker/crop_title_test.go
@@ -27,8 +27,7 @@ import (
// Group/Hierarchy output: it carries doc_type_kwd + positions and no
// block-kind field. cropImageChunks derives the crop type from doc_type_kwd.
// A text chunk with positions gets a rendered preview; a chunk without
-// positions is left untouched; and the retired ck_type must not appear in
-// the returned chunk shape.
+// positions is left untouched.
func TestCropTitleChunks_CropsPositionedChunk(t *testing.T) {
ctx := context.Background()
pos := jsonPositions(t, []float64{1, 10, 100, 10, 100})
@@ -48,9 +47,6 @@ func TestCropTitleChunks_CropsPositionedChunk(t *testing.T) {
if got[1]["image"] != nil {
t.Errorf("chunk without positions should not be cropped, got %v", got[1]["image"])
}
- if _, ok := got[0]["ck_type"]; ok {
- t.Errorf("retired ck_type must not appear in output chunk: %v", got[0])
- }
}
// TestCropTitleChunks_CropsImageWithPageList covers the image variant: a
diff --git a/internal/ingestion/component/chunker/custom_delim_test.go b/internal/ingestion/component/chunker/custom_delim_test.go
index 06ba1f33ca..78efdacd4e 100644
--- a/internal/ingestion/component/chunker/custom_delim_test.go
+++ b/internal/ingestion/component/chunker/custom_delim_test.go
@@ -53,9 +53,6 @@ func assertPlainCustomDelimiterMetadata(t *testing.T, chunks []map[string]any) {
if _, ok := chunk["doc_type_kwd"]; ok {
t.Errorf("chunk[%d] must not synthesize doc_type_kwd: %v", i, chunk)
}
- if _, ok := chunk["ck_type"]; ok {
- t.Errorf("chunk[%d] must not emit the retired ck_type: %v", i, chunk)
- }
}
}
diff --git a/internal/ingestion/component/chunker/general.go b/internal/ingestion/component/chunker/general.go
index 146a3c7f00..2eece16f45 100644
--- a/internal/ingestion/component/chunker/general.go
+++ b/internal/ingestion/component/chunker/general.go
@@ -756,8 +756,8 @@ func splitMarkdownUnits(units []schema.ChunkDoc, pattern *regexp.Regexp, keepDel
}
unit.DocType = itemDocType(unit)
// Media units are content attachments, not block-kind boundaries.
- // Base normalized their CKType to DocType here, so an image-only
- // heading folded into the preceding chunk; preserve that behavior.
+ // Base normalized the media type here, so an image-only heading
+ // folded into the preceding chunk; preserve that behavior.
if unit.DocType == "table" || unit.DocType == "image" {
clearStructuralLayout(&unit)
}
diff --git a/internal/ingestion/component/chunker/table_wire_invariant_test.go b/internal/ingestion/component/chunker/table_wire_invariant_test.go
deleted file mode 100644
index c7d08ddc5d..0000000000
--- a/internal/ingestion/component/chunker/table_wire_invariant_test.go
+++ /dev/null
@@ -1,67 +0,0 @@
-package chunker
-
-import (
- "testing"
-
- "ragflow/internal/agent/runtime"
-)
-
-// TestTableWireHasNoRowIRChunkTypes is the cross-chunker guard of the wire
-// contract: the row IR (ck_type table_header/table_row) was deleted, so no
-// chunker may ever emit those values again. Every registered ingestion
-// chunker is enumerated rather than a hand-picked list, so a new chunker that
-// reintroduces the row IR fails here.
-func TestTableWireHasNoRowIRChunkTypes(t *testing.T) {
- inputs := map[string]any{
- "name": "orders.xlsx",
- "file_type": "xlsx",
- "output_format": "json",
- "json": []map[string]any{
- spreadsheetSegmentItem("Sheet1", []string{"ID", "Status"}, [][]string{{"A-1", "paid"}, {"A-2", "open"}}, 1, 2),
- },
- }
- tested := 0
- for _, name := range runtime.DefaultRegistry.NamesByCategory(runtime.CategoryIngestion) {
- factory, _, metadata, ok := runtime.DefaultRegistry.Lookup(name)
- if !ok || factory == nil {
- continue
- }
- if _, isChunker := metadata.Outputs["chunks"]; !isChunker {
- continue
- }
- component, err := factory(name, nil)
- if err != nil {
- // Some chunkers (e.g. GroupTitle) declare required parameters;
- // without them there is no component to exercise.
- t.Logf("skipping %s: %v", name, err)
- continue
- }
- out, err := component.Invoke(t.Context(), nil, inputs)
- if err != nil {
- t.Logf("skipping %s: %v", name, err)
- continue
- }
- if out["_ERROR"] != nil {
- t.Logf("skipping %s: %v", name, out["_ERROR"])
- continue
- }
- chunks, ok := out["chunks"].([]map[string]any)
- if !ok {
- t.Errorf("%s returned chunks of type %T, want []map[string]any", name, out["chunks"])
- continue
- }
- if len(chunks) == 0 {
- t.Logf("skipping %s: no chunks for the fixture", name)
- continue
- }
- for i, chunk := range chunks {
- if kind, ok := chunk["ck_type"]; ok {
- t.Errorf("%s chunk[%d] emits retired ck_type %v: the row IR must not come back", name, i, kind)
- }
- }
- tested++
- }
- if tested < 4 {
- t.Fatalf("exercised %d chunkers, want the registered chunker set", tested)
- }
-}
diff --git a/internal/ingestion/component/chunker/token.go b/internal/ingestion/component/chunker/token.go
index 6351a056f4..fa4688e649 100644
--- a/internal/ingestion/component/chunker/token.go
+++ b/internal/ingestion/component/chunker/token.go
@@ -288,7 +288,7 @@ func stripChunkerRuntimeTimestamps(inputs map[string]any) map[string]any {
// the Title/Group/Hierarchy chunkers, mirroring the TokenChunker JSON path
// (cropImageChunks at token.go:513). A nil engine (or an undecodable payload)
// skips cropping, but the returned maps are still normalized so retired
-// ck_type and chunker-internal structural layout never leave the component.
+// chunker-internal structural layout never leaves the component.
func cropTitleChunks(ctx context.Context, engine deepdoctype.PDFEngine, chunks []map[string]any) []map[string]any {
if len(chunks) == 0 {
return chunks
@@ -310,11 +310,10 @@ func cropTitleChunks(ctx context.Context, engine deepdoctype.PDFEngine, chunks [
}
// normalizeTitleChunkMaps keeps the Title-family output shape independent of
-// whether a PDF engine is available: retired ck_type and chunker-internal
-// structural layout never leave the component.
+// whether a PDF engine is available: chunker-internal structural layout
+// never leaves the component.
func normalizeTitleChunkMaps(chunks []map[string]any) {
for _, chunk := range chunks {
- delete(chunk, "ck_type")
if layout, _ := chunk["layout"].(string); isStructuralLayout(layout) {
delete(chunk, "layout")
}
diff --git a/internal/ingestion/component/chunker/token_test.go b/internal/ingestion/component/chunker/token_test.go
index a370a29de3..d69f1ec25d 100644
--- a/internal/ingestion/component/chunker/token_test.go
+++ b/internal/ingestion/component/chunker/token_test.go
@@ -405,9 +405,6 @@ func TestTokenChunker_InvokeJSONPayload_KeepsNonTextStandalone(t *testing.T) {
if got != wantTypes[i] {
t.Errorf("chunk %d: doc_type_kwd = %q, want %q (full chunk: %+v)", i, got, wantTypes[i], ch)
}
- if _, ok := ch["ck_type"]; ok {
- t.Errorf("chunk %d must not emit the retired ck_type: %+v", i, ch)
- }
}
// The two text segments on either side of the table/image must remain
// distinct — they must NOT be merged across the non-text segments.
diff --git a/internal/ingestion/component/schema/chunk_docitem_test.go b/internal/ingestion/component/schema/chunk_docitem_test.go
index a678bc720b..7c97f44e62 100644
--- a/internal/ingestion/component/schema/chunk_docitem_test.go
+++ b/internal/ingestion/component/schema/chunk_docitem_test.go
@@ -179,19 +179,19 @@ func TestDocItemsFromMaps_ParallelsMapPath(t *testing.T) {
{
name: "text with extra passthrough",
maps: []map[string]any{
- {"text": "hello", parser.DocTypeKey: parser.DocTypeText, "ck_type": "heading", "layout": "1", "page_number": 3},
+ {"text": "hello", parser.DocTypeKey: parser.DocTypeText, "layout": "1", "page_number": 3},
},
},
{
name: "image with extra passthrough",
maps: []map[string]any{
- {"text": "alt", "image": "data:image/png;base64,xxx", parser.DocTypeKey: parser.DocTypeImage, "ck_type": "image", "page_number": 2},
+ {"text": "alt", "image": "data:image/png;base64,xxx", parser.DocTypeKey: parser.DocTypeImage, "page_number": 2},
},
},
{
name: "table with sheet and positions",
maps: []map[string]any{
- {"table": &entity.TableData{}, parser.DocTypeKey: parser.DocTypeTable, "ck_type": parser.DocTypeTable, "sheet": "s", "positions": [][]float64{{1, 2}}, "sheet_index": 0},
+ {"table": &entity.TableData{}, parser.DocTypeKey: parser.DocTypeTable, "sheet": "s", "positions": [][]float64{{1, 2}}, "sheet_index": 0},
},
},
{
@@ -203,7 +203,8 @@ func TestDocItemsFromMaps_ParallelsMapPath(t *testing.T) {
{
name: "mixed typed and opaque",
maps: []map[string]any{
- {"text": "a", parser.DocTypeKey: parser.DocTypeText, "ck_type": "text"},
+ {"text": "a", parser.DocTypeKey: parser.DocTypeText},
+
{"foo": "bar"},
},
},
diff --git a/internal/ingestion/component/schema/chunk_types.go b/internal/ingestion/component/schema/chunk_types.go
index cf2de2738b..89a2c988c3 100644
--- a/internal/ingestion/component/schema/chunk_types.go
+++ b/internal/ingestion/component/schema/chunk_types.go
@@ -155,11 +155,9 @@ func (d *ChunkDoc) UnmarshalJSON(data []byte) error {
// tagger emits tag_feas and the vocabulary comes from a tag source file,
// so no chunk carries tag_kwd. Deleted rather than left in raw so it
// cannot fall through to Extra and be re-emitted by MarshalJSON.
- // ck_type is a retired parser key: it has no field, but must stay in this
- // list so an old payload cannot leak it back through Extra.
for _, key := range []string{
"text", "doc_type_kwd", "mom", "img_id",
- "ck_type", "tk_nums", "layout", "layout_type", "layoutno", "image",
+ "tk_nums", "layout", "layout_type", "layoutno", "image",
"context_above", "context_below", "questions", "keywords", "summary",
"chunk_order_int", "title_tks", "title_sm_tks", "content_ltks",
"content_sm_ltks", "tag_kwd", "page_number", "top_int", "_pdf_positions", "positions",
@@ -185,10 +183,6 @@ func (d ChunkDoc) MarshalJSON() ([]byte, error) {
return nil, err
}
for k, v := range d.Extra {
- // ck_type is retired; a stray Extra key must not resurrect it.
- if k == "ck_type" {
- continue
- }
if _, exists := raw[k]; !exists {
raw[k] = v
}
@@ -299,9 +293,6 @@ func (d ChunkDoc) ToMap() map[string]any {
return map[string]any{}
}
for k, raw := range d.Extra {
- if k == "ck_type" {
- continue
- }
out[k] = decodeExtraValue(raw)
}
// Every pre-index chunk has one canonical text field. Keep the key even
diff --git a/internal/ingestion/component/schema/schema_test.go b/internal/ingestion/component/schema/schema_test.go
index 862f438d81..97d336e13b 100644
--- a/internal/ingestion/component/schema/schema_test.go
+++ b/internal/ingestion/component/schema/schema_test.go
@@ -267,7 +267,7 @@ func TestChunkDocSpreadsheetFieldsRoundTrip(t *testing.T) {
// round trip through Extra; the index boundary strips them from the store.
func TestChunkDocLegacyRowIRKeysPassThrough(t *testing.T) {
var decoded ChunkDoc
- raw := `{"text":"row","doc_type_kwd":"text","ck_type":"table_row","table_id":"sheet-2","headers":["ID","Status"],"cells":["A-100","paid"],"row_start":42,"row_end":42,"col_start":1,"col_end":3}`
+ raw := `{"text":"row","doc_type_kwd":"text","table_id":"sheet-2","headers":["ID","Status"],"cells":["A-100","paid"],"row_start":42,"row_end":42,"col_start":1,"col_end":3}`
if err := json.Unmarshal([]byte(raw), &decoded); err != nil {
t.Fatalf("unmarshal: %v", err)
}
@@ -277,32 +277,6 @@ func TestChunkDocLegacyRowIRKeysPassThrough(t *testing.T) {
t.Errorf("legacy key %q lost in round trip: %#v", key, out)
}
}
- if _, ok := out["ck_type"]; ok {
- t.Errorf("retired ck_type must not survive the round trip: %#v", out)
- }
-}
-
-// A programmatic Extra entry must not resurrect the retired ck_type key on
-// either projection (MarshalJSON or ToMap).
-func TestChunkDocRetiredCKTypeCannotBeResurrectedViaExtra(t *testing.T) {
- d := ChunkDoc{
- Text: "body",
- Extra: map[string]json.RawMessage{"ck_type": json.RawMessage(`"heading"`)},
- }
- if _, ok := d.ToMap()["ck_type"]; ok {
- t.Fatalf("ToMap resurrected ck_type: %#v", d.ToMap())
- }
- data, err := json.Marshal(d)
- if err != nil {
- t.Fatalf("marshal: %v", err)
- }
- var raw map[string]any
- if err := json.Unmarshal(data, &raw); err != nil {
- t.Fatalf("unmarshal: %v", err)
- }
- if _, ok := raw["ck_type"]; ok {
- t.Fatalf("MarshalJSON resurrected ck_type: %s", data)
- }
}
func TestChunkDocVisualParentMetadataRoundTrip(t *testing.T) {
diff --git a/internal/ingestion/task/indexdoc/process.go b/internal/ingestion/task/indexdoc/process.go
index e18bab93b2..b6eaafd08e 100644
--- a/internal/ingestion/task/indexdoc/process.go
+++ b/internal/ingestion/task/indexdoc/process.go
@@ -154,11 +154,8 @@ func cleanupConsumedChunkFields(ck map[string]any) {
// persist fields — but Go's chunker hands them on, and the write boundary is
// where they stop: Elasticsearch and Vastbase would add a field for each key,
// and Infinity would drop each key at insert with a warning.
-//
-// ck_type is retired but stays here as a tombstone: a legacy producer can still
-// hand it on, and no engine has a column for it.
var pipelineOnlyFields = []string{
- "ck_type", "tk_nums", "layout", "layout_type", "layoutno", "image",
+ "tk_nums", "layout", "layout_type", "layoutno", "image",
"context_above", "context_below", "page_number", "_preserve_order",
"table_id", "sheet", "sheet_index", "headers", "cells",
"row_start", "row_end", "col_start", "col_end",
diff --git a/internal/ingestion/task/indexdoc/process_test.go b/internal/ingestion/task/indexdoc/process_test.go
index a909ad7b19..adb07e1c27 100644
--- a/internal/ingestion/task/indexdoc/process_test.go
+++ b/internal/ingestion/task/indexdoc/process_test.go
@@ -668,14 +668,14 @@ func TestCleanupConsumedChunkFields_ImportantKwdDropsEmptyParts(t *testing.T) {
// TestProcessChunksForPipeline_StripsPipelineOnlyFields pins the index boundary
// against the parser/chunker bookkeeping keys: none of them is a chunk-store
-// column, and a strict engine rejects the whole insert over one of them
-// ("Column ck_type not found in table", InfinityException 3013). ES only
-// swallowed them because its mapping is dynamic.
+// column, and a strict engine rejects the whole insert over one of them (e.g.
+// Infinity would reject an unknown column). ES only swallowed them because its
+// mapping is dynamic.
func TestProcessChunksForPipeline_StripsPipelineOnlyFields(t *testing.T) {
ck := map[string]any{
"text": "hello",
// Every bookkeeping key the chunkers/parsers can leave on a chunk.
- "ck_type": "text", "tk_nums": 3, "layout": "text", "layout_type": "text",
+ "tk_nums": 3, "layout": "text", "layout_type": "text",
"layoutno": "0", "image": "data:image/png;base64,AAAA",
"context_above": "above", "context_below": "below", "page_number": 2,
"table_id": "t1", "sheet": "s1", "sheet_index": 0,
diff --git a/internal/parser/parser/doc_item.go b/internal/parser/parser/doc_item.go
index da131a19cc..29fa18dc18 100644
--- a/internal/parser/parser/doc_item.go
+++ b/internal/parser/parser/doc_item.go
@@ -175,13 +175,13 @@ func DocItemFromMap(m map[string]any) DocItem {
func textItemFromMap(m map[string]any) DocItem {
return TextItem{
Text: stringField(m, "text"),
- Extra: extraFromMap(m, "text", DocTypeKey, "ck_type"),
+ Extra: extraFromMap(m, "text", DocTypeKey),
}
}
func imageItemFromMap(m map[string]any) DocItem {
pos := wireToPositions(m["positions"], hasSheetIndex(m))
- skip := []string{"text", "image", DocTypeKey, "ck_type", "sheet"}
+ skip := []string{"text", "image", DocTypeKey, "sheet"}
if pos != nil {
// Parsed into the typed field: it owns "positions", so strip it from
// Extra. Both [][]float64 (office/xlsx) and [][]any (PDF crop regions
@@ -209,7 +209,7 @@ func tableItemFromMap(m map[string]any) DocItem {
return RawItem{M: m}
}
pos := wireToPositions(m["positions"], hasSheetIndex(m))
- skip := []string{"table", DocTypeKey, "ck_type", "sheet"}
+ skip := []string{"table", DocTypeKey, "sheet"}
if pos != nil {
skip = append(skip, "positions")
}
@@ -270,10 +270,6 @@ func ItemsToMaps(items []DocItem) []map[string]any {
// present, so a stray Extra key can never shadow text/table/image/doc_type_kwd.
func mergeExtra(m, extra map[string]any) map[string]any {
for k, v := range extra {
- // ck_type is retired; a stray Extra key must not resurrect it.
- if k == "ck_type" {
- continue
- }
if _, ok := m[k]; !ok {
m[k] = v
}
diff --git a/internal/parser/parser/doc_item_test.go b/internal/parser/parser/doc_item_test.go
index 3ab4d180b2..4e7d08482e 100644
--- a/internal/parser/parser/doc_item_test.go
+++ b/internal/parser/parser/doc_item_test.go
@@ -52,8 +52,7 @@ func TestTextItem_ToMap(t *testing.T) {
}
func TestTextItem_ExtraDoesNotClobberCanonical(t *testing.T) {
- // An attacker/stray Extra key must never shadow the typed fields, and a
- // retired ck_type must not be resurrected through Extra.
+ // An attacker/stray Extra key must never shadow the typed fields.
got := TextItem{
Text: "real",
Extra: map[string]any{
@@ -292,24 +291,6 @@ func TestDocItemFromMap_WireShapePreserved(t *testing.T) {
}
}
-// A legacy payload may still carry the retired ck_type key. Typed conversion
-// must drop it rather than route it through Extra and re-emit it.
-func TestDocItemFromMap_DropsRetiredCKType(t *testing.T) {
- legacy := map[string]any{
- "text": "h",
- DocTypeKey: DocTypeText,
- "ck_type": "heading",
- "layout": "heading",
- }
- got := DocItemFromMap(legacy).ToMap()
- if _, ok := got["ck_type"]; ok {
- t.Fatalf("retired ck_type leaked through ToMap: %#v", got)
- }
- if got["layout"] != "heading" {
- t.Fatalf("layout lost during conversion: %#v", got)
- }
-}
-
// DocItemsFromMaps maps a []map[string]any to typed DocItems the same way
// DocItemFromMap does, preserving order and skipping nothing.
func TestMapsToItems_OrderAndType(t *testing.T) {
diff --git a/internal/parser/parser/html_parser_align_test.go b/internal/parser/parser/html_parser_align_test.go
index dcf5f6610e..51a1554a54 100644
--- a/internal/parser/parser/html_parser_align_test.go
+++ b/internal/parser/parser/html_parser_align_test.go
@@ -61,9 +61,6 @@ func TestHTMLParser_TableProducesStructuredItems(t *testing.T) {
if tableCount != 1 {
t.Fatalf("structured doc_type_kwd:\"table\" item count = %d, want exactly 1", tableCount)
}
- if _, ok := res.JSON[tableIdx].ToMap()["ck_type"]; ok {
- t.Errorf("retired ck_type present on structured table item: %#v", res.JSON[tableIdx].ToMap())
- }
if !strings.Contains(tableText, "<table") ||
!strings.Contains(tableText, "Name") ||
!strings.Contains(tableText, "Alice") {
@@ -144,9 +141,6 @@ func TestHTMLParser_NestedTableProducesStructuredItems(t *testing.T) {
if tableCount != 1 {
t.Fatalf("structured doc_type_kwd:\"table\" item count = %d, want exactly 1", tableCount)
}
- if _, ok := res.JSON[tableIdx].ToMap()["ck_type"]; ok {
- t.Errorf("retired ck_type present on structured nested table item: %#v", res.JSON[tableIdx].ToMap())
- }
if !strings.Contains(tableText, "<table") ||
!strings.Contains(tableText, "Name") ||
!strings.Contains(tableText, "Alice") {
diff --git a/internal/parser/parser/markdown_parser_test.go b/internal/parser/parser/markdown_parser_test.go
index 1de9e9e125..3218b3e246 100644
--- a/internal/parser/parser/markdown_parser_test.go
+++ b/internal/parser/parser/markdown_parser_test.go
@@ -273,9 +273,6 @@ func TestMarkdownParser_ParseWithResult_RendersTableInline(t *testing.T) {
switch kd {
case "table":
tableCount++
- if _, ok := item["ck_type"]; ok {
- t.Errorf("retired ck_type present on table item: %#v", item)
- }
td, ok := item["table"].(*entity.TableData)
if !ok || td == nil {
t.Fatalf("table item missing *entity.TableData: %#v", item)
@@ -648,9 +645,6 @@ func TestMarkdownParser_TableNotCollapsed(t *testing.T) {
if _, ok := item["table"].(*entity.TableData); !ok {
t.Fatalf("table item missing *entity.TableData: %#v", item)
}
- if _, ok := item["ck_type"]; ok {
- t.Fatalf("retired ck_type present on table item: %#v", item)
- }
}
}
concat := all.String()
@@ -713,9 +707,6 @@ func TestMarkdownParser_TableWithSurroundingText(t *testing.T) {
if _, ok := item["table"].(*entity.TableData); !ok {
t.Fatalf("table item missing *entity.TableData: %#v", item)
}
- if _, ok := item["ck_type"]; ok {
- t.Fatalf("retired ck_type present on table item: %#v", item)
- }
}
}
@@ -760,9 +751,6 @@ func TestMarkdownParser_NonTableHTMLBlockNotTable(t *testing.T) {
for _, item := range ItemsToMaps(res.JSON) {
text, _ := item["text"].(string)
if strings.Contains(text, "<div") || strings.Contains(text, "<style") {
- if _, ok := item["ck_type"]; ok {
- t.Fatalf("retired ck_type present on non-table item: %q", text)
- }
if kd, _ := item["doc_type_kwd"].(string); kd == "table" {
t.Fatalf("non-table HTML block wrongly tagged doc_type_kwd:\"table\": %q", text)
}
@@ -833,9 +821,6 @@ func TestMarkdownParser_RawHTMLTableHandled(t *testing.T) {
if _, ok := item["table"].(*entity.TableData); !ok {
t.Fatalf("raw table item missing *entity.TableData: %#v", item)
}
- if _, ok := item["ck_type"]; ok {
- t.Fatalf("retired ck_type present on raw table item: %#v", item)
- }
}
}
if sawInlined {
diff --git a/internal/parser/parser/pdf_result.go b/internal/parser/parser/pdf_result.go
index 4cc77653cf..6933b073e7 100644
--- a/internal/parser/parser/pdf_result.go
+++ b/internal/parser/parser/pdf_result.go
@@ -128,10 +128,6 @@ func (p *PDFParser) NormalizeResult(ctx context.Context, filename string, result
}
fields["doc_type_kwd"] = "text"
}
- // ck_type is retired: the content type lives on doc_type_kwd and the
- // block kind on layout. A legacy raw item can still carry the key, so
- // drop it here as well — the typed conversion already does.
- delete(fields, "ck_type")
items = appendPDFItem(items, fields)
}
if droppedPositions > 0 {
diff --git a/internal/parser/parser/pdf_result_test.go b/internal/parser/parser/pdf_result_test.go
index d2b00d4812..903517c0fc 100644
--- a/internal/parser/parser/pdf_result_test.go
+++ b/internal/parser/parser/pdf_result_test.go
@@ -142,9 +142,6 @@ func TestPDFNormalizeResultFlattenPreservesContentAndClearsMediaSubtype(t *testi
if fields["doc_type_kwd"] != "text" {
t.Fatalf("flattened item %d still has media type: %#v", i, fields)
}
- if _, ok := fields["ck_type"]; ok {
- t.Fatalf("flattened item %d still carries the retired ck_type: %#v", i, fields)
- }
}
text, _ := result.JSON[0].ToMap()["text"].(string)
for _, want := range []string{"Cell", "Caption", "Footnote"} {
@@ -165,7 +162,7 @@ func TestPDFNormalizeResultFlattenPreservesContentAndClearsMediaSubtype(t *testi
func TestPDFNormalizeResultDowngradesUnstructuredTableSubtype(t *testing.T) {
result := NewPDFParser().NormalizeResult(t.Context(), "sample.pdf", ParseResult{OutputFormat: "json", JSON: MapsToItems([]map[string]any{{
- "doc_type_kwd": "table", "ck_type": "table", "layout": "table", "text": "Recognized text without cells",
+ "doc_type_kwd": "table", "layout": "table", "text": "Recognized text without cells",
}})})
if result.Err != nil {
t.Fatal(result.Err)
@@ -174,9 +171,6 @@ func TestPDFNormalizeResultDowngradesUnstructuredTableSubtype(t *testing.T) {
if item["doc_type_kwd"] != "text" || item["text"] != "Recognized text without cells" {
t.Fatalf("unstructured table kept an inconsistent subtype: %#v", item)
}
- if _, ok := item["ck_type"]; ok {
- t.Fatalf("retired ck_type survived normalization: %#v", item)
- }
}
func TestPDFNormalizeResultKeepsPositionedUnstructuredTables(t *testing.T) {
@@ -184,7 +178,7 @@ func TestPDFNormalizeResultKeepsPositionedUnstructuredTables(t *testing.T) {
p := NewPDFParser()
p.FlattenMediaToText = flatten
result := p.NormalizeResult(t.Context(), "sample.pdf", ParseResult{OutputFormat: "json", JSON: MapsToItems([]map[string]any{{
- "doc_type_kwd": "table", "ck_type": "table", "text": "Recognized table without cells",
+ "doc_type_kwd": "table", "text": "Recognized table without cells",
"positions": [][]any{{2, 10, 30, 20, 40}},
}})})
if result.Err != nil || len(result.JSON) != 1 {