RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.
Go agent-harnessagent-native-databaseagentic-aiagentic-retrievalagentic-searchaiai-agentscontext-enginecontext-engineeringcontext-managementharness-engineeringknowledge-compilationragretrieval-augmented-generationretrieval-harnesssearch-harness
refactor(ingestion): remove all ck_type tombstones (#21317)
jay777217aac10a22b69
diff --git a/internal/ingestion/component/chunker/block_kind_contract_cgo_test.go b/internal/ingestion/component/chunker/block_kind_contract_cgo_test.goindex 66e657d230..c72cdd72ff 100644--- a/internal/ingestion/component/chunker/block_kind_contract_cgo_test.go+++ b/internal/ingestion/component/chunker/block_kind_contract_cgo_test.go@@ -24,20 +24,16 @@ import (// TestCropTitleChunksNeverEmitsChunkStructure pins that the Title-family crop// path keeps the output shape identical whether or not a PDF engine is-// available: retired ck_type and structural layout never leave the component.+// available: structural layout never leaves the component.func TestCropTitleChunksNeverEmitsChunkStructure(t *testing.T) {for _, engine := range []deepdoctype.PDFEngine{nil, mockCropEngine{}} {in := []map[string]any{{"text": "body","doc_type_kwd": "text",- "ck_type": "heading","layout": "heading",}}got := cropTitleChunks(t.Context(), engine, in)for i, ck := range got {- if kind, ok := ck["ck_type"]; ok {- t.Errorf("engine=%T chunk[%d] emits retired ck_type %v: %#v", engine, i, kind, ck)- }if layout, ok := ck["layout"]; ok {t.Errorf("engine=%T chunk[%d] leaks structural layout %v: %#v", engine, i, layout, ck)}diff --git a/internal/ingestion/component/chunker/block_kind_contract_test.go b/internal/ingestion/component/chunker/block_kind_contract_test.goindex 86f7283fa6..471b9a8c9b 100644--- a/internal/ingestion/component/chunker/block_kind_contract_test.go+++ b/internal/ingestion/component/chunker/block_kind_contract_test.go@@ -20,10 +20,9 @@ import ("ragflow/internal/ingestion/component/schema")-// TestChunkerOutputDropsRetiredCKType pins the output contract: ck_type is a-// retired parser key and no chunker may emit it, even when the input payload-// still carries a legacy ck_type.-func TestChunkerOutputDropsRetiredCKType(t *testing.T) {+// TestChunkerOutputHasNoBlockKindMetadata pins the output contract: chunker+// output carries no block-kind metadata, structural or otherwise.+func TestChunkerOutputHasNoBlockKindMetadata(t *testing.T) {cases := []struct {name stringrun func(t *testing.T) []map[string]any@@ -38,8 +37,8 @@ func TestChunkerOutputDropsRetiredCKType(t *testing.T) {out, err := c.Invoke(t.Context(), nil, map[string]any{"name": "t.md", "file_type": "md", "output_format": "json","json": []map[string]any{- {"text": "# Heading", "doc_type_kwd": "text", "layout": "heading", "ck_type": "heading"},- {"text": "body", "doc_type_kwd": "text", "ck_type": "text"},+ {"text": "# Heading", "doc_type_kwd": "text", "layout": "heading"},+ {"text": "body", "doc_type_kwd": "text"},},})if err != nil {@@ -58,8 +57,8 @@ func TestChunkerOutputDropsRetiredCKType(t *testing.T) {out, err := c.Invoke(t.Context(), nil, map[string]any{"name": "t.md", "output_format": "json","json": []map[string]any{- {"text": "# Heading", "doc_type_kwd": "text", "layout": "heading", "ck_type": "heading"},- {"text": "body", "doc_type_kwd": "text", "ck_type": "text"},+ {"text": "# Heading", "doc_type_kwd": "text", "layout": "heading"},+ {"text": "body", "doc_type_kwd": "text"},},})if err != nil {@@ -94,8 +93,8 @@ func TestChunkerOutputDropsRetiredCKType(t *testing.T) {out, err := c.Invoke(t.Context(), nil, map[string]any{"name": "t.md","chunks": []map[string]any{- {"text": "# Heading", "doc_type_kwd": "text", "layout": "heading", "ck_type": "heading"},- {"text": "body", "doc_type_kwd": "text", "ck_type": "text"},+ {"text": "# Heading", "doc_type_kwd": "text", "layout": "heading"},+ {"text": "body", "doc_type_kwd": "text"},},})if err != nil {@@ -113,9 +112,6 @@ func TestChunkerOutputDropsRetiredCKType(t *testing.T) {t.Fatal("no chunks emitted")}for i, ck := range chunks {- if kind, ok := ck["ck_type"]; ok {- t.Errorf("chunk[%d] emits retired ck_type %v: %#v", i, kind, ck)- }if layout, ok := ck["layout"].(string); ok && isStructuralLayout(layout) {t.Errorf("chunk[%d] leaks structural layout %q: %#v", i, layout, ck)}@@ -149,9 +145,9 @@ func TestChunkOutputBoundaryClearsStructuralLayout(t *testing.T) {}}-// TestTitleReadsLayoutNotRetiredCKType pins that a legacy ck_type alone no-// longer promotes a line to a heading; only layout does.-func TestTitleReadsLayoutNotRetiredCKType(t *testing.T) {+// TestTitleDoesNotPromotePlainTextToHeading pins that heading detection+// requires parser layout metadata.+func TestTitleDoesNotPromotePlainTextToHeading(t *testing.T) {c, err := NewGroupTitleChunker(map[string]any{"levels": [][]string{{`^# `}}})if err != nil {t.Fatalf("NewGroupTitleChunker: %v", err)@@ -159,9 +155,9 @@ func TestTitleReadsLayoutNotRetiredCKType(t *testing.T) {out, err := c.Invoke(t.Context(), nil, map[string]any{"name": "t.docx","chunks": []map[string]any{- {"text": "Introduction", "doc_type_kwd": "text", "ck_type": "heading"},+ {"text": "Introduction", "doc_type_kwd": "text"},{"text": "Body.", "doc_type_kwd": "text"},- {"text": "Conclusion", "doc_type_kwd": "text", "ck_type": "heading"},+ {"text": "Conclusion", "doc_type_kwd": "text"},{"text": "Final.", "doc_type_kwd": "text"},},})@@ -169,7 +165,7 @@ func TestTitleReadsLayoutNotRetiredCKType(t *testing.T) {t.Fatalf("Invoke: %v", err)}if chunks := outputChunks(t, out); len(chunks) != 1 {- t.Fatalf("legacy ck_type must not create headings: got %d chunks, want 1", len(chunks))+ t.Fatalf("plain text created headings: got %d chunks, want 1", len(chunks))}}diff --git a/internal/ingestion/component/chunker/crop_title_test.go b/internal/ingestion/component/chunker/crop_title_test.goindex 8f402ea953..6dce870773 100644--- a/internal/ingestion/component/chunker/crop_title_test.go+++ b/internal/ingestion/component/chunker/crop_title_test.go@@ -27,8 +27,7 @@ import (// Group/Hierarchy output: it carries doc_type_kwd + positions and no// block-kind field. cropImageChunks derives the crop type from doc_type_kwd.// A text chunk with positions gets a rendered preview; a chunk without-// positions is left untouched; and the retired ck_type must not appear in-// the returned chunk shape.+// positions is left untouched.func TestCropTitleChunks_CropsPositionedChunk(t *testing.T) {ctx := context.Background()pos := jsonPositions(t, []float64{1, 10, 100, 10, 100})@@ -48,9 +47,6 @@ func TestCropTitleChunks_CropsPositionedChunk(t *testing.T) {if got[1]["image"] != nil {t.Errorf("chunk without positions should not be cropped, got %v", got[1]["image"])}- if _, ok := got[0]["ck_type"]; ok {- t.Errorf("retired ck_type must not appear in output chunk: %v", got[0])- }}// TestCropTitleChunks_CropsImageWithPageList covers the image variant: adiff --git a/internal/ingestion/component/chunker/custom_delim_test.go b/internal/ingestion/component/chunker/custom_delim_test.goindex 06ba1f33ca..78efdacd4e 100644--- a/internal/ingestion/component/chunker/custom_delim_test.go+++ b/internal/ingestion/component/chunker/custom_delim_test.go@@ -53,9 +53,6 @@ func assertPlainCustomDelimiterMetadata(t *testing.T, chunks []map[string]any) {if _, ok := chunk["doc_type_kwd"]; ok {t.Errorf("chunk[%d] must not synthesize doc_type_kwd: %v", i, chunk)}- if _, ok := chunk["ck_type"]; ok {- t.Errorf("chunk[%d] must not emit the retired ck_type: %v", i, chunk)- }}}diff --git a/internal/ingestion/component/chunker/general.go b/internal/ingestion/component/chunker/general.goindex 146a3c7f00..2eece16f45 100644--- a/internal/ingestion/component/chunker/general.go+++ b/internal/ingestion/component/chunker/general.go@@ -756,8 +756,8 @@ func splitMarkdownUnits(units []schema.ChunkDoc, pattern *regexp.Regexp, keepDel}unit.DocType = itemDocType(unit)// Media units are content attachments, not block-kind boundaries.- // Base normalized their CKType to DocType here, so an image-only- // heading folded into the preceding chunk; preserve that behavior.+ // Base normalized the media type here, so an image-only heading+ // folded into the preceding chunk; preserve that behavior.if unit.DocType == "table" || unit.DocType == "image" {clearStructuralLayout(&unit)}diff --git a/internal/ingestion/component/chunker/table_wire_invariant_test.go b/internal/ingestion/component/chunker/table_wire_invariant_test.godeleted file mode 100644index c7d08ddc5d..0000000000--- a/internal/ingestion/component/chunker/table_wire_invariant_test.go+++ /dev/null@@ -1,67 +0,0 @@-package chunker--import (- "testing"-- "ragflow/internal/agent/runtime"-)--// TestTableWireHasNoRowIRChunkTypes is the cross-chunker guard of the wire-// contract: the row IR (ck_type table_header/table_row) was deleted, so no-// chunker may ever emit those values again. Every registered ingestion-// chunker is enumerated rather than a hand-picked list, so a new chunker that-// reintroduces the row IR fails here.-func TestTableWireHasNoRowIRChunkTypes(t *testing.T) {- inputs := map[string]any{- "name": "orders.xlsx",- "file_type": "xlsx",- "output_format": "json",- "json": []map[string]any{- spreadsheetSegmentItem("Sheet1", []string{"ID", "Status"}, [][]string{{"A-1", "paid"}, {"A-2", "open"}}, 1, 2),- },- }- tested := 0- for _, name := range runtime.DefaultRegistry.NamesByCategory(runtime.CategoryIngestion) {- factory, _, metadata, ok := runtime.DefaultRegistry.Lookup(name)- if !ok || factory == nil {- continue- }- if _, isChunker := metadata.Outputs["chunks"]; !isChunker {- continue- }- component, err := factory(name, nil)- if err != nil {- // Some chunkers (e.g. GroupTitle) declare required parameters;- // without them there is no component to exercise.- t.Logf("skipping %s: %v", name, err)- continue- }- out, err := component.Invoke(t.Context(), nil, inputs)- if err != nil {- t.Logf("skipping %s: %v", name, err)- continue- }- if out["_ERROR"] != nil {- t.Logf("skipping %s: %v", name, out["_ERROR"])- continue- }- chunks, ok := out["chunks"].([]map[string]any)- if !ok {- t.Errorf("%s returned chunks of type %T, want []map[string]any", name, out["chunks"])- continue- }- if len(chunks) == 0 {- t.Logf("skipping %s: no chunks for the fixture", name)- continue- }- for i, chunk := range chunks {- if kind, ok := chunk["ck_type"]; ok {- t.Errorf("%s chunk[%d] emits retired ck_type %v: the row IR must not come back", name, i, kind)- }- }- tested++- }- if tested < 4 {- t.Fatalf("exercised %d chunkers, want the registered chunker set", tested)- }-}diff --git a/internal/ingestion/component/chunker/token.go b/internal/ingestion/component/chunker/token.goindex 6351a056f4..fa4688e649 100644--- a/internal/ingestion/component/chunker/token.go+++ b/internal/ingestion/component/chunker/token.go@@ -288,7 +288,7 @@ func stripChunkerRuntimeTimestamps(inputs map[string]any) map[string]any {// the Title/Group/Hierarchy chunkers, mirroring the TokenChunker JSON path// (cropImageChunks at token.go:513). A nil engine (or an undecodable payload)// skips cropping, but the returned maps are still normalized so retired-// ck_type and chunker-internal structural layout never leave the component.+// chunker-internal structural layout never leaves the component.func cropTitleChunks(ctx context.Context, engine deepdoctype.PDFEngine, chunks []map[string]any) []map[string]any {if len(chunks) == 0 {return chunks@@ -310,11 +310,10 @@ func cropTitleChunks(ctx context.Context, engine deepdoctype.PDFEngine, chunks [}// normalizeTitleChunkMaps keeps the Title-family output shape independent of-// whether a PDF engine is available: retired ck_type and chunker-internal-// structural layout never leave the component.+// whether a PDF engine is available: chunker-internal structural layout+// never leaves the component.func normalizeTitleChunkMaps(chunks []map[string]any) {for _, chunk := range chunks {- delete(chunk, "ck_type")if layout, _ := chunk["layout"].(string); isStructuralLayout(layout) {delete(chunk, "layout")}diff --git a/internal/ingestion/component/chunker/token_test.go b/internal/ingestion/component/chunker/token_test.goindex a370a29de3..d69f1ec25d 100644--- a/internal/ingestion/component/chunker/token_test.go+++ b/internal/ingestion/component/chunker/token_test.go@@ -405,9 +405,6 @@ func TestTokenChunker_InvokeJSONPayload_KeepsNonTextStandalone(t *testing.T) {if got != wantTypes[i] {t.Errorf("chunk %d: doc_type_kwd = %q, want %q (full chunk: %+v)", i, got, wantTypes[i], ch)}- if _, ok := ch["ck_type"]; ok {- t.Errorf("chunk %d must not emit the retired ck_type: %+v", i, ch)- }}// The two text segments on either side of the table/image must remain// distinct — they must NOT be merged across the non-text segments.diff --git a/internal/ingestion/component/schema/chunk_docitem_test.go b/internal/ingestion/component/schema/chunk_docitem_test.goindex a678bc720b..7c97f44e62 100644--- a/internal/ingestion/component/schema/chunk_docitem_test.go+++ b/internal/ingestion/component/schema/chunk_docitem_test.go@@ -179,19 +179,19 @@ func TestDocItemsFromMaps_ParallelsMapPath(t *testing.T) {{name: "text with extra passthrough",maps: []map[string]any{- {"text": "hello", parser.DocTypeKey: parser.DocTypeText, "ck_type": "heading", "layout": "1", "page_number": 3},+ {"text": "hello", parser.DocTypeKey: parser.DocTypeText, "layout": "1", "page_number": 3},},},{name: "image with extra passthrough",maps: []map[string]any{- {"text": "alt", "image": "data:image/png;base64,xxx", parser.DocTypeKey: parser.DocTypeImage, "ck_type": "image", "page_number": 2},+ {"text": "alt", "image": "data:image/png;base64,xxx", parser.DocTypeKey: parser.DocTypeImage, "page_number": 2},},},{name: "table with sheet and positions",maps: []map[string]any{- {"table": &entity.TableData{}, parser.DocTypeKey: parser.DocTypeTable, "ck_type": parser.DocTypeTable, "sheet": "s", "positions": [][]float64{{1, 2}}, "sheet_index": 0},+ {"table": &entity.TableData{}, parser.DocTypeKey: parser.DocTypeTable, "sheet": "s", "positions": [][]float64{{1, 2}}, "sheet_index": 0},},},{@@ -203,7 +203,8 @@ func TestDocItemsFromMaps_ParallelsMapPath(t *testing.T) {{name: "mixed typed and opaque",maps: []map[string]any{- {"text": "a", parser.DocTypeKey: parser.DocTypeText, "ck_type": "text"},+ {"text": "a", parser.DocTypeKey: parser.DocTypeText},+{"foo": "bar"},},},diff --git a/internal/ingestion/component/schema/chunk_types.go b/internal/ingestion/component/schema/chunk_types.goindex cf2de2738b..89a2c988c3 100644--- a/internal/ingestion/component/schema/chunk_types.go+++ b/internal/ingestion/component/schema/chunk_types.go@@ -155,11 +155,9 @@ func (d *ChunkDoc) UnmarshalJSON(data []byte) error {// tagger emits tag_feas and the vocabulary comes from a tag source file,// so no chunk carries tag_kwd. Deleted rather than left in raw so it// cannot fall through to Extra and be re-emitted by MarshalJSON.- // ck_type is a retired parser key: it has no field, but must stay in this- // list so an old payload cannot leak it back through Extra.for _, key := range []string{"text", "doc_type_kwd", "mom", "img_id",- "ck_type", "tk_nums", "layout", "layout_type", "layoutno", "image",+ "tk_nums", "layout", "layout_type", "layoutno", "image","context_above", "context_below", "questions", "keywords", "summary","chunk_order_int", "title_tks", "title_sm_tks", "content_ltks","content_sm_ltks", "tag_kwd", "page_number", "top_int", "_pdf_positions", "positions",@@ -185,10 +183,6 @@ func (d ChunkDoc) MarshalJSON() ([]byte, error) {return nil, err}for k, v := range d.Extra {- // ck_type is retired; a stray Extra key must not resurrect it.- if k == "ck_type" {- continue- }if _, exists := raw[k]; !exists {raw[k] = v}@@ -299,9 +293,6 @@ func (d ChunkDoc) ToMap() map[string]any {return map[string]any{}}for k, raw := range d.Extra {- if k == "ck_type" {- continue- }out[k] = decodeExtraValue(raw)}// Every pre-index chunk has one canonical text field. Keep the key evendiff --git a/internal/ingestion/component/schema/schema_test.go b/internal/ingestion/component/schema/schema_test.goindex 862f438d81..97d336e13b 100644--- a/internal/ingestion/component/schema/schema_test.go+++ b/internal/ingestion/component/schema/schema_test.go@@ -267,7 +267,7 @@ func TestChunkDocSpreadsheetFieldsRoundTrip(t *testing.T) {// round trip through Extra; the index boundary strips them from the store.func TestChunkDocLegacyRowIRKeysPassThrough(t *testing.T) {var decoded ChunkDoc- raw := `{"text":"row","doc_type_kwd":"text","ck_type":"table_row","table_id":"sheet-2","headers":["ID","Status"],"cells":["A-100","paid"],"row_start":42,"row_end":42,"col_start":1,"col_end":3}`+ raw := `{"text":"row","doc_type_kwd":"text","table_id":"sheet-2","headers":["ID","Status"],"cells":["A-100","paid"],"row_start":42,"row_end":42,"col_start":1,"col_end":3}`if err := json.Unmarshal([]byte(raw), &decoded); err != nil {t.Fatalf("unmarshal: %v", err)}@@ -277,32 +277,6 @@ func TestChunkDocLegacyRowIRKeysPassThrough(t *testing.T) {t.Errorf("legacy key %q lost in round trip: %#v", key, out)}}- if _, ok := out["ck_type"]; ok {- t.Errorf("retired ck_type must not survive the round trip: %#v", out)- }-}--// A programmatic Extra entry must not resurrect the retired ck_type key on-// either projection (MarshalJSON or ToMap).-func TestChunkDocRetiredCKTypeCannotBeResurrectedViaExtra(t *testing.T) {- d := ChunkDoc{- Text: "body",- Extra: map[string]json.RawMessage{"ck_type": json.RawMessage(`"heading"`)},- }- if _, ok := d.ToMap()["ck_type"]; ok {- t.Fatalf("ToMap resurrected ck_type: %#v", d.ToMap())- }- data, err := json.Marshal(d)- if err != nil {- t.Fatalf("marshal: %v", err)- }- var raw map[string]any- if err := json.Unmarshal(data, &raw); err != nil {- t.Fatalf("unmarshal: %v", err)- }- if _, ok := raw["ck_type"]; ok {- t.Fatalf("MarshalJSON resurrected ck_type: %s", data)- }}func TestChunkDocVisualParentMetadataRoundTrip(t *testing.T) {diff --git a/internal/ingestion/task/indexdoc/process.go b/internal/ingestion/task/indexdoc/process.goindex e18bab93b2..b6eaafd08e 100644--- a/internal/ingestion/task/indexdoc/process.go+++ b/internal/ingestion/task/indexdoc/process.go@@ -154,11 +154,8 @@ func cleanupConsumedChunkFields(ck map[string]any) {// persist fields — but Go's chunker hands them on, and the write boundary is// where they stop: Elasticsearch and Vastbase would add a field for each key,// and Infinity would drop each key at insert with a warning.-//-// ck_type is retired but stays here as a tombstone: a legacy producer can still-// hand it on, and no engine has a column for it.var pipelineOnlyFields = []string{- "ck_type", "tk_nums", "layout", "layout_type", "layoutno", "image",+ "tk_nums", "layout", "layout_type", "layoutno", "image","context_above", "context_below", "page_number", "_preserve_order","table_id", "sheet", "sheet_index", "headers", "cells","row_start", "row_end", "col_start", "col_end",diff --git a/internal/ingestion/task/indexdoc/process_test.go b/internal/ingestion/task/indexdoc/process_test.goindex a909ad7b19..adb07e1c27 100644--- a/internal/ingestion/task/indexdoc/process_test.go+++ b/internal/ingestion/task/indexdoc/process_test.go@@ -668,14 +668,14 @@ func TestCleanupConsumedChunkFields_ImportantKwdDropsEmptyParts(t *testing.T) {// TestProcessChunksForPipeline_StripsPipelineOnlyFields pins the index boundary// against the parser/chunker bookkeeping keys: none of them is a chunk-store-// column, and a strict engine rejects the whole insert over one of them-// ("Column ck_type not found in table", InfinityException 3013). ES only-// swallowed them because its mapping is dynamic.+// column, and a strict engine rejects the whole insert over one of them (e.g.+// Infinity would reject an unknown column). ES only swallowed them because its+// mapping is dynamic.func TestProcessChunksForPipeline_StripsPipelineOnlyFields(t *testing.T) {ck := map[string]any{"text": "hello",// Every bookkeeping key the chunkers/parsers can leave on a chunk.- "ck_type": "text", "tk_nums": 3, "layout": "text", "layout_type": "text",+ "tk_nums": 3, "layout": "text", "layout_type": "text","layoutno": "0", "image": "data:image/png;base64,AAAA","context_above": "above", "context_below": "below", "page_number": 2,"table_id": "t1", "sheet": "s1", "sheet_index": 0,diff --git a/internal/parser/parser/doc_item.go b/internal/parser/parser/doc_item.goindex da131a19cc..29fa18dc18 100644--- a/internal/parser/parser/doc_item.go+++ b/internal/parser/parser/doc_item.go@@ -175,13 +175,13 @@ func DocItemFromMap(m map[string]any) DocItem {func textItemFromMap(m map[string]any) DocItem {return TextItem{Text: stringField(m, "text"),- Extra: extraFromMap(m, "text", DocTypeKey, "ck_type"),+ Extra: extraFromMap(m, "text", DocTypeKey),}}func imageItemFromMap(m map[string]any) DocItem {pos := wireToPositions(m["positions"], hasSheetIndex(m))- skip := []string{"text", "image", DocTypeKey, "ck_type", "sheet"}+ skip := []string{"text", "image", DocTypeKey, "sheet"}if pos != nil {// Parsed into the typed field: it owns "positions", so strip it from// Extra. Both [][]float64 (office/xlsx) and [][]any (PDF crop regions@@ -209,7 +209,7 @@ func tableItemFromMap(m map[string]any) DocItem {return RawItem{M: m}}pos := wireToPositions(m["positions"], hasSheetIndex(m))- skip := []string{"table", DocTypeKey, "ck_type", "sheet"}+ skip := []string{"table", DocTypeKey, "sheet"}if pos != nil {skip = append(skip, "positions")}@@ -270,10 +270,6 @@ func ItemsToMaps(items []DocItem) []map[string]any {// present, so a stray Extra key can never shadow text/table/image/doc_type_kwd.func mergeExtra(m, extra map[string]any) map[string]any {for k, v := range extra {- // ck_type is retired; a stray Extra key must not resurrect it.- if k == "ck_type" {- continue- }if _, ok := m[k]; !ok {m[k] = v}diff --git a/internal/parser/parser/doc_item_test.go b/internal/parser/parser/doc_item_test.goindex 3ab4d180b2..4e7d08482e 100644--- a/internal/parser/parser/doc_item_test.go+++ b/internal/parser/parser/doc_item_test.go@@ -52,8 +52,7 @@ func TestTextItem_ToMap(t *testing.T) {}func TestTextItem_ExtraDoesNotClobberCanonical(t *testing.T) {- // An attacker/stray Extra key must never shadow the typed fields, and a- // retired ck_type must not be resurrected through Extra.+ // An attacker/stray Extra key must never shadow the typed fields.got := TextItem{Text: "real",Extra: map[string]any{@@ -292,24 +291,6 @@ func TestDocItemFromMap_WireShapePreserved(t *testing.T) {}}-// A legacy payload may still carry the retired ck_type key. Typed conversion-// must drop it rather than route it through Extra and re-emit it.-func TestDocItemFromMap_DropsRetiredCKType(t *testing.T) {- legacy := map[string]any{- "text": "h",- DocTypeKey: DocTypeText,- "ck_type": "heading",- "layout": "heading",- }- got := DocItemFromMap(legacy).ToMap()- if _, ok := got["ck_type"]; ok {- t.Fatalf("retired ck_type leaked through ToMap: %#v", got)- }- if got["layout"] != "heading" {- t.Fatalf("layout lost during conversion: %#v", got)- }-}-// DocItemsFromMaps maps a []map[string]any to typed DocItems the same way// DocItemFromMap does, preserving order and skipping nothing.func TestMapsToItems_OrderAndType(t *testing.T) {diff --git a/internal/parser/parser/html_parser_align_test.go b/internal/parser/parser/html_parser_align_test.goindex dcf5f6610e..51a1554a54 100644--- a/internal/parser/parser/html_parser_align_test.go+++ b/internal/parser/parser/html_parser_align_test.go@@ -61,9 +61,6 @@ func TestHTMLParser_TableProducesStructuredItems(t *testing.T) {if tableCount != 1 {t.Fatalf("structured doc_type_kwd:\"table\" item count = %d, want exactly 1", tableCount)}- if _, ok := res.JSON[tableIdx].ToMap()["ck_type"]; ok {- t.Errorf("retired ck_type present on structured table item: %#v", res.JSON[tableIdx].ToMap())- }if !strings.Contains(tableText, "<table") ||!strings.Contains(tableText, "Name") ||!strings.Contains(tableText, "Alice") {@@ -144,9 +141,6 @@ func TestHTMLParser_NestedTableProducesStructuredItems(t *testing.T) {if tableCount != 1 {t.Fatalf("structured doc_type_kwd:\"table\" item count = %d, want exactly 1", tableCount)}- if _, ok := res.JSON[tableIdx].ToMap()["ck_type"]; ok {- t.Errorf("retired ck_type present on structured nested table item: %#v", res.JSON[tableIdx].ToMap())- }if !strings.Contains(tableText, "<table") ||!strings.Contains(tableText, "Name") ||!strings.Contains(tableText, "Alice") {diff --git a/internal/parser/parser/markdown_parser_test.go b/internal/parser/parser/markdown_parser_test.goindex 1de9e9e125..3218b3e246 100644--- a/internal/parser/parser/markdown_parser_test.go+++ b/internal/parser/parser/markdown_parser_test.go@@ -273,9 +273,6 @@ func TestMarkdownParser_ParseWithResult_RendersTableInline(t *testing.T) {switch kd {case "table":tableCount++- if _, ok := item["ck_type"]; ok {- t.Errorf("retired ck_type present on table item: %#v", item)- }td, ok := item["table"].(*entity.TableData)if !ok || td == nil {t.Fatalf("table item missing *entity.TableData: %#v", item)@@ -648,9 +645,6 @@ func TestMarkdownParser_TableNotCollapsed(t *testing.T) {if _, ok := item["table"].(*entity.TableData); !ok {t.Fatalf("table item missing *entity.TableData: %#v", item)}- if _, ok := item["ck_type"]; ok {- t.Fatalf("retired ck_type present on table item: %#v", item)- }}}concat := all.String()@@ -713,9 +707,6 @@ func TestMarkdownParser_TableWithSurroundingText(t *testing.T) {if _, ok := item["table"].(*entity.TableData); !ok {t.Fatalf("table item missing *entity.TableData: %#v", item)}- if _, ok := item["ck_type"]; ok {- t.Fatalf("retired ck_type present on table item: %#v", item)- }}}@@ -760,9 +751,6 @@ func TestMarkdownParser_NonTableHTMLBlockNotTable(t *testing.T) {for _, item := range ItemsToMaps(res.JSON) {text, _ := item["text"].(string)if strings.Contains(text, "<div") || strings.Contains(text, "<style") {- if _, ok := item["ck_type"]; ok {- t.Fatalf("retired ck_type present on non-table item: %q", text)- }if kd, _ := item["doc_type_kwd"].(string); kd == "table" {t.Fatalf("non-table HTML block wrongly tagged doc_type_kwd:\"table\": %q", text)}@@ -833,9 +821,6 @@ func TestMarkdownParser_RawHTMLTableHandled(t *testing.T) {if _, ok := item["table"].(*entity.TableData); !ok {t.Fatalf("raw table item missing *entity.TableData: %#v", item)}- if _, ok := item["ck_type"]; ok {- t.Fatalf("retired ck_type present on raw table item: %#v", item)- }}}if sawInlined {diff --git a/internal/parser/parser/pdf_result.go b/internal/parser/parser/pdf_result.goindex 4cc77653cf..6933b073e7 100644--- a/internal/parser/parser/pdf_result.go+++ b/internal/parser/parser/pdf_result.go@@ -128,10 +128,6 @@ func (p *PDFParser) NormalizeResult(ctx context.Context, filename string, result}fields["doc_type_kwd"] = "text"}- // ck_type is retired: the content type lives on doc_type_kwd and the- // block kind on layout. A legacy raw item can still carry the key, so- // drop it here as well — the typed conversion already does.- delete(fields, "ck_type")items = appendPDFItem(items, fields)}if droppedPositions > 0 {diff --git a/internal/parser/parser/pdf_result_test.go b/internal/parser/parser/pdf_result_test.goindex d2b00d4812..903517c0fc 100644--- a/internal/parser/parser/pdf_result_test.go+++ b/internal/parser/parser/pdf_result_test.go@@ -142,9 +142,6 @@ func TestPDFNormalizeResultFlattenPreservesContentAndClearsMediaSubtype(t *testiif fields["doc_type_kwd"] != "text" {t.Fatalf("flattened item %d still has media type: %#v", i, fields)}- if _, ok := fields["ck_type"]; ok {- t.Fatalf("flattened item %d still carries the retired ck_type: %#v", i, fields)- }}text, _ := result.JSON[0].ToMap()["text"].(string)for _, want := range []string{"Cell", "Caption", "Footnote"} {@@ -165,7 +162,7 @@ func TestPDFNormalizeResultFlattenPreservesContentAndClearsMediaSubtype(t *testifunc TestPDFNormalizeResultDowngradesUnstructuredTableSubtype(t *testing.T) {result := NewPDFParser().NormalizeResult(t.Context(), "sample.pdf", ParseResult{OutputFormat: "json", JSON: MapsToItems([]map[string]any{{- "doc_type_kwd": "table", "ck_type": "table", "layout": "table", "text": "Recognized text without cells",+ "doc_type_kwd": "table", "layout": "table", "text": "Recognized text without cells",}})})if result.Err != nil {t.Fatal(result.Err)@@ -174,9 +171,6 @@ func TestPDFNormalizeResultDowngradesUnstructuredTableSubtype(t *testing.T) {if item["doc_type_kwd"] != "text" || item["text"] != "Recognized text without cells" {t.Fatalf("unstructured table kept an inconsistent subtype: %#v", item)}- if _, ok := item["ck_type"]; ok {- t.Fatalf("retired ck_type survived normalization: %#v", item)- }}func TestPDFNormalizeResultKeepsPositionedUnstructuredTables(t *testing.T) {@@ -184,7 +178,7 @@ func TestPDFNormalizeResultKeepsPositionedUnstructuredTables(t *testing.T) {p := NewPDFParser()p.FlattenMediaToText = flattenresult := p.NormalizeResult(t.Context(), "sample.pdf", ParseResult{OutputFormat: "json", JSON: MapsToItems([]map[string]any{{- "doc_type_kwd": "table", "ck_type": "table", "text": "Recognized table without cells",+ "doc_type_kwd": "table", "text": "Recognized table without cells","positions": [][]any{{2, 10, 30, 20, 40}},}})})if result.Err != nil || len(result.JSON) != 1 {