RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.
Go agent-harnessagent-native-databaseagentic-aiagentic-retrievalagentic-searchaiai-agentscontext-enginecontext-engineeringcontext-managementharness-engineeringknowledge-compilationragretrieval-augmented-generationretrieval-harnesssearch-harness
fix(knowledge_compile): make compiled rows writable to Infinity (#20208)
qinling0210da98d6267124
diff --git a/internal/engine/infinity/chunk.go b/internal/engine/infinity/chunk.goindex 502c565046..00fa7865ea 100644--- a/internal/engine/infinity/chunk.go+++ b/internal/engine/infinity/chunk.go@@ -25,6 +25,7 @@ import ("ragflow/internal/common""ragflow/internal/engine/types""ragflow/internal/utility"+ "reflect""regexp""slices""sort"@@ -134,21 +135,15 @@ func (e *Engine) createChunkStoreWithDB(db *infinity.Database, baseName, datasetcommon.Info("Successfully added vector column", zap.String("column", vectorColName))}- // A metadata-only Wiki MAP write can create the dataset table before a- // table-parser document arrives. Reconcile the parser-specific column on- // the existing-table path so creation order does not make table chunks- // unwritable.+ // Reconcile the parser-specific column on the existing-table path: the+ // table may have been created ahead of its first table-parser document.if parserID == "table" {chunkDataExists, checkErr := e.columnExists(table, "chunk_data")if checkErr != nil {return fmt.Errorf("failed to check chunk_data column: %w", checkErr)}- if !chunkDataExists {- if _, addErr := table.AddColumns(infinity.TableSchema{- &infinity.ColumnDefinition{Name: "chunk_data", DataType: "json", Default: "{}"},- }); addErr != nil {- return fmt.Errorf("failed to add chunk_data column: %w", addErr)- }+ if err := e.ensureChunkDataColumn(table, chunkDataExists); err != nil {+ return err}}} else {@@ -284,6 +279,32 @@ func (e *Engine) createChunkStoreWithDB(db *infinity.Database, baseName, datasetreturn nil}+// parserIDFromChunks reports the parser that produced a batch: a chunk carrying+// chunk_data comes from the table parser (Python's create_idx(parser_id)).+func parserIDFromChunks(chunks []map[string]interface{}) string {+ for _, chunk := range chunks {+ if chunkData, ok := chunk["chunk_data"].(map[string]interface{}); ok && chunkData != nil {+ return "table"+ }+ }+ return ""+}++// ensureChunkDataColumn adds the column the table parser writes when a table+// created ahead of its first table-parser document lacks it. exists is the+// caller's already-known column existence.+func (e *Engine) ensureChunkDataColumn(table *infinity.Table, exists bool) error {+ if exists {+ return nil+ }+ if _, err := table.AddColumns(infinity.TableSchema{+ &infinity.ColumnDefinition{Name: "chunk_data", DataType: "json", Default: "{}"},+ }); err != nil {+ return fmt.Errorf("failed to add chunk_data column: %w", err)+ }+ return nil+}+// InsertChunks inserts documents into a dataset table;// Table name format: {baseName}_{datasetID}// Auto-create the table if it doesn't exist@@ -325,14 +346,8 @@ func (e *Engine) InsertChunks(ctx context.Context, chunks []map[string]interfacereturn nil, fmt.Errorf("cannot infer vector size from chunks")}- // Determine parser_id from chunk structure- parserID := ""- if chunkData, ok := chunks[0]["chunk_data"].(map[string]interface{}); ok && chunkData != nil {- parserID = "table"- }-// Create table- if err := e.createChunkStoreWithDB(db, baseName, datasetID, vectorSize, parserID); err != nil {+ if err := e.createChunkStoreWithDB(db, baseName, datasetID, vectorSize, parserIDFromChunks(chunks)); err != nil {return nil, fmt.Errorf("failed to create table: %w", err)}@@ -355,11 +370,15 @@ func (e *Engine) InsertChunks(ctx context.Context, chunks []map[string]interface// ShowColumns returns a result set where Data contains arrays of column valuesre := regexp.MustCompile(`Embedding\([a-z]+,(\d+)\)`)+ hasChunkData := falseif nameArr, ok := result.Data["name"]; ok {if typeArr, ok := result.Data["type"]; ok {for i := 0; i < len(nameArr); i++ {colName, _ := nameArr[i].(string)colType, _ := typeArr[i].(string)+ if colName == "chunk_data" {+ hasChunkData = true+ }matches := re.FindStringSubmatch(colType)if len(matches) >= 2 {size, _ := strconv.Atoi(matches[1])@@ -369,6 +388,15 @@ func (e *Engine) InsertChunks(ctx context.Context, chunks []map[string]interface}}+ // Reconcile the parser column here too: an existing table is opened through+ // GetTable and never reaches createChunkStoreWithDB. Best effort, like Python's+ // ensure_columns — a column that stays missing surfaces on the insert below.+ if parserIDFromChunks(chunks) == "table" {+ if err := e.ensureChunkDataColumn(table, hasChunkData); err != nil {+ common.Warn("Failed to add chunk_data column", zap.Error(err))+ }+ }+// Transform chunks using helper functioninsertChunks := make([]map[string]interface{}, len(chunks))for i, chunk := range chunks {@@ -2438,31 +2466,39 @@ func equivalentConditionToStr(condition map[string]interface{}, tableColumns mapreturn strings.Join(cond, " AND ")}-// decodeJSONFields converts Infinity JSON-column strings back into their Go-// array/object representation. Legacy list rows used ###-joined strings, so-// preserve read compatibility for those columns.+// decodeJSONFields converts Infinity JSON-column values back into their Go+// representation, mirroring Python's read-side parse_json_list for the list+// columns (infinity_conn.py:937-947): a slice passes through, nil becomes [], a+// JSON non-list value is wrapped, a legacy "###" string is split, and object+// columns such as source_chunk_hashes keep their shape.func decodeJSONFields(chunk map[string]interface{}) {for fieldName, value := range chunk {- fieldNameLower := strings.ToLower(fieldName)- if !fieldJSON(fieldNameLower) {+ if !fieldJSON(fieldName) {continue}+ listField := fieldJSONList(fieldName)text, ok := value.(string)if !ok {+ if listField {+ chunk[fieldName] = asJSONList(value)+ }continue}if text == "" {- if fieldJSONList(fieldNameLower) {+ if listField {chunk[fieldName] = []interface{}{}}continue}var decoded interface{}if err := json.Unmarshal([]byte(text), &decoded); err == nil {+ if listField {+ decoded = asJSONList(decoded)+ }chunk[fieldName] = decodedcontinue}- if fieldJSONList(fieldNameLower) {+ if listField {parts := strings.Split(text, "###")decodedList := make([]interface{}, 0, len(parts))for _, part := range parts {@@ -2475,6 +2511,19 @@ func decodeJSONFields(chunk map[string]interface{}) {}}+// asJSONList normalizes a list-column value the way Python's parse_json_list+// does (infinity_conn.py:938-944): a slice passes through unchanged, a nil value+// becomes [] and any other scalar is wrapped in a single-element list.+func asJSONList(value interface{}) interface{} {+ if value == nil {+ return []interface{}{}+ }+ if rv := reflect.ValueOf(value); rv.Kind() == reflect.Slice || rv.Kind() == reflect.Array {+ return value+ }+ return []interface{}{value}+}+// calculateScores calculates _score = score_column + pagerankfunc calculateScores(chunks []map[string]interface{}, scoreColumn, pagerankField string) []map[string]interface{} {for i := range chunks {@@ -2637,9 +2686,11 @@ func numericSlice(v interface{}) ([]interface{}, bool) {// Also handles:// - kb_id: extracts first element if it's a list// - position_int, page_num_int, top_int: converts arrays to hex strings+// - fieldJSON / fieldJSONList: json column -> JSON string (Infinity json columns)// - tag_kwd: joins with ### separator// - question_kwd: joins with newline separator-// - chunk_data: dict -> JSON string+// - other *_kwd keyword fields: lists join with ###+// - chunk_data / extra: dict -> JSON string (chunk-table varchar columns)// - Missing embeddings filled with zeros if embeddingCols providedfunc transformChunkFields(chunk map[string]interface{}, embeddingCols [][2]interface{}) map[string]interface{} {d := make(map[string]interface{})@@ -2744,17 +2795,27 @@ func transformChunkFields(chunk map[string]interface{}, embeddingCols [][2]inter}case "chunk_data":d["chunk_data"] = utility.ConvertMapToJSONString(v)+ case "extra":+ // Python json-encodes a dict-valued `extra` (infinity_conn.py:572-581);+ // raw, the SDK reads the map as a sparse-vector literal and rejects the+ // string keys. meta_fields is not a chunk column — InsertMetadata+ // encodes it on the doc-meta path (infinity/metadata.go).+ d[k] = utility.ConvertMapToJSONString(v)default:if fieldJSON(k) {- d[k] = encodeJSONField(v)+ d[k] = encodeJSONField(k, v)} else if strings.HasSuffix(k, "_feas") {jsonBytes, _ := json.Marshal(v)d[k] = string(jsonBytes)} else if fieldKeyword(k) {- // keyword fields with list values -> ### joined- if list, ok := v.([]interface{}); ok {+ // keyword fields with list values -> ### joined; accept the+ // Go-native []string as well as []interface{}.+ switch list := v.(type) {+ case []string:+ d[k] = strings.Join(list, "###")+ case []interface{}:d[k] = strings.Join(utility.ConvertToStringSlice(list), "###")- } else {+ default:d[k] = v}} else {@@ -2789,13 +2850,34 @@ func transformChunkFields(chunk map[string]interface{}, embeddingCols [][2]interreturn d}-func encodeJSONField(value interface{}) interface{} {- if text, ok := value.(string); ok && json.Valid([]byte(text)) {+// encodeJSONField renders a json column value as the string Infinity stores. The+// Go SDK cannot encode a []string constant (3058), so a list is JSON-dumped; a+// non-list value falls back to the column default ("[]" for the json-list columns,+// infinity_conn.py:557-558; "{}" for the object one). Strings pass through, so+// re-transforming a row stays idempotent.+func encodeJSONField(fieldName string, value interface{}) interface{} {+ listField := fieldJSONList(fieldName)+ fallback := "{}"+ if listField {+ fallback = "[]"+ }+ if text, ok := value.(string); ok {+ if text == "" {+ return fallback+ }return text}+ if value == nil {+ return fallback+ }+ if listField {+ if rv := reflect.ValueOf(value); rv.Kind() != reflect.Slice && rv.Kind() != reflect.Array {+ return fallback+ }+ }encoded, err := json.Marshal(value)if err != nil {- return value+ return fallback}return string(encoded)}diff --git a/internal/engine/infinity/chunk_insert_test.go b/internal/engine/infinity/chunk_insert_test.gonew file mode 100644index 0000000000..d4c6622237--- /dev/null+++ b/internal/engine/infinity/chunk_insert_test.go@@ -0,0 +1,50 @@+// Copyright 2026 The InfiniFlow Authors. All Rights Reserved.+//+// Licensed under the Apache License, Version 2.0 (the "License");+// you may not use this file except in compliance with the License.+// You may obtain a copy of the License at+//+// http://www.apache.org/licenses/LICENSE-2.0+//+// Unless required by applicable law or agreed to in writing, software+// distributed under the License is distributed on an "AS IS" BASIS,+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+// See the License for the specific language governing permissions and+// limitations under the License.++package infinity++import "testing"++// TestParserIDFromChunks pins the table-parser detection that gates the+// chunk_data column reconcile: only a batch that carries chunk_data needs the+// parser-specific column, and any position in the batch counts (a mixed batch+// still writes chunk_data on the rows that have it).+func TestParserIDFromChunks(t *testing.T) {+ var nilMap map[string]interface{}++ tests := []struct {+ name string+ chunks []map[string]interface{}+ want string+ }{+ {name: "no chunks", chunks: nil, want: ""},+ {name: "plain chunk", chunks: []map[string]interface{}{{"id": "c1"}}, want: ""},+ {name: "nil chunk_data", chunks: []map[string]interface{}{{"chunk_data": nil}}, want: ""},+ {name: "typed nil chunk_data", chunks: []map[string]interface{}{{"chunk_data": nilMap}}, want: ""},+ {name: "empty chunk_data", chunks: []map[string]interface{}{{"chunk_data": map[string]interface{}{}}}, want: "table"},+ {+ name: "table chunk after a plain chunk",+ chunks: []map[string]interface{}{{"id": "c1"}, {"chunk_data": map[string]interface{}{"rows": []interface{}{}}}},+ want: "table",+ },+ }++ for _, tc := range tests {+ t.Run(tc.name, func(t *testing.T) {+ if got := parserIDFromChunks(tc.chunks); got != tc.want {+ t.Fatalf("parserIDFromChunks() = %q, want %q", got, tc.want)+ }+ })+ }+}diff --git a/internal/engine/infinity/chunk_transform_test.go b/internal/engine/infinity/chunk_transform_test.goindex a0193cee78..5c38ca22b1 100644--- a/internal/engine/infinity/chunk_transform_test.go+++ b/internal/engine/infinity/chunk_transform_test.go@@ -15,8 +15,16 @@package infinityimport (+ "encoding/json"+ "os""reflect"+ "regexp"+ "strings""testing"++ "ragflow/internal/utility"++ infinitysdk "github.com/infiniflow/infinity-go-sdk")// TestTransformChunkFields_IngestionShape is the T0 unit-tier read-back@@ -144,33 +152,6 @@ func TestTransformChunkFieldsJoinsNativeKeywordSlice(t *testing.T) {}}-// TestTransformChunkFieldsJSONSlices serializes Go-native slices for Infinity-// JSON columns. Passing []string directly to the SDK is rejected as an-// unsupported slice element type.-func TestTransformChunkFieldsJSONSlices(t *testing.T) {- got := transformChunkFields(map[string]interface{}{- "source_chunk_ids": []string{"chunk-1", "chunk-2"},- "source_doc_ids": []string{"doc-1"},- }, nil)-- if want := `["chunk-1","chunk-2"]`; got["source_chunk_ids"] != want {- t.Errorf("source_chunk_ids = %#v, want %q", got["source_chunk_ids"], want)- }- if want := `["doc-1"]`; got["source_doc_ids"] != want {- t.Errorf("source_doc_ids = %#v, want %q", got["source_doc_ids"], want)- }-}--func TestTransformChunkFieldsKeepsEncodedJSON(t *testing.T) {- got := transformChunkFields(map[string]interface{}{- "source_doc_ids": `["doc-1"]`,- }, nil)-- if want := `["doc-1"]`; got["source_doc_ids"] != want {- t.Errorf("source_doc_ids = %#v, want %q", got["source_doc_ids"], want)- }-}-func TestDecodeJSONFields(t *testing.T) {row := map[string]interface{}{"source_doc_ids": `["doc-1","doc-2"]`,@@ -207,3 +188,318 @@ func TestTransformChunkFieldsDropsTenantRoutingField(t *testing.T) {t.Fatalf("kb_id = %#v, want %q", got["kb_id"], "kb-1")}}++// compiledRowStringLists are the string-list values the compile path hands over:+// Python's _JSON_LIST_FIELDS columns plus the keyword column children_kwd.+var compiledRowStringLists = map[string]interface{}{+ "source_chunk_ids": []string{"c1", "c2"},+ "source_doc_ids": []string{"d1"},+ "compilation_template_ids": []string{"t1"},+ "doc_ids_kwd": []string{"d1"},+ "entity_names_kwd": []string{"alpha", "玄德"},+ "outlinks_kwd": []string{"entity/beta"},+ "related_kb_pages_kwd": []string{"entity/beta"},+ "rechunked_from_chunk_ids": []string{"c9"},+ "children_kwd": []string{"r1", "r2"},+}++// compiledRowForTransform builds a post-indexdoc compiled chunk row.+func compiledRowForTransform() map[string]interface{} {+ row := map[string]interface{}{+ "id": "c1",+ "doc_id": "d1",+ "kb_id": "kb1",+ "content_with_weight": "compiled body",+ "compile_kwd": "tree",+ "raptor_kwd": "root",+ "raptor_layer_int": 2,+ "md_with_weight": "# page",+ "extra": map[string]interface{}{"raptor_method": "gmm"},+ "q_3_vec": []float64{0.1, 0.2, 0.3},+ }+ for k, v := range compiledRowStringLists {+ row[k] = v+ }+ return row+}++// TestInsertValuesAreEncodableConstants is the regression test for the reported+// failure+//+// insert chunk batch 0-32 after 3 attempts: failed to insert chunks to+// dataset: InfinityException(3058, Unsupported slice element type: string)+//+// The SDK encodes every value as a constant and rejects a Go-native []string+// (expression_parser.go parseSliceConstantValue), so the raw values must NOT+// encode while everything transformChunkFields emits must. If a future SDK+// accepts string slices the repro is gone, so the test skips instead of failing.+func TestInsertValuesAreEncodableConstants(t *testing.T) {+ for k, v := range compiledRowStringLists {+ if _, err := infinitysdk.ParseConstantValue(v); err == nil {+ t.Skipf("SDK now supports string slices; the 3058 repro is gone (raw %s=%#v is encodable)", k, v)+ }+ }++ transformed := transformChunkFields(compiledRowForTransform(), nil)+ for k, v := range transformed {+ if _, err := infinitysdk.ParseConstantValue(v); err != nil {+ t.Errorf("transformed %s = %#v cannot be encoded for the insert: %v", k, v, err)+ }+ }+ // The json columns travel as JSON arrays (Python json.dumps) and the+ // keyword column as the ### join — strings either way, never a Go slice.+ for _, k := range []string{+ "source_chunk_ids", "source_doc_ids", "compilation_template_ids", "doc_ids_kwd",+ "entity_names_kwd", "outlinks_kwd", "related_kb_pages_kwd", "rechunked_from_chunk_ids",+ } {+ if s, ok := transformed[k].(string); !ok || !strings.HasPrefix(s, "[") {+ t.Errorf("transformed %s = %#v, want a JSON array string", k, transformed[k])+ }+ }+ if want := "r1###r2"; transformed["children_kwd"] != want {+ t.Errorf("children_kwd = %#v, want %q", transformed["children_kwd"], want)+ }+ if _, ok := transformed["extra"].(string); !ok {+ t.Errorf("extra = %#v, want a JSON string", transformed["extra"])+ }+}++// TestTransformedCompiledRowNamesOnlyMappingColumns covers the failure that+// follows 3058: Infinity rejects a column the table does not have, which a+// Go-only field such as tenant_id triggered.+func TestTransformedCompiledRowNamesOnlyMappingColumns(t *testing.T) {+ columns := chunkMappingColumns(t)+ vectorColumn := regexp.MustCompile(`^q_\d+_vec$`)+ for k := range transformChunkFields(compiledRowForTransform(), nil) {+ if columns[k] || vectorColumn.MatchString(k) {+ continue+ }+ t.Errorf("transformed key %q is not a column of conf/infinity_mapping.json", k)+ }+}++// TestFieldJSONMatchesJSONMappingColumns keeps the write side's encoding set in+// step with the schema: a column conf/infinity_mapping.json declares as json must+// be encoded by fieldJSON before the insert (Infinity answers 3058 for a raw Go+// slice), and a column fieldJSON encodes must be declared json (a JSON string in+// a typed column is just as wrong).+func TestFieldJSONMatchesJSONMappingColumns(t *testing.T) {+ jsonColumns := 0+ for column, columnType := range mappingColumnTypes(t) {+ isJSON := columnType == "json"+ if isJSON {+ jsonColumns+++ }+ if encoded := fieldJSON(column); encoded != isJSON {+ t.Errorf("fieldJSON(%q) = %v, but conf/infinity_mapping.json declares type %q", column, encoded, columnType)+ }+ }+ if jsonColumns == 0 {+ t.Fatal("conf/infinity_mapping.json parsed with no json column; the mapping read is broken")+ }+}++// chunkMappingColumns reads the column set the chunk table is created from.+func chunkMappingColumns(t *testing.T) map[string]bool {+ columns := mappingColumnTypes(t)+ out := make(map[string]bool, len(columns))+ for name := range columns {+ out[name] = true+ }+ return out+}++// mappingColumnTypes reads conf/infinity_mapping.json as column -> declared type.+func mappingColumnTypes(t *testing.T) map[string]string {+ t.Helper()+ path, err := utility.FindConfFileInProject("infinity_mapping.json")+ if err != nil || path == nil {+ t.Fatalf("locate infinity_mapping.json: %v", err)+ }+ data, err := os.ReadFile(*path)+ if err != nil {+ t.Fatalf("read %s: %v", *path, err)+ }+ var columns map[string]struct {+ Type string `json:"type"`+ }+ if err := json.Unmarshal(data, &columns); err != nil {+ t.Fatalf("parse %s: %v", *path, err)+ }+ out := make(map[string]string, len(columns))+ for name, def := range columns {+ out[name] = def.Type+ }+ return out+}++// TestTransformChunkFieldsSerializesJSONListColumns is the regression for the+// reported 3058: the compile path hands over Go-native []string for the+// json-typed provenance columns, which the Go SDK cannot encode. Python+// json-encodes them (_JSON_LIST_FIELDS).+func TestTransformChunkFieldsSerializesJSONListColumns(t *testing.T) {+ got := transformChunkFields(map[string]interface{}{+ "source_chunk_ids": []string{"c1", "c2"},+ "source_doc_ids": []interface{}{"d1"},+ "compilation_template_ids": []string{"t1"},+ "entity_names_kwd": []string{"刘备", "玄德"},+ "outlinks_kwd": []interface{}{"concept/三国演义"},+ "related_kb_pages_kwd": []string{"entity/person/刘备"},+ }, nil)++ want := map[string]string{+ "source_chunk_ids": `["c1","c2"]`,+ "source_doc_ids": `["d1"]`,+ "compilation_template_ids": `["t1"]`,+ "entity_names_kwd": `["刘备","玄德"]`,+ "outlinks_kwd": `["concept/三国演义"]`,+ "related_kb_pages_kwd": `["entity/person/刘备"]`,+ }+ for field, expected := range want {+ if got[field] != expected {+ t.Errorf("%s = %#v, want %q", field, got[field], expected)+ }+ }+}++// TestTransformChunkFieldsJSONListColumnEdgeCases: empty list -> "[]", an+// already serialized value survives, nil -> "[]".+func TestTransformChunkFieldsJSONListColumnEdgeCases(t *testing.T) {+ got := transformChunkFields(map[string]interface{}{+ "source_chunk_ids": []string{},+ "source_doc_ids": `["already"]`,+ "outlinks_kwd": nil,+ }, nil)++ if want := "[]"; got["source_chunk_ids"] != want {+ t.Errorf("empty source_chunk_ids = %#v, want %q", got["source_chunk_ids"], want)+ }+ if want := `["already"]`; got["source_doc_ids"] != want {+ t.Errorf("serialized source_doc_ids = %#v, want %q", got["source_doc_ids"], want)+ }+ if want := "[]"; got["outlinks_kwd"] != want {+ t.Errorf("nil outlinks_kwd = %#v, want %q", got["outlinks_kwd"], want)+ }+}++// TestTransformChunkFieldsJoinsGenericKeywordSlices covers the keyword branch+// beyond important_kwd: the compile path emits children_kwd / entity_names as+// plain []string, which must be joined like the JSON-decoded []interface{} form.+func TestTransformChunkFieldsJoinsGenericKeywordSlices(t *testing.T) {+ got := transformChunkFields(map[string]interface{}{+ "children_kwd": []string{"c1", "c2"},+ "raptor_kwd": "root",+ }, nil)++ if want := "c1###c2"; got["children_kwd"] != want {+ t.Errorf("children_kwd = %#v, want %q", got["children_kwd"], want)+ }+ if want := "root"; got["raptor_kwd"] != want {+ t.Errorf("raptor_kwd = %#v, want %q", got["raptor_kwd"], want)+ }+}++// TestTransformChunkFieldsSerializesMapColumns pins the dict-valued varchar+// column `extra` (infinity_conn.py:572-581).+func TestTransformChunkFieldsSerializesMapColumns(t *testing.T) {+ got := transformChunkFields(map[string]interface{}{+ "extra": map[string]interface{}{"raptor_method": "gmm"},+ }, nil)++ if want := `{"raptor_method":"gmm"}`; got["extra"] != want {+ t.Errorf("extra = %#v, want %q", got["extra"], want)+ }+}++// TestTransformChunkFieldsSerializesObjectJSONColumn pins the object-typed json+// column (source_chunk_hashes): a map is dumped and a nil value falls back to+// the mapping default "{}".+func TestTransformChunkFieldsSerializesObjectJSONColumn(t *testing.T) {+ got := transformChunkFields(map[string]interface{}{+ "source_chunk_hashes": map[string]interface{}{"c1": "h1"},+ }, nil)+ if want := `{"c1":"h1"}`; got["source_chunk_hashes"] != want {+ t.Errorf("source_chunk_hashes = %#v, want %q", got["source_chunk_hashes"], want)+ }++ got = transformChunkFields(map[string]interface{}{"source_chunk_hashes": nil}, nil)+ if want := "{}"; got["source_chunk_hashes"] != want {+ t.Errorf("nil source_chunk_hashes = %#v, want %q", got["source_chunk_hashes"], want)+ }+}++// TestDecodeJSONFieldsNormalizesListColumns pins the parse_json_list cases that+// do not arrive as JSON strings (infinity_conn.py:938-944): a nil value is [],+// a JSON non-list value is wrapped, an object column keeps its shape.+func TestDecodeJSONFieldsNormalizesListColumns(t *testing.T) {+ row := map[string]interface{}{+ "related_kb_pages_kwd": nil,+ "outlinks_kwd": float64(3.5),+ "source_doc_ids": `[]`,+ "entity_names_kwd": []interface{}{"e1"},+ "source_chunk_hashes": nil,+ "content_with_weight": `{"x":1}`,+ "docnm_kwd": "plain",+ }++ decodeJSONFields(row)++ if want := []interface{}{}; !reflect.DeepEqual(row["related_kb_pages_kwd"], want) {+ t.Errorf("nil related_kb_pages_kwd = %#v, want %#v", row["related_kb_pages_kwd"], want)+ }+ if want := []interface{}{float64(3.5)}; !reflect.DeepEqual(row["outlinks_kwd"], want) {+ t.Errorf("scalar outlinks_kwd = %#v, want %#v", row["outlinks_kwd"], want)+ }+ if want := []interface{}{}; !reflect.DeepEqual(row["source_doc_ids"], want) {+ t.Errorf("empty source_doc_ids = %#v, want %#v", row["source_doc_ids"], want)+ }+ if want := []interface{}{"e1"}; !reflect.DeepEqual(row["entity_names_kwd"], want) {+ t.Errorf("decoded entity_names_kwd = %#v, want %#v", row["entity_names_kwd"], want)+ }+ if row["source_chunk_hashes"] != nil {+ t.Errorf("object column source_chunk_hashes = %#v, want it untouched", row["source_chunk_hashes"])+ }+ if row["content_with_weight"] != `{"x":1}` {+ t.Errorf("content_with_weight should be untouched, got %#v", row["content_with_weight"])+ }+ if row["docnm_kwd"] != "plain" {+ t.Errorf("docnm_kwd should be untouched, got %#v", row["docnm_kwd"])+ }+}++// TestApplyFieldMappingsDecodesJSONList confirms the search-path post-processor+// normalizes JSON-string columns into slices, so get chunk / search / get fields+// all hand readers the same shape.+func TestApplyFieldMappingsDecodesJSONList(t *testing.T) {+ chunks := []map[string]interface{}{{+ "source_doc_ids": `["d1","d2"]`,+ "source_chunk_ids": `["c1"]`,+ "entity_names_kwd": `["e1","e2"]`,+ "outlinks_kwd": `["o1"]`,+ "doc_ids_kwd": `["x1"]`,+ }}++ applyFieldMappings(chunks)++ for _, key := range []string{"source_doc_ids", "source_chunk_ids", "entity_names_kwd", "outlinks_kwd", "doc_ids_kwd"} {+ if _, ok := chunks[0][key].([]interface{}); !ok {+ t.Errorf("%s not decoded into []interface{}, got %#v", key, chunks[0][key])+ }+ }+}++// TestJSONColumnsMatchCaseInsensitively pins that the write path+// (transformChunkFields) and the read path (decodeJSONFields) agree on a json+// column name regardless of its spelling.+func TestJSONColumnsMatchCaseInsensitively(t *testing.T) {+ got := transformChunkFields(map[string]interface{}{"Source_Doc_IDs": []string{"d1"}}, nil)+ if want := `["d1"]`; got["Source_Doc_IDs"] != want {+ t.Errorf("Source_Doc_IDs = %#v, want %q", got["Source_Doc_IDs"], want)+ }++ row := map[string]interface{}{"Source_Doc_IDs": nil}+ decodeJSONFields(row)+ if want := []interface{}{}; !reflect.DeepEqual(row["Source_Doc_IDs"], want) {+ t.Errorf("nil Source_Doc_IDs = %#v, want %#v", row["Source_Doc_IDs"], want)+ }+}diff --git a/internal/engine/infinity/common.go b/internal/engine/infinity/common.goindex 71c4561356..eb28450bc9 100644--- a/internal/engine/infinity/common.go+++ b/internal/engine/infinity/common.go@@ -159,9 +159,11 @@ func fieldKeyword(fieldName string) bool {}// fieldJSON reports fields stored as Infinity JSON columns. The Infinity Go-// SDK expects JSON columns as encoded strings, not Go slices or maps.+// SDK expects JSON columns as encoded strings, not Go slices or maps. Column+// names are matched case-insensitively so the write path (transformChunkFields)+// and the read path (decodeJSONFields) agree on every spelling.func fieldJSON(fieldName string) bool {- switch fieldName {+ switch strings.ToLower(fieldName) {case "source_chunk_ids", "source_doc_ids", "compilation_template_ids","doc_ids_kwd", "entity_names_kwd", "entity_names", "outlinks_kwd","related_kb_pages_kwd", "claims", "page_ids", "source_chunk_hashes",@@ -175,7 +177,7 @@ func fieldJSON(fieldName string) bool {// fieldJSONList reports JSON columns whose value is an array and whose filter// semantics are membership rather than whole-document equality.func fieldJSONList(fieldName string) bool {- switch fieldName {+ switch strings.ToLower(fieldName) {case "source_chunk_ids", "source_doc_ids", "compilation_template_ids","doc_ids_kwd", "entity_names_kwd", "entity_names", "outlinks_kwd","related_kb_pages_kwd", "claims", "page_ids",diff --git a/internal/ingestion/component/knowledge_compiler/component.go b/internal/ingestion/component/knowledge_compiler/component.goindex 832d40713f..7eb1113b6e 100644--- a/internal/ingestion/component/knowledge_compiler/component.go+++ b/internal/ingestion/component/knowledge_compiler/component.go@@ -392,12 +392,9 @@ var variantCompileKWD = map[common.Variant]string{}// productsToChunkDocs converts the internal compiled Product rows into-// schema.ChunkDoc values aligned to conf/infinity_mapping.json (lines 1–77).-// The compile_kwd discriminator marks them as compiled knowledge units-// (distinct from plain chunks); variant-specific columns are populated from-// Product.Meta using stable keys (see each variant's build site for the-// contract). The original kind/level/name/size meta is also carried under-// "kc_"-prefixed Extra keys so no information is lost.+// schema.ChunkDoc values aligned to conf/infinity_mapping.json. Product.Meta is+// not persisted: only engine columns are written, because Infinity rejects an+// insert naming an unknown column.func productsToChunkDocs(products []common.Product) ([]schema.ChunkDoc, error) {docs := make([]schema.ChunkDoc, 0, len(products))for _, p := range products {@@ -437,11 +434,6 @@ func productsToChunkDocs(products []common.Product) ([]schema.ChunkDoc, error) {return nil, err}}- if p.TenantID != "" {- if err := doc.SetExtraValue("tenant_id", p.TenantID); err != nil {- return nil, err- }- }compileKWD := variantCompileKWD[p.Variant]// A variant may pin a finer-grained compile_kwd per row via Meta// (structure stamps the inferred compile kind — list/set/hypergraph —@@ -503,16 +495,10 @@ func productsToChunkDocs(products []common.Product) ([]schema.ChunkDoc, error) {return nil, err}}- // Per-variant fine-grained columns (conf/infinity_mapping.json §45–77).+ // Per-variant fine-grained columns (conf/infinity_mapping.json §45–97).if err := applyVariantColumns(&doc, p); err != nil {return nil, err}- // Preserve the raw Product.Meta under kc_* for round-trip fidelity.- for k, v := range p.Meta {- if err := doc.SetExtraValue("kc_"+k, v); err != nil {- return nil, err- }- }docs = append(docs, doc)}return docs, nil@@ -548,9 +534,6 @@ func applyVariantColumns(doc *schema.ChunkDoc, p common.Product) error {if err := doc.SetExtraValue("slug_kwd", fullSlug); err != nil {return err}- if err := doc.SetExtraValue("artifact_slug_kwd", fullSlug); err != nil {- return err- }}}if v := metaString(p.Meta, "title"); v != "" {@@ -574,18 +557,17 @@ func applyVariantColumns(doc *schema.ChunkDoc, p common.Product) error {return err}}- // Section rows also carry level/index so a retriever can scope to a- // sub-section of a wiki page.- if v, ok := metaInt(p.Meta, "section_level"); ok {- if err := doc.SetExtraValue("section_level_int", v); err != nil {- return err- }- if err := doc.SetExtraValue("depth_int", v); err != nil {+ // md_with_weight is the page-body column Python writes with the same+ // value as content_with_weight (wiki_incremental.py:2190-2191), i.e. the+ // rendered body; section rows have no page body of their own.+ if metaString(p.Meta, "kind") == "page" && p.Content != "" {+ if err := doc.SetExtraValue("md_with_weight", p.Content); err != nil {return err}}- if v, ok := metaInt(p.Meta, "section_index"); ok {- if err := doc.SetExtraValue("section_index_int", v); err != nil {+ // Section depth goes to depth_int; there is no section_level_int column.+ if v, ok := metaInt(p.Meta, "section_level"); ok {+ if err := doc.SetExtraValue("depth_int", v); err != nil {return err}}@@ -732,20 +714,15 @@ func metaString(m map[string]any, key string) string {return v}-// setTitleTokens populates the ChunkDoc title_tks / title_sm_tks fields from a-// title string, mirroring how the chunker/tokenizer components tokenize titles-// (coarse → TitleTks, fine-grained → TitleSmTks). Errors are ignored: when the-// tokenizer pool is uninitialised (no-CGo test path) the fields stay empty,-// matching the chunker's graceful-degrade behaviour.+// setTitleTokens populates ChunkDoc.title_tks. Python's page row has no+// title_sm_tks, and the Infinity writer folds title_kwd / title_sm_tks into+// `docnm`, so the twin would make docnm map-order dependent.func setTitleTokens(doc *schema.ChunkDoc, title string) {if title == "" {return}if tks, err := tokenizer.Tokenize(title); err == nil && tks != "" {doc.TitleTks = tks- if sm, err := tokenizer.FineGrainedTokenize(tks); err == nil && sm != "" {- doc.TitleSmTks = sm- }}}diff --git a/internal/ingestion/component/knowledge_compiler/component_test.go b/internal/ingestion/component/knowledge_compiler/component_test.goindex ea7d31905f..58a8807d5c 100644--- a/internal/ingestion/component/knowledge_compiler/component_test.go+++ b/internal/ingestion/component/knowledge_compiler/component_test.go@@ -229,7 +229,8 @@ func TestKnowledgeCompiler_Structure_EndToEnd(t *testing.T) {if !ok {continue}- if kind, _ := cm["kc_kind"].(string); kind == "graph" {+ // knowledge_graph_kwd discriminates the graph row.+ if kind, _ := cm["knowledge_graph_kwd"].(string); kind == "graph" {graphChunks++}}@@ -349,10 +350,10 @@ func runVariant(t *testing.T, variant string, extra map[string]any) []map[stringfunc TestKnowledgeCompiler_Wiki_EndToEnd(t *testing.T) {installProseDeps(t)chunks := runVariant(t, "wiki", nil)- // wiki produces a "page" chunk (kind stored as kc_kind).+ // wiki produces a "page" chunk (kind carried by compile_kwd=wiki_page).foundPage := falsefor _, c := range chunks {- if kind, _ := c["kc_kind"].(string); kind == "page" {+ if kind, _ := c["compile_kwd"].(string); kind == "wiki_page" {foundPage = true}}@@ -430,7 +431,7 @@ func invokeWikiCompiler(t *testing.T, inputs map[string]any) []map[string]any {func hasWikiPageChunk(chunks []map[string]any) bool {for _, cm := range chunks {- if kind, _ := cm["kc_kind"].(string); kind == "page" {+ if kind, _ := cm["compile_kwd"].(string); kind == "wiki_page" {return true}}@@ -535,7 +536,7 @@ func TestKnowledgeCompiler_Tree_EndToEnd(t *testing.T) {chunks := runVariant(t, "tree", nil)foundRoot := falsefor _, c := range chunks {- if kind, _ := c["kc_kind"].(string); kind == "root" {+ if kind, _ := c["raptor_kwd"].(string); kind == "root" {foundRoot = true}}@@ -551,7 +552,7 @@ func TestKnowledgeCompiler_Tree_EndToEnd(t *testing.T) {}foundRootCoarse := falsefor _, c := range chunksCoarse {- if kind, _ := c["kc_kind"].(string); kind == "root" {+ if kind, _ := c["raptor_kwd"].(string); kind == "root" {foundRootCoarse = true}}@@ -570,7 +571,7 @@ func TestKnowledgeCompiler_Mindmap_EndToEnd(t *testing.T) {chunks := runVariant(t, "mindmap", nil)entityCount := 0for _, c := range chunks {- kind, _ := c["kc_kind"].(string)+ kind, _ := c["knowledge_graph_kwd"].(string)if kind == "entity" {entityCount++if _, ok := c["name_kwd"]; !ok {@@ -764,7 +765,7 @@ func TestKnowledgeCompiler_Tree_DegenerateNoInfiniteLoop(t *testing.T) {foundRoot := falsefor _, r := range raw {if cm, ok := r.(map[string]any); ok {- if k, _ := cm["kc_kind"].(string); k == "root" {+ if k, _ := cm["raptor_kwd"].(string); k == "root" {foundRoot = true}}@@ -876,7 +877,7 @@ func TestKnowledgeCompiler_Wiki_UpdateMergesExistingPage(t *testing.T) {if !ok {continue}- if cm["compile_kwd"] == "wiki_page" && cm["kc_kind"] == "page" && cm["slug_kwd"] == "entity/person/alpha" {+ if cm["compile_kwd"] == "wiki_page" && cm["slug_kwd"] == "entity/person/alpha" {page = cmbreak}@@ -1441,9 +1442,8 @@ func TestKnowledgeCompiler_BuildInputsAcceptsMapSliceChunks(t *testing.T) {}// TestProductsToChunkDocs_PageVsSectionCompileKWD locks the page/section-// discriminator: a wiki page product is stamped compile_kwd="wiki_page" and a-// wiki section product compile_kwd="wiki_section", so a page search on-// compile_kwd="wiki_page" (engine_service / kcWikiPageStore) returns pages only.+// discriminator (compile_kwd) and the schema-column contract: the page body goes+// to md_with_weight and no Go-only field (kc_*, tenant_id) is emitted.func TestProductsToChunkDocs_PageVsSectionCompileKWD(t *testing.T) {page := common.Product{ID: "page-id", DocID: "d1", TenantID: "t1", Variant: common.VariantWiki,@@ -1460,17 +1460,33 @@ func TestProductsToChunkDocs_PageVsSectionCompileKWD(t *testing.T) {t.Fatalf("productsToChunkDocs: %v", err)}var pageKWD, sectionKWD string- var sectionParent string+ var sectionParent, pageBody, sectionBody stringfor _, d := range docs {- // Product.Meta is preserved under the kc_* round-trip keys; the page/- // section kind lives at "kc_kind".- kind, _ := d.GetExtraString("kc_kind")- if kind == "page" {- pageKWD, _ = d.GetExtraString("compile_kwd")+ // compile_kwd IS the page/section discriminator (wiki_page /+ // wiki_section); there is no separate kind column.+ kind, _ := d.GetExtraString("compile_kwd")+ if kind == "wiki_page" {+ pageKWD = kind+ pageBody, _ = d.GetExtraString("md_with_weight")+ // Python's page row has only title_tks; the Infinity writer folds+ // title_sm_tks into docnm.+ if _, ok := d.ToMap()["title_sm_tks"]; ok {+ t.Errorf("page row must not carry title_sm_tks (docnm would become map-order dependent)")+ }}- if kind == "section" {- sectionKWD, _ = d.GetExtraString("compile_kwd")+ if kind == "wiki_section" {+ sectionKWD = kindsectionParent, _ = d.GetExtraString("parent_kwd")+ sectionBody, _ = d.GetExtraString("md_with_weight")+ }+ // No Go-only field: Infinity rejects an unknown column.+ for k := range d.Extra {+ if strings.HasPrefix(k, "kc_") {+ t.Errorf("compiled row carries the non-schema field %q", k)+ }+ }+ if _, ok := d.GetExtraString("tenant_id"); ok {+ t.Errorf("compiled row must not carry tenant_id (not a chunk column)")}}if pageKWD != "wiki_page" {@@ -1482,6 +1498,12 @@ func TestProductsToChunkDocs_PageVsSectionCompileKWD(t *testing.T) {if sectionParent != "page-id" {t.Errorf("section parent_kwd = %q, want page-id", sectionParent)}+ if pageBody != "# Alpha\n\nBody" {+ t.Errorf("page md_with_weight = %q, want the rendered page body", pageBody)+ }+ if sectionBody != "" {+ t.Errorf("section md_with_weight = %q, want empty (only page rows carry the page body)", sectionBody)+ }}// TestMain installs the stub resolvers for the variant unit tests.diff --git a/internal/ingestion/component/knowledge_compiler/golden/golden_test.go b/internal/ingestion/component/knowledge_compiler/golden/golden_test.goindex 8c55674021..5e1d4a522e 100644--- a/internal/ingestion/component/knowledge_compiler/golden/golden_test.go+++ b/internal/ingestion/component/knowledge_compiler/golden/golden_test.go@@ -48,20 +48,20 @@ func TestAnalyzeTreeProducts_TreeShape(t *testing.T) {vector := json.RawMessage(`[0.1,0.2,0.3]`)chunks := []schema.ChunkDoc{{Text: "root summary", Extra: mustExtras(t, map[string]any{- "id": "r1", "doc_id": "d1", "tenant_id": "t1", "compile_kwd": "tree",- "kc_kind": "root", "kc_level": float64(-1), "q_3_vec": vector,+ "id": "r1", "doc_id": "d1", "compile_kwd": "tree",+ "raptor_kwd": "root", "raptor_layer_int": float64(-1), "q_3_vec": vector,})},{Text: "leaf A", Extra: mustExtras(t, map[string]any{- "id": "a1", "doc_id": "d1", "tenant_id": "t1", "compile_kwd": "tree",- "kc_kind": "summary", "kc_level": float64(0), "parent_kwd": "r1", "q_3_vec": vector,+ "id": "a1", "doc_id": "d1", "compile_kwd": "tree",+ "raptor_kwd": "summary", "raptor_layer_int": float64(0), "parent_kwd": "r1", "q_3_vec": vector,})},{Text: "leaf B", Extra: mustExtras(t, map[string]any{- "id": "b1", "doc_id": "d1", "tenant_id": "t1", "compile_kwd": "tree",- "kc_kind": "summary", "kc_level": float64(0), "parent_kwd": "r1", "q_3_vec": vector,+ "id": "b1", "doc_id": "d1", "compile_kwd": "tree",+ "raptor_kwd": "summary", "raptor_layer_int": float64(0), "parent_kwd": "r1", "q_3_vec": vector,})},{Text: "mid A", Extra: mustExtras(t, map[string]any{- "id": "m1", "doc_id": "d1", "tenant_id": "t1", "compile_kwd": "tree",- "kc_kind": "summary", "kc_level": float64(1), "parent_kwd": "r1", "q_3_vec": vector,+ "id": "m1", "doc_id": "d1", "compile_kwd": "tree",+ "raptor_kwd": "summary", "raptor_layer_int": float64(1), "parent_kwd": "r1", "q_3_vec": vector,})},}m := AnalyzeTreeProducts(chunks)@@ -85,12 +85,12 @@ func TestAnalyzeTreeProducts_TreeShape(t *testing.T) {func TestAnalyzeTreeProducts_DetectsDanglingParent(t *testing.T) {chunks := []schema.ChunkDoc{{Text: "root", Extra: mustExtras(t, map[string]any{- "id": "r1", "doc_id": "d1", "tenant_id": "t1", "compile_kwd": "tree",- "kc_kind": "root",+ "id": "r1", "doc_id": "d1", "compile_kwd": "tree",+ "raptor_kwd": "root",})},{Text: "orphan", Extra: mustExtras(t, map[string]any{- "id": "o1", "doc_id": "d1", "tenant_id": "t1", "compile_kwd": "tree",- "kc_kind": "summary", "kc_level": float64(0), "parent_kwd": "missing-parent",+ "id": "o1", "doc_id": "d1", "compile_kwd": "tree",+ "raptor_kwd": "summary", "raptor_layer_int": float64(0), "parent_kwd": "missing-parent",})},}m := AnalyzeTreeProducts(chunks)@@ -134,12 +134,12 @@ func TestAnalyzeTreeProducts_IgnoresUnknownSourceIDs(t *testing.T) {vector := json.RawMessage(`[0.1,0.2,0.3]`)chunks := []schema.ChunkDoc{{Text: "root", Extra: mustExtras(t, map[string]any{- "id": "r1", "doc_id": "d1", "tenant_id": "t1", "compile_kwd": "tree",- "kc_kind": "root", "kc_level": float64(-1), "q_3_vec": vector,+ "id": "r1", "doc_id": "d1", "compile_kwd": "tree",+ "raptor_kwd": "root", "raptor_layer_int": float64(-1), "q_3_vec": vector,})},{Text: "leaf", Extra: mustExtras(t, map[string]any{- "id": "a1", "doc_id": "d1", "tenant_id": "t1", "compile_kwd": "tree",- "kc_kind": "summary", "kc_level": float64(0), "parent_kwd": "r1", "q_3_vec": vector,+ "id": "a1", "doc_id": "d1", "compile_kwd": "tree",+ "raptor_kwd": "summary", "raptor_layer_int": float64(0), "parent_kwd": "r1", "q_3_vec": vector,"source_chunk_ids": []string{"chunk-01", "chunk-02", "leaked-unknown-id"},})},}@@ -153,8 +153,8 @@ func TestAnalyzeTreeProducts_IgnoresUnknownSourceIDs(t *testing.T) {}func TestExtraFloat_Roundtrip(t *testing.T) {- doc := schema.ChunkDoc{Extra: mustExtras(t, map[string]any{"kc_level": float64(3)})}- if v, ok := extraFloat(doc, "kc_level"); !ok || v != 3 {+ doc := schema.ChunkDoc{Extra: mustExtras(t, map[string]any{"raptor_layer_int": float64(3)})}+ if v, ok := extraFloat(doc, "raptor_layer_int"); !ok || v != 3 {t.Errorf("extraFloat = %v/%v, want 3/true", v, ok)}if _, ok := extraFloat(doc, "missing"); ok {diff --git a/internal/ingestion/component/knowledge_compiler/golden/metrics.go b/internal/ingestion/component/knowledge_compiler/golden/metrics.goindex 27224bb628..02bec14b27 100644--- a/internal/ingestion/component/knowledge_compiler/golden/metrics.go+++ b/internal/ingestion/component/knowledge_compiler/golden/metrics.go@@ -50,7 +50,8 @@ func AnalyzeTreeProducts(chunks []schema.ChunkDoc, validSourceIDs ...string) Trem := TreeMetrics{AllParented: true, VectorOK: true, SchemaOK: true, covered: make(map[string]bool)}maxLevel := -1for _, c := range chunks {- kind, _ := c.GetExtraString("kc_kind")+ // raptor_kwd / raptor_layer_int are the tree row's kind and depth.+ kind, _ := c.GetExtraString("raptor_kwd")// Auxiliary tree-graph rows (kind entity/relation/graph, from the// structure-graph projection) are not part of the RAPTOR tree structure,// so they are excluded from the structural metrics (ProductCount,@@ -60,7 +61,7 @@ func AnalyzeTreeProducts(chunks []schema.ChunkDoc, validSourceIDs ...string) Tre}m.ProductCount++level := 0- if lf, ok := extraFloat(c, "kc_level"); ok {+ if lf, ok := extraFloat(c, "raptor_layer_int"); ok {level = int(lf)}switch kind {@@ -102,9 +103,8 @@ func AnalyzeTreeProducts(chunks []schema.ChunkDoc, validSourceIDs ...string) Tre}id, _ := c.GetExtraString("id")docID, _ := c.GetExtraString("doc_id")- tenant, _ := c.GetExtraString("tenant_id")ck, _ := c.GetExtraString("compile_kwd")- if id == "" || docID == "" || tenant == "" || c.Text == "" || ck == "" {+ if id == "" || docID == "" || c.Text == "" || ck == "" {m.SchemaOK = false}}diff --git a/internal/ingestion/component/knowledge_compiler/golden_test.go b/internal/ingestion/component/knowledge_compiler/golden_test.goindex 544ab7cfaa..cbde245b01 100644--- a/internal/ingestion/component/knowledge_compiler/golden_test.go+++ b/internal/ingestion/component/knowledge_compiler/golden_test.go@@ -153,8 +153,7 @@ func TestGolden_Structure_ProductCount(t *testing.T) {for _, p := range prods {id, _ := p.GetExtraString("id")docID, _ := p.GetExtraString("doc_id")- tenant, _ := p.GetExtraString("tenant_id")- if id == "" || docID == "" || tenant == "" || p.Text == "" {+ if id == "" || docID == "" || p.Text == "" {t.Fatalf("structure: product missing schema fields: %+v", p)}if !chunkHasVector(p) {@@ -166,7 +165,8 @@ func TestGolden_Structure_ProductCount(t *testing.T) {// compact graph blob was removed by #19474).graphCount, nodeCount, edgeCount := 0, 0, 0for _, p := range prods {- kind, _ := p.GetExtraString("kc_kind")+ // knowledge_graph_kwd discriminates the structure graph rows.+ kind, _ := p.GetExtraString("knowledge_graph_kwd")switch kind {case "graph":graphCount++@@ -199,8 +199,7 @@ func TestGolden_Wiki_ProductCount(t *testing.T) {for _, p := range prods {id, _ := p.GetExtraString("id")docID, _ := p.GetExtraString("doc_id")- tenant, _ := p.GetExtraString("tenant_id")- if id == "" || docID == "" || tenant == "" || p.Text == "" {+ if id == "" || docID == "" || p.Text == "" {t.Fatalf("wiki: product missing schema fields: %+v", p)}if !chunkHasVector(p) {@@ -210,7 +209,8 @@ func TestGolden_Wiki_ProductCount(t *testing.T) {foundPage := falsefor _, p := range prods {- if kind, _ := p.GetExtraString("kc_kind"); kind == "page" {+ // compile_kwd=wiki_page is the page/section discriminator.+ if kwd, _ := p.GetExtraString("compile_kwd"); kwd == "wiki_page" {foundPage = true}}diff --git a/internal/ingestion/knowledge_compile/reader.go b/internal/ingestion/knowledge_compile/reader.goindex 43c2a4df9c..81123c81f5 100644--- a/internal/ingestion/knowledge_compile/reader.go+++ b/internal/ingestion/knowledge_compile/reader.go@@ -97,24 +97,17 @@ type engineReader struct {eng engine.DocEngine}-// compiledSelectFields are the columns needed to reconstruct a Product from a-// stored compiled chunk document.-//-// wiki_incremental port: the list also selects `kc_kind` and-// `create_timestamp_flt` (+`create_time`) so the reader can round-trip the-// wiki product kind (page/section) and the original creation timestamp without-// re-deriving them (see productFromChunkMap). Without these in the SELECT list,-// the stored values are invisible to the reader and every merged row would fall-// back to the compile_kwd-derived kind / a fresh now() timestamp — which both-// breaks the page/section filter and re-stamps creation time on every rebuild.+// compiledSelectFields are the columns needed to reconstruct a Product. Every+// name must exist in the target engine schema (Infinity rejects a SELECT of an+// unknown column); payload = content_with_weight, kind = compile_kwd.var compiledSelectFields = []string{- "id", "doc_id", "tenant_id", "compile_kwd",+ "id", "doc_id", "compile_kwd","available_int",- "content_with_weight", "kc_payload",+ "content_with_weight","source_chunk_ids", "source_doc_ids","name_kwd", "entity_type_kwd", "from_entity_kwd", "to_entity_kwd",- "slug_kwd", "type",- "kc_kind", "create_timestamp_flt", "create_time",+ "slug_kwd",+ "create_timestamp_flt", "create_time","compilation_template_kind_kwd", "compilation_template_ids",// The product kind discriminator for structure/tree: the component stores it// under knowledge_graph_kwd (structure: graph/entity/relation) / raptor_kwd@@ -141,7 +134,7 @@ var compiledSelectFields = []string{var wikiSelectFields = []string{"page_type_kwd", "topic_kwd", "plan_group_kwd", "generation_kwd", "title_kwd","entity_names_kwd", "summary_with_weight",- "related_kb_pages_kwd", "outlinks_kwd", "section_level_int",+ "related_kb_pages_kwd", "outlinks_kwd", "depth_int","q_*_vec",}@@ -331,8 +324,8 @@ func (r engineReader) LoadDocumentWikiPagesBySlugs(ctx context.Context, tenant,}// productFromChunkMap reconstructs a kccommon.Product from a stored compiled-// chunk document. It reads the payload from kc_payload (falling back to-// content_with_weight) and the embedding from the q_<dim>_vec column.+// chunk document. It reads the payload from content_with_weight and the+// embedding from the q_<dim>_vec column.//// expect is the variant the caller is querying for. The stored compile_kwd is// reverse-mapped via KwdToVariant and compared against expect; a mismatch (or@@ -341,10 +334,7 @@ func (r engineReader) LoadDocumentWikiPagesBySlugs(ctx context.Context, tenant,// the plan: we never rely on the raw string equality alone, so unknown kinds// are rejected consistently rather than leaking into the wrong bucket.func productFromChunkMap(c map[string]interface{}, tenant string, expect kccommon.Variant) (kccommon.Product, bool) {- content, _ := c["kc_payload"].(string)- if content == "" {- content, _ = c["content_with_weight"].(string)- }+ content, _ := c["content_with_weight"].(string)if content == "" {return kccommon.Product{}, false}@@ -421,11 +411,13 @@ func productFromChunkMap(c map[string]interface{}, tenant string, expect kccommoif v := metaStringSlice(c, "outlinks_kwd"); len(v) > 0 {meta["outlinks"] = v}- if v, ok := metaInt(c, "section_level_int"); ok {- meta["section_level"] = v- }- if v, ok := c["type"].(string); ok && v != "" {- meta["type"] = v+ // Section depth comes from depth_int. Only the wiki variant consumes+ // Meta["section_level"] (applyVariantColumns); tree rows also stamp depth_int,+ // but they keep their depth in raptor_layer_int.+ if expect == kccommon.VariantWiki {+ if v, ok := metaInt(c, "depth_int"); ok {+ meta["section_level"] = v+ }}// Restore the structure/tree product kind. The component stores it under// knowledge_graph_kwd (structure: graph/entity/relation) / raptor_kwd (tree:@@ -442,16 +434,10 @@ func productFromChunkMap(c map[string]interface{}, tenant string, expect kccommometa["kind"] = "entity"}}- // wiki_incremental port: round-trip the wiki product kind so the- // dataset-level merge can reliably distinguish pages from sections. The- // merged writer stores kc_kind; when present it is authoritative. Without- // it (legacy rows), derive from compile_kwd: wiki_page -> "page",- // wiki_section -> "section". This fix is what stops the processBatch- // "Meta.kind==page" filter from deleting every wiki page (previously kind- // was empty for wiki pages that had no entity/relation endpoint).- if v := asString(c["kc_kind"]); v != "" {- meta["kind"] = v- } else if variant == compileKwdWikiPage {+ // compile_kwd IS the page/section discriminator (there is no kind column):+ // wiki_page -> "page", wiki_section -> "section". Without it the+ // processBatch "Meta.kind==page" filter would drop every wiki page.+ if variant == compileKwdWikiPage {meta["kind"] = "page"} else if variant == compileKwdWikiSection {meta["kind"] = "section"@@ -525,10 +511,10 @@ func (r engineReader) SearchSimilar(ctx context.Context, tenant, kb string, variIndexNames: []string{fmt.Sprintf("ragflow_%s", tenant)},KbIDs: []string{kb},Limit: topN,- SelectFields: append([]string{"id", "doc_id", "kb_id", "content_with_weight", "kc_payload",+ SelectFields: append([]string{"id", "doc_id", "kb_id", "content_with_weight","name_kwd", "entity_type_kwd", "from_entity_kwd", "to_entity_kwd", "slug_kwd",- "type", "source_chunk_ids", "source_doc_ids", "available_int", "compile_kwd",- "kc_kind", "create_timestamp_flt", "create_time"},+ "source_chunk_ids", "source_doc_ids", "available_int", "compile_kwd",+ "create_timestamp_flt", "create_time"},wikiSelectFields...),Filter: map[string]interface{}{"available_int": 1,diff --git a/internal/ingestion/knowledge_compile/reader_test.go b/internal/ingestion/knowledge_compile/reader_test.goindex 17da6c06e3..ae62dffa0a 100644--- a/internal/ingestion/knowledge_compile/reader_test.go+++ b/internal/ingestion/knowledge_compile/reader_test.go@@ -51,7 +51,7 @@ func TestWikiReadersFilterDisabledDocuments(t *testing.T) {eng := &fakeEngine{searchChunks: []map[string]interface{}{{"id": "page-1", "doc_id": "active-doc", "compile_kwd": "wiki_page",- "available_int": 0, "kc_payload": "page content", "slug_kwd": "entity/page",+ "available_int": 0, "content_with_weight": "page content", "slug_kwd": "entity/page","page_type_kwd": "entity", "title_kwd": "page", "source_doc_ids": []string{"active-doc"},}}}r := engineReader{eng: eng}@@ -82,21 +82,20 @@ func TestSearchSimilarFiltersByVariant(t *testing.T) {searchChunks: []map[string]interface{}{// dirty row: wrong variant (must be skipped by the in-memory guard).{- "id": "dirty",- "doc_id": "kb",- "available_int": 1,- "compile_kwd": "wiki_section", // not wiki_page- "kc_payload": "{\"c\":1}",- "_score": 0.99,+ "id": "dirty",+ "doc_id": "kb",+ "available_int": 1,+ "compile_kwd": "wiki_section", // not wiki_page+ "content_with_weight": "{\"c\":1}",+ "_score": 0.99,},- // good row: correct variant + kc_kind round-trip.+ // good row: correct variant (wiki_page => kind "page").{"id": "page1","doc_id": "kb","available_int": 1,"compile_kwd": "wiki_page",- "kc_kind": "page",- "kc_payload": "{\"c\":1}",+ "content_with_weight": "{\"c\":1}","create_timestamp_flt": 1700000000.0,"create_time": "2023-11-14T22:13:20Z","_score": 0.95,@@ -113,7 +112,7 @@ func TestSearchSimilarFiltersByVariant(t *testing.T) {t.Fatalf("expected the wiki_page row to win, got %q", p.ID)}if p.Meta["kind"] != "page" {- t.Fatalf("expected kc_kind round-trip to 'page', got %v", p.Meta["kind"])+ t.Fatalf("expected compile_kwd=wiki_page to map to kind 'page', got %v", p.Meta["kind"])}if p.Merged != true {t.Fatalf("expected merged=true for available_int=1 row")@@ -136,7 +135,7 @@ func TestSearchSimilarFiltersByVariant(t *testing.T) {t.Fatalf("expected compile_kwd=%q filter, got %v", compileKwdWikiPage, eng.lastSearchReq.Filter["compile_kwd"])}// SelectFields must include the round-trip columns added in the 8th review.- for _, want := range []string{"kc_kind", "create_timestamp_flt", "create_time"} {+ for _, want := range []string{"compile_kwd", "content_with_weight", "create_timestamp_flt", "create_time"} {found := falsefor _, f := range eng.lastSearchReq.SelectFields {if f == want {@@ -156,12 +155,12 @@ func TestSearchSimilarSkipsNonMerged(t *testing.T) {eng := &fakeEngine{searchChunks: []map[string]interface{}{{- "id": "doc1",- "doc_id": "doc",- "available_int": 0,- "compile_kwd": "wiki_page",- "kc_payload": "{\"c\":1}",- "_score": 0.99,+ "id": "doc1",+ "doc_id": "doc",+ "available_int": 0,+ "compile_kwd": "wiki_page",+ "content_with_weight": "{\"c\":1}",+ "_score": 0.99,},},}diff --git a/internal/ingestion/knowledge_compile/structure_merge_test.go b/internal/ingestion/knowledge_compile/structure_merge_test.goindex 586b73ef06..8da59fe663 100644--- a/internal/ingestion/knowledge_compile/structure_merge_test.go+++ b/internal/ingestion/knowledge_compile/structure_merge_test.go@@ -338,7 +338,7 @@ func TestProductFromChunkMap_RestoresStructureTreeKind(t *testing.T) {"id": "s1","doc_id": "d1","compile_kwd": "list",- "kc_payload": `{"entities":[]}`,+ "content_with_weight": `{"entities":[]}`,"knowledge_graph_kwd": "graph","name_kwd": "engine","source_chunk_ids": []interface{}{"c1"},@@ -363,7 +363,7 @@ func TestProductFromChunkMap_RestoresStructureTreeKind(t *testing.T) {"id": "t1","doc_id": "d1","compile_kwd": "tree",- "kc_payload": "overall summary",+ "content_with_weight": "overall summary","raptor_kwd": "root","source_chunk_ids": []interface{}{"c1"},"source_doc_ids": []interface{}{"d1"},diff --git a/internal/ingestion/knowledge_compile/wiki_contribution.go b/internal/ingestion/knowledge_compile/wiki_contribution.goindex ebca83b223..ff3ba9fa8e 100644--- a/internal/ingestion/knowledge_compile/wiki_contribution.go+++ b/internal/ingestion/knowledge_compile/wiki_contribution.go@@ -99,7 +99,6 @@ func (s *engineWikiContributionStore) Put(ctx context.Context, tenantID, datasetrow := map[string]any{"id": wikiContributionStateID(datasetID, contribution.DocumentID),"doc_id": "wiki_contribution:" + contribution.DocumentID,- "tenant_id": tenantID,"kb_id": datasetID,"compile_kwd": wikiContributionCompileKWD,"scope_kwd": "doc",diff --git a/internal/ingestion/knowledge_compile/wiki_graph_test.go b/internal/ingestion/knowledge_compile/wiki_graph_test.goindex deb3c537e8..fe0d516a88 100644--- a/internal/ingestion/knowledge_compile/wiki_graph_test.go+++ b/internal/ingestion/knowledge_compile/wiki_graph_test.go@@ -65,12 +65,20 @@ func TestProjectWikiGraphRowsWritesSearchContent(t *testing.T) {if entity == nil {t.Fatal("missing entity")}- for _, field := range []string{"title_tks", "title_sm_tks", "content_ltks", "content_sm_ltks"} {+ // Python tokenizes slug + description into content_ltks and writes no title+ // token column on the entity row (dataset_wiki_generator.py:750-773); the+ // row's title stays in title_kwd / the payload.+ for _, field := range []string{"content_ltks", "content_sm_ltks"} {value, ok := entity[field].(string)if !ok || strings.TrimSpace(value) == "" {t.Fatalf("entity[%q] = %#v, want non-empty search content", field, entity[field])}}+ for _, field := range []string{"title_tks", "title_sm_tks"} {+ if _, ok := entity[field]; ok {+ t.Fatalf("entity row must not carry %q (Python writes no title token column)", field)+ }+ }}// TestProjectWikiGraphRowsCrossBatchEdgeSurvives verifies a relation whose two@@ -217,9 +225,9 @@ func TestProjectWikiGraphRowsDropsDanglingAndSelfLoop(t *testing.T) {}}-// TestProjectWikiGraphRowsFieldMapping verifies the reader-facing display-// columns are written (aliases_kwd from entity_names_kwd, description_with_weight-// from summary_with_weight), and the id is the xxhash colon-namespaced form.+// TestProjectWikiGraphRowsFieldMapping verifies the entity row matches Python's+// shape (dataset_wiki_generator.py:759-776): display fields in the+// content_with_weight payload, schema columns only, xxhash id.func TestProjectWikiGraphRowsFieldMapping(t *testing.T) {w := engineWriter{}pages := []wikiPageProjection{@@ -244,12 +252,6 @@ func TestProjectWikiGraphRowsFieldMapping(t *testing.T) {if e == nil {t.Fatalf("missing entity")}- if e["aliases_kwd"] == nil {- t.Fatalf("expected aliases_kwd (mapped from entity_names_kwd) to be set")- }- if e["description_with_weight"] != "the alpha page" {- t.Fatalf("expected description_with_weight=summary, got %v", e["description_with_weight"])- }if e["entity_type_kwd"] != "wiki_concept" {t.Fatalf("expected entity_type_kwd=wiki_concept, got %v", e["entity_type_kwd"])}@@ -263,14 +265,27 @@ func TestProjectWikiGraphRowsFieldMapping(t *testing.T) {if id != wantID {t.Fatalf("id mismatch: got %q want %q", id, wantID)}- // content_with_weight JSON must round-trip the slug/page_type/title/aliases/summary/weight.+ // content_with_weight JSON is the canvas payload (Python keys), the+ // authoritative source for name/aliases/description/type.raw, _ := e["content_with_weight"].(string)var decoded map[string]anyif err := json.Unmarshal([]byte(raw), &decoded); err != nil {t.Fatalf("content_with_weight not valid JSON: %v", err)}- if decoded["slug"] != "concept/alpha" || decoded["page_type"] != "concept" {- t.Fatalf("content_with_weight missing slug/page_type: %v", decoded)+ if decoded["slug"] != "concept/alpha" || decoded["type"] != "concept" {+ t.Fatalf("content_with_weight missing slug/type: %v", decoded)+ }+ if decoded["name"] != "Alpha" || decoded["description"] != "the alpha page" {+ t.Fatalf("content_with_weight missing name/description: %v", decoded)+ }+ if aliases, _ := decoded["aliases"].([]any); len(aliases) != 2 || aliases[0] != "A" {+ t.Fatalf("content_with_weight aliases = %v, want [A Alpha Prime]", decoded["aliases"])+ }+ // No Go-only field: none of these exists in an engine mapping.+ for _, forbidden := range []string{"aliases_kwd", "description_with_weight", "tenant_id", "from_id", "to_id"} {+ if _, ok := e[forbidden]; ok {+ t.Fatalf("entity row must not carry the non-schema field %q", forbidden)+ }}}diff --git a/internal/ingestion/knowledge_compile/wiki_map_store.go b/internal/ingestion/knowledge_compile/wiki_map_store.goindex 8ac6bc9bd3..6d72e3e843 100644--- a/internal/ingestion/knowledge_compile/wiki_map_store.go+++ b/internal/ingestion/knowledge_compile/wiki_map_store.go@@ -76,7 +76,6 @@ func (s *wikiMapVersionStore) PutWikiMapActiveState(ctx context.Context, state krow := map[string]interface{}{"id": state.Key,"doc_id": "wiki_map_active:" + state.DocumentID,- "tenant_id": state.TenantID,"kb_id": state.DatasetID,"compile_kwd": wikiMapActiveCompileKWD,"scope_kwd": "doc",@@ -88,13 +87,6 @@ func (s *wikiMapVersionStore) PutWikiMapActiveState(ctx context.Context, state kreturn err}-// NewWikiMapVersionStore returns the DocStore-backed immutable Wiki MAP cache.-// Its rows remain non-searchable through available_int=0 and the compile-// discriminator, while preserving every chunk/hash version for reuse.-func NewWikiMapVersionStore(docEngine engine.DocEngine) kccommon.WikiMapVersionStore {- return &wikiMapVersionStore{engine: docEngine}-}-// NewWikiMapVersionStoreWithVectorSizeResolver creates a Wiki MAP store that// can initialize an Infinity chunk table before inserting metadata-only MAP// rows. Elasticsearch does not need this resolver because it can create its@@ -236,7 +228,6 @@ func wikiMapVersionRow(version kccommon.WikiMapVersion) map[string]interface{} {// Keep immutable MAP history in a separate document namespace so source// document deletion cannot remove a reusable chunk/hash version."doc_id": wikiMapCacheDocID(version.DocumentID),- "tenant_id": version.TenantID,"kb_id": version.DatasetID,"compile_kwd": wikiMapExtractCompileKWD,"scope_kwd": "doc",diff --git a/internal/ingestion/knowledge_compile/wiki_map_store_test.go b/internal/ingestion/knowledge_compile/wiki_map_store_test.goindex 6b8d521f9e..1d777c231f 100644--- a/internal/ingestion/knowledge_compile/wiki_map_store_test.go+++ b/internal/ingestion/knowledge_compile/wiki_map_store_test.go@@ -120,7 +120,7 @@ func (e *wikiMapStoreEngine) InsertChunks(_ context.Context, chunks []map[stringfunc TestWikiMapVersionStoreUsesNonSearchableDocStoreRows(t *testing.T) {engine := &wikiMapStoreEngine{rows: map[string]map[string]interface{}{}}- store := NewWikiMapVersionStore(engine)+ store := NewWikiMapVersionStoreWithVectorSizeResolver(engine, nil)version := kccommon.WikiMapVersion{Key: "version-a",TenantID: "tenant-1",@@ -228,7 +228,7 @@ func TestWikiMapVersionStoreCreatesMissingInfinityStore(t *testing.T) {func TestWikiMapActiveStateRequiresResolverOnlyWhenInfinityStoreMissing(t *testing.T) {engine := &wikiMapStoreEngine{rows: map[string]map[string]interface{}{}, engineType: "infinity"}- store := NewWikiMapVersionStore(engine).(kccommon.WikiMapActiveStateStore)+ store := NewWikiMapVersionStoreWithVectorSizeResolver(engine, nil).(kccommon.WikiMapActiveStateStore)state := kccommon.WikiMapActiveState{Key: "active-a", TenantID: "tenant-1", DatasetID: "kb-1",DocumentID: "doc-1", Payload: []byte(`{}`),diff --git a/internal/ingestion/knowledge_compile/writer.go b/internal/ingestion/knowledge_compile/writer.goindex b5a13744c6..3efbae47c2 100644--- a/internal/ingestion/knowledge_compile/writer.go+++ b/internal/ingestion/knowledge_compile/writer.go@@ -538,13 +538,11 @@ func (w engineWriter) WriteMergedStructure(ctx context.Context, tenant, kb strin// relation identity plus the raw compile kind."id": bid,"doc_id": kb,- "tenant_id": tenant,"kb_id": kb,"available_int": 1,"compile_kwd": ckwd,"scope_kwd": "dataset","content_with_weight": payload,- "kc_payload": payload,"source_doc_ids": b.SourceDocIDs,"source_chunk_ids": b.SourceChunkIDs,"create_time": now.Format("2006-01-02 15:04:05"),@@ -601,7 +599,6 @@ func (w engineWriter) WriteMergedStructure(ctx context.Context, tenant, kb strinrows = append(rows, map[string]interface{}{"id": "dataset_build_meta_" + hashStr(tenant+"\x00"+kb+"\x00"+ckwd),"doc_id": kb,- "tenant_id": tenant,"kb_id": kb,"available_int": 0,"compile_kwd": ckwd,@@ -658,7 +655,7 @@ func mergeExistingStructureBuckets(ctx context.Context, eng engine.DocEngine, baSelectFields: []string{"id", "compile_kwd", "compilation_template_ids", "compilation_template_kind_kwd","knowledge_graph_kwd", "name_kwd", "entity_type_kwd", "from_entity_kwd", "to_entity_kwd",- "content_with_weight", "kc_payload", "source_doc_ids", "source_chunk_ids",+ "content_with_weight", "source_doc_ids", "source_chunk_ids",},Filter: map[string]interface{}{"kb_id": kb,@@ -765,11 +762,9 @@ func structureRowIdentity(row map[string]interface{}) string {}func structureRowPayload(row map[string]interface{}) map[string]interface{} {- for _, field := range []string{"kc_payload", "content_with_weight"} {- value := structureString(row[field])- if value == "" {- continue- }+ // The payload is content_with_weight verbatim (Python json.dumps's it there);+ // there is no separate payload column.+ if value := structureString(row["content_with_weight"]); value != "" {var payload map[string]interface{}if json.Unmarshal([]byte(value), &payload) == nil {return payload@@ -959,7 +954,6 @@ func mergedChunkMap(tenant, kb, runID, inputHash string, now time.Time, p kccommm := map[string]interface{}{"id": datasetLevelID(tenant, kb, p),"doc_id": kb,- "tenant_id": tenant,"kb_id": kb,"available_int": 1,// scope_kwd marks this row as dataset-level (O1=B). It is the unified@@ -969,7 +963,6 @@ func mergedChunkMap(tenant, kb, runID, inputHash string, now time.Time, p kccomm"scope_kwd": "dataset","compile_kwd": compileKwdForVariant(p.Variant),"content_with_weight": p.Content,- "kc_payload": p.Content, // raw payload, for Reader reconstruction"source_doc_ids": srcDocIDs,"source_chunk_ids": srcChunkIDs,"plan_kwd": runID,@@ -977,16 +970,9 @@ func mergedChunkMap(tenant, kb, runID, inputHash string, now time.Time, p kccomm"create_time": now.Format("2006-01-02 15:04:05"),"create_timestamp_flt": float64(now.Unix()),}- // wiki_incremental port: persist the product kind so the Reader can round-trip- // page vs section without re-deriving it from compile_kwd. The merged writer- // carries the authoritative kc_kind; legacy rows without it are derived in- // productFromChunkMap (compile_kwd wiki_page -> "page", wiki_section ->- // "section"). Without this, the dataset-level merge could not distinguish a- // wiki page from a section and the processBatch "Meta.kind==page" filter would- // be unreliable.- if kind := metaString(p.Meta, "kind"); kind != "" {- m["kc_kind"] = kind- }+ // The product kind (page vs section) is carried by compile_kwd+ // (wiki_page / wiki_section), which is what Python stores and what+ // productFromChunkMap derives Meta["kind"] from — there is no kind column.// wiki_incremental port: preserve the original creation timestamp across a// page merge. If the incoming merged product already carries// created_at_unix (restored by the Reader from create_timestamp_flt), reuse@@ -1015,7 +1001,6 @@ func mergedChunkMap(tenant, kb, runID, inputHash string, now time.Time, p kccommfullSlug = pageType + "/" + slug}m["slug_kwd"] = fullSlug- m["artifact_slug_kwd"] = fullSlug}if v := metaString(p.Meta, "title"); v != "" {m["title_kwd"] = v@@ -1035,6 +1020,11 @@ func mergedChunkMap(tenant, kb, runID, inputHash string, now time.Time, p kccommif v := metaString(p.Meta, "summary"); v != "" {m["summary_with_weight"] = v}+ // The merged wiki page keeps the page body in md_with_weight (the column+ // GetWikiPage / the artifact API read), as Python does; sections have none.+ if p.Variant == kccommon.VariantWiki && metaString(p.Meta, "kind") == "page" && p.Content != "" {+ m["md_with_weight"] = p.Content+ }if v := metaStringSlice(p.Meta, "entity_names"); len(v) > 0 {m["entity_names_kwd"] = v}@@ -1299,7 +1289,7 @@ func metaFloat(m map[string]any, key string) (float64, bool) {// KwdToVariant is the inverse of compileKwdForVariant: it maps a stored// compile_kwd back to its compiler Variant. Both wiki_page and wiki_section// map to VariantWiki (same product family); the page/section distinction is-// carried by the kc_kind field, not the variant. Returns an error for an+// carried by compile_kwd itself, not the variant. Returns an error for an// unknown kwd so callers can reject dirty/foreign rows. Structure products// stamp the inferred compile kind verbatim (list/set/hypergraph), which are NOT// in the KindToVariant whitelist, so they are mapped to VariantStructure@@ -1386,24 +1376,15 @@ func wikiGraphBareKey(slug string) string {return strings.TrimSpace(s)}-// tokenizeWikiGraphContent prepares the lexical fields used by wiki graph-// keyword search. Keep the raw text as a fallback so graph rows remain-// searchable when the tokenizer pool is unavailable during startup or tests.-func tokenizeWikiGraphContent(title, slug, description string) (string, string, string, string) {- title = strings.TrimSpace(title)+// tokenizeWikiGraphContent tokenizes Python's entity search text,+// `slug + " " + description` (dataset_wiki_generator.py:750-773); the entity row+// gets no title token column.+func tokenizeWikiGraphContent(slug, description string) (string, string) {content := strings.TrimSpace(strings.Join([]string{slug, description}, " "))- if title == "" && content == "" {- return "", "", "", ""+ if content == "" {+ return "", ""}- titleLTKS, err := tokenizer.Tokenize(title)- if err != nil || titleLTKS == "" {- titleLTKS = title- }- titleSMLTKS, err := tokenizer.FineGrainedTokenize(titleLTKS)- if err != nil || titleSMLTKS == "" {- titleSMLTKS = titleLTKS- }contentLTKS, err := tokenizer.Tokenize(content)if err != nil || contentLTKS == "" {contentLTKS = content@@ -1412,7 +1393,7 @@ func tokenizeWikiGraphContent(title, slug, description string) (string, string,if err != nil || contentSMLTKS == "" {contentSMLTKS = contentLTKS}- return titleLTKS, titleSMLTKS, contentLTKS, contentSMLTKS+ return contentLTKS, contentSMLTKS}// wikiPageProjection is the subset of a merged wiki_page row that the graph@@ -1829,39 +1810,37 @@ func (w engineWriter) projectWikiGraphRows(_ context.Context, tenant, kb string,seen[entityID] = trueweight := len(p.Outlinks) // raw outlink count; 0 is allowed (Python parity)+ // Python's canvas payload (dataset_wiki_generator.py:751-758): aliases /+ // description / type live inside it. No such columns exist, and Infinity+ // rejects an insert naming one.content, err := json.Marshal(map[string]any{- "slug": p.Slug,- "page_type": p.PageType,- "title": p.Title,- "aliases": p.Aliases,- "summary": p.Summary,- "weight": weight,+ "slug": p.Slug,+ "name": p.Title,+ "aliases": p.Aliases,+ "description": p.Summary,+ "type": p.PageType,+ "weight": weight,})if err != nil {return nil, err}- titleLTKS, titleSMLTKS, contentLTKS, contentSMLTKS := tokenizeWikiGraphContent(p.Title, p.Slug, p.Summary)+ contentLTKS, contentSMLTKS := tokenizeWikiGraphContent(p.Slug, p.Summary)rows = append(rows, map[string]interface{}{- "id": entityID,- "doc_id": kb,- "tenant_id": tenant,- "kb_id": kb,- "available_int": 1,- "compile_kwd": compileKwdWikiEntity,- "type_kwd": "wiki_" + p.PageType,- "entity_type_kwd": "wiki_" + p.PageType,- "slug_kwd": p.Slug,- "title_kwd": p.Title,- "aliases_kwd": p.Aliases,- "description_with_weight": p.Summary,- "weight_int": weight,- "source_chunk_ids": p.SourceChunkIDs,- "source_doc_ids": capSourceDocs(p.SourceDocIDs),- "title_tks": titleLTKS,- "title_sm_tks": titleSMLTKS,- "content_ltks": contentLTKS,- "content_sm_ltks": contentSMLTKS,- "content_with_weight": string(content),+ "id": entityID,+ "doc_id": kb,+ "kb_id": kb,+ "available_int": 1,+ "compile_kwd": compileKwdWikiEntity,+ "type_kwd": "wiki_" + p.PageType,+ "entity_type_kwd": "wiki_" + p.PageType,+ "slug_kwd": p.Slug,+ "title_kwd": p.Title,+ "weight_int": weight,+ "source_chunk_ids": p.SourceChunkIDs,+ "source_doc_ids": capSourceDocs(p.SourceDocIDs),+ "content_ltks": contentLTKS,+ "content_sm_ltks": contentSMLTKS,+ "content_with_weight": string(content),})// Relations: one edge per outlink whose target page exists in this@@ -1905,13 +1884,10 @@ func (w engineWriter) projectWikiGraphRows(_ context.Context, tenant, kb string,rows = append(rows, map[string]interface{}{"id": relID,"doc_id": kb,- "tenant_id": tenant,"kb_id": kb,"available_int": 1,"compile_kwd": compileKwdWikiRelation,"type_kwd": compileKwdWikiRelation,- "from_id": entityID,- "to_id": wikiGraphXXHash("wiki_entity", kb, tgt),"from_kwd": p.Slug,"to_kwd": tgt,"source_doc_ids": srcDocs,diff --git a/internal/ingestion/knowledge_compile/writer_test.go b/internal/ingestion/knowledge_compile/writer_test.goindex 861def5426..69e2fb657e 100644--- a/internal/ingestion/knowledge_compile/writer_test.go+++ b/internal/ingestion/knowledge_compile/writer_test.go@@ -2,6 +2,7 @@ package knowledge_compileimport ("context"+ "strings""testing""time"@@ -69,6 +70,7 @@ func TestMergedChunkMapKeepsWikiFields(t *testing.T) {Content: "# Alpha\n\nBody",Vector: []float32{0.1, 0.2, 0.3},Meta: map[string]any{+ "kind": "page","slug": "entity/alpha","title": "Alpha","page_type": "entity",@@ -85,13 +87,16 @@ func TestMergedChunkMapKeepsWikiFields(t *testing.T) {cases := map[string]string{"slug_kwd": "entity/alpha",- "artifact_slug_kwd": "entity/alpha","title_kwd": "Alpha","page_type_kwd": "entity","topic_kwd": "Knowledge/Core/Alpha","summary_with_weight": "A page about Alpha",- "plan_kwd": "run-abc",- "input_hash_kwd": "hash-123",+ // The wiki page body goes to md_with_weight, the column Python writes+ // and reads (wiki_incremental.py:2190); artifact_slug_kwd was a Go-only+ // column that exists in no engine mapping.+ "md_with_weight": "# Alpha\n\nBody",+ "plan_kwd": "run-abc",+ "input_hash_kwd": "hash-123",}// The wall-clock audit fields must be stamped from `now`, not omitted.if m["create_time"] != now.Format("2006-01-02 15:04:05") {@@ -111,8 +116,12 @@ func TestMergedChunkMapKeepsWikiFields(t *testing.T) {if m["doc_id"] != "kb1" || m["available_int"] != 1 {t.Errorf("merged flags wrong: doc_id=%v available_int=%v", m["doc_id"], m["available_int"])}- if _, ok := m["kc_merged"]; ok {- t.Errorf("merged row must not persist undefined kc_merged field")+ // No kc_* key may be persisted: they exist in no engine mapping, and+ // Infinity rejects an insert that carries an unknown column.+ for k := range m {+ if strings.HasPrefix(k, "kc_") {+ t.Errorf("merged row must not persist the non-schema field %q", k)+ }}if m["q_3_vec"] == nil {t.Errorf("vector column missing")@@ -127,8 +136,7 @@ func TestProductFromChunkMapRestoresWikiFields(t *testing.T) {"id": "wiki/1","doc_id": "d1","compile_kwd": "wiki_page",- "content_with_weight": "# Alpha",- "kc_payload": "# Alpha\n\nBody",+ "content_with_weight": "# Alpha\n\nBody","slug_kwd": "entity/alpha","page_type_kwd": "entity","topic_kwd": "Alpha",@@ -136,7 +144,7 @@ func TestProductFromChunkMapRestoresWikiFields(t *testing.T) {"summary_with_weight": "A page about Alpha","entity_names_kwd": []interface{}{"Alpha"},"related_kb_pages_kwd": []interface{}{"entity/beta"},- "section_level_int": float64(2),+ "depth_int": float64(2),}p, ok := productFromChunkMap(c, "t1", kccommon.VariantWiki)if !ok {diff --git a/internal/ingestion/task/knowledge_compiler_wiring.go b/internal/ingestion/task/knowledge_compiler_wiring.goindex 770e37ae36..ca2b8ba66e 100644--- a/internal/ingestion/task/knowledge_compiler_wiring.go+++ b/internal/ingestion/task/knowledge_compiler_wiring.go@@ -404,7 +404,12 @@ func putWikiActiveStates(ctx context.Context, docEngine engine.DocEngine, statesif len(states) == 0 {return nil}- store, ok := knowledge_compile.NewWikiMapVersionStore(docEngine).(kc.WikiMapActiveStateStore)+ // Every state comes from one document's compile, so they share a tenant and+ // dataset: the resolver only runs when the Infinity chunk table is missing.+ store, ok := knowledge_compile.NewWikiMapVersionStoreWithVectorSizeResolver(+ docEngine,+ wikiMapVectorSizeResolver(states[0].TenantID, states[0].DatasetID),+ ).(kc.WikiMapActiveStateStore)if !ok {return fmt.Errorf("Wiki dirty compile: active MAP store is unavailable")}@@ -416,6 +421,52 @@ func putWikiActiveStates(ctx context.Context, docEngine engine.DocEngine, statesreturn nil}+// wikiMapVectorSizeResolver resolves, lazily, the vector size a missing Infinity+// chunk table needs. The document-level compile path holds no embedder, so it+// binds the dataset's — kb.tenant_embd_id, else kb.embd_id — like a compile task.+func wikiMapVectorSizeResolver(tenantID, datasetID string) func(context.Context) (int, error) {+ svc := service.NewModelProviderService()+ modelSolver := service.NewModelSolver()+ return func(ctx context.Context) (int, error) {+ kb, err := dao.NewKnowledgebaseDAO().GetByID(ctx, dao.DB, datasetID)+ if err != nil {+ return 0, fmt.Errorf("Wiki MAP chunk store: load dataset %s: %w", datasetID, err)+ }+ embedder := &kcEmbedder{svc: svc, solver: modelSolver, tenantID: tenantID, embdID: datasetEmbeddingID(kb)}+ return embedderVectorSize(ctx, embedder)+ }+}++// datasetEmbeddingID mirrors Python's embedding binding for a compile task:+// kb.tenant_embd_id wins, else kb.embd_id, and an empty id leaves the embedder to+// its tenant-default fallback (rag/svr/task_executor.py:1454-1487).+func datasetEmbeddingID(kb *entity.Knowledgebase) string {+ if kb == nil {+ return ""+ }+ if kb.TenantEmbdID != nil && strings.TrimSpace(*kb.TenantEmbdID) != "" {+ return *kb.TenantEmbdID+ }+ return kb.EmbdID+}++// embedderVectorSize reports the vector size a missing Infinity chunk table must+// be created with: an unset dimension is measured with a one-input probe (the+// signal Python sizes the table with in create_idx, task_executor.py:737-740).+func embedderVectorSize(ctx context.Context, embed kc.Embedder) (int, error) {+ if dimension := embed.Dimensions(); dimension > 0 {+ return dimension, nil+ }+ vectors, err := embed.Encode(ctx, []string{"ok"})+ if err != nil {+ return 0, err+ }+ if len(vectors) == 0 || len(vectors[0]) == 0 {+ return 0, fmt.Errorf("embedding returned an empty vector")+ }+ return len(vectors[0]), nil+}+func copyStringAnyMap(source map[string]any) map[string]any {result := make(map[string]any, len(source))for key, value := range source {@@ -541,17 +592,7 @@ func newKnowledgeCompilerDepsResolver() kc.DepsResolver {embedder := &kcEmbedder{svc: svc, solver: modelSolver, tenantID: tenantID, embdID: embeddingModel}mapStore := knowledge_compile.NewWikiMapVersionStoreWithVectorSizeResolver(engine.Get(), func(ctx context.Context) (int, error) {- if dimension := embedder.Dimensions(); dimension > 0 {- return dimension, nil- }- vectors, err := embedder.Encode(ctx, []string{"ok"})- if err != nil {- return 0, err- }- if len(vectors) == 0 || len(vectors[0]) == 0 {- return 0, fmt.Errorf("embedding returned an empty vector")- }- return len(vectors[0]), nil+ return embedderVectorSize(ctx, embedder)})return kc.Deps{@@ -806,7 +847,7 @@ func (s *kcWikiPageStore) FindSimilarPages(ctx context.Context, tenantID, dataseIndexNames: []string{fmt.Sprintf("ragflow_%s", tenantID)},KbIDs: []string{datasetID},Limit: k,- SelectFields: []string{"id", "slug_kwd", "title_kwd", "page_type_kwd", "topic_kwd", "plan_group_kwd", "summary_with_weight", "content_with_weight", "entity_names_kwd", "related_kb_pages_kwd", "outlinks_kwd", "source_chunk_ids", "kc_content_md_raw", "_score"},+ SelectFields: []string{"id", "slug_kwd", "title_kwd", "page_type_kwd", "topic_kwd", "plan_group_kwd", "summary_with_weight", "content_with_weight", "md_with_weight", "entity_names_kwd", "related_kb_pages_kwd", "outlinks_kwd", "source_chunk_ids", "_score"},// compile_kwd="wiki_page" is the schema-backed discriminator for wiki// pages (sections carry compile_kwd="wiki_section"); there is no// "kc_kind" column in the chunk schema, so filtering on it would return@@ -842,7 +883,7 @@ func (s *kcWikiPageStore) GetPageBySlug(ctx context.Context, tenantID, datasetIDIndexNames: []string{fmt.Sprintf("ragflow_%s", tenantID)},KbIDs: []string{datasetID},Limit: 1,- SelectFields: []string{"id", "slug_kwd", "title_kwd", "page_type_kwd", "topic_kwd", "plan_group_kwd", "summary_with_weight", "content_with_weight", "entity_names_kwd", "related_kb_pages_kwd", "outlinks_kwd", "source_chunk_ids", "kc_content_md_raw", "_score"},+ SelectFields: []string{"id", "slug_kwd", "title_kwd", "page_type_kwd", "topic_kwd", "plan_group_kwd", "summary_with_weight", "content_with_weight", "md_with_weight", "entity_names_kwd", "related_kb_pages_kwd", "outlinks_kwd", "source_chunk_ids", "_score"},Filter: map[string]interface{}{"compile_kwd": "wiki_page","slug_kwd": slug,@@ -864,7 +905,7 @@ func (s *kcWikiPageStore) FindPagesBySourceChunks(ctx context.Context, tenantID,IndexNames: []string{fmt.Sprintf("ragflow_%s", tenantID)},KbIDs: []string{datasetID},Limit: k,- SelectFields: []string{"id", "slug_kwd", "title_kwd", "page_type_kwd", "topic_kwd", "summary_with_weight", "content_with_weight", "entity_names_kwd", "related_kb_pages_kwd", "outlinks_kwd", "source_chunk_ids", "kc_content_md_raw", "_score"},+ SelectFields: []string{"id", "slug_kwd", "title_kwd", "page_type_kwd", "topic_kwd", "summary_with_weight", "content_with_weight", "md_with_weight", "entity_names_kwd", "related_kb_pages_kwd", "outlinks_kwd", "source_chunk_ids", "_score"},Filter: map[string]interface{}{"compile_kwd": "wiki_page","source_chunk_ids": chunkIDs,@@ -887,15 +928,18 @@ func (s *kcWikiPageStore) FindPagesBySourceChunks(ctx context.Context, tenantID,func wikiPageCandidateFromRow(row map[string]interface{}) kc.WikiPageCandidate {return kc.WikiPageCandidate{- ID: strings.TrimSpace(anyString(row["id"])),- Slug: strings.TrimSpace(anyString(row["slug_kwd"])),- Title: strings.TrimSpace(anyString(row["title_kwd"])),- PageType: strings.TrimSpace(anyString(row["page_type_kwd"])),- Topic: strings.TrimSpace(anyString(row["topic_kwd"])),- PlanGroup: strings.TrimSpace(anyString(row["plan_group_kwd"])),- Summary: strings.TrimSpace(anyString(row["summary_with_weight"])),- ContentMD: strings.TrimSpace(anyString(row["content_with_weight"])),- ContentMDRaw: strings.TrimSpace(anyString(row["kc_content_md_raw"])),+ ID: strings.TrimSpace(anyString(row["id"])),+ Slug: strings.TrimSpace(anyString(row["slug_kwd"])),+ Title: strings.TrimSpace(anyString(row["title_kwd"])),+ PageType: strings.TrimSpace(anyString(row["page_type_kwd"])),+ Topic: strings.TrimSpace(anyString(row["topic_kwd"])),+ PlanGroup: strings.TrimSpace(anyString(row["plan_group_kwd"])),+ Summary: strings.TrimSpace(anyString(row["summary_with_weight"])),+ ContentMD: strings.TrimSpace(anyString(row["content_with_weight"])),+ // md_with_weight is the page-body column Python writes and reads+ // (wiki_incremental.py:2190/:2251); page rows fall back to+ // content_with_weight when it was not stamped.+ ContentMDRaw: firstNonEmptyString(row["md_with_weight"], row["content_with_weight"]),EntityNames: anyStrings(row["entity_names_kwd"]),RelatedKBPages: anyStrings(row["related_kb_pages_kwd"]),Outlinks: anyStrings(row["outlinks_kwd"]),@@ -904,6 +948,16 @@ func wikiPageCandidateFromRow(row map[string]interface{}) kc.WikiPageCandidate {}}+// firstNonEmptyString returns the first candidate that renders non-empty.+func firstNonEmptyString(values ...interface{}) string {+ for _, v := range values {+ if s := strings.TrimSpace(anyString(v)); s != "" {+ return s+ }+ }+ return ""+}+func anyString(v interface{}) string {switch x := v.(type) {case string:diff --git a/internal/ingestion/task/knowledge_compiler_wiring_test.go b/internal/ingestion/task/knowledge_compiler_wiring_test.goindex ab4a81abf4..99b565bdd6 100644--- a/internal/ingestion/task/knowledge_compiler_wiring_test.go+++ b/internal/ingestion/task/knowledge_compiler_wiring_test.go@@ -20,6 +20,7 @@ import ("testing""ragflow/internal/agent/runtime"+ "ragflow/internal/entity")// TestKnowledgeCompilerRegisteredByWiring locks the composition-root contract:@@ -41,3 +42,38 @@ func TestKnowledgeCompilerRegisteredByWiring(t *testing.T) {t.Fatal("component \"Compiler\" registered with a nil factory")}}++// TestDatasetEmbeddingIDBindsTheDatasetModel mirrors Python's compile-task+// binding: the tenant-level model wins over the dataset's own, a blank tenant id+// falls back to the dataset model, and an empty id defers to the tenant default.+func TestDatasetEmbeddingIDBindsTheDatasetModel(t *testing.T) {+ tenantModel := "tenant-model-1"+ blank := " "++ tests := []struct {+ name string+ kb *entity.Knowledgebase+ want string+ }{+ {name: "nil knowledgebase", kb: nil, want: ""},+ {name: "dataset model", kb: &entity.Knowledgebase{EmbdID: "bge-large"}, want: "bge-large"},+ {+ name: "tenant model wins",+ kb: &entity.Knowledgebase{EmbdID: "bge-large", TenantEmbdID: &tenantModel},+ want: "tenant-model-1",+ },+ {+ name: "blank tenant model falls back",+ kb: &entity.Knowledgebase{EmbdID: "bge-large", TenantEmbdID: &blank},+ want: "bge-large",+ },+ }++ for _, tc := range tests {+ t.Run(tc.name, func(t *testing.T) {+ if got := datasetEmbeddingID(tc.kb); got != tc.want {+ t.Fatalf("datasetEmbeddingID() = %q, want %q", got, tc.want)+ }+ })+ }+}diff --git a/internal/service/dataset_artifact_keyword_integration_test.go b/internal/service/dataset_artifact_keyword_integration_test.goindex 0f3f1728dc..8ea9870d14 100644--- a/internal/service/dataset_artifact_keyword_integration_test.go+++ b/internal/service/dataset_artifact_keyword_integration_test.go@@ -23,7 +23,6 @@ func wikiIntRow(id, pageType, slug, title, topic, summary string, outlinks int)"scope_kwd": "dataset","compile_kwd": CompileKwdWikiPage,"slug_kwd": fullSlug,- "artifact_slug_kwd": fullSlug,"title_kwd": title,"page_type_kwd": pageType,"topic_kwd": topic,diff --git a/internal/service/dataset_artifact_service.go b/internal/service/dataset_artifact_service.goindex ef20e5161b..4aee8ab937 100644--- a/internal/service/dataset_artifact_service.go+++ b/internal/service/dataset_artifact_service.go@@ -615,7 +615,7 @@ func (s *DatasetArtifactService) GetWikiGraph(ctx context.Context, tenantID, datorderBy = (&types.OrderByExpr{}).Desc("weight_int")}entityChunks, _, err := s.searchCompiledWithMatch(ctx, tenantID, datasetID, entityFilter,- []string{"id", "slug_kwd", "title_kwd", "aliases_kwd", "description_with_weight", "entity_type_kwd", "weight_int", "source_chunk_ids"},+ []string{"id", "slug_kwd", "title_kwd", "entity_type_kwd", "weight_int", "source_chunk_ids", "content_with_weight"},0, limit, orderBy, matchExprs)if err != nil {return nil, err@@ -645,12 +645,23 @@ func (s *DatasetArtifactService) GetWikiGraph(ctx context.Context, tenantID, datpageType = bareSlug[:idx]bareSlug = bareSlug[idx+1:]}+ // name / aliases / description live in the content_with_weight payload+ // (as Python writes them); the columns they replaced exist in no mapping.+ payload := wikiEntityPayload(c)+ name := firstStringValue(c["title_kwd"])+ if v := payloadString(payload, "name"); v != "" {+ name = v+ }+ entityType := pageType+ if v := payloadString(payload, "type"); v != "" {+ entityType = v+ }graph.Entities = append(graph.Entities, WikiGraphEntity{Slug: bareSlug,- Name: firstStringValue(c["title_kwd"]),- Aliases: toStringSlice(c["aliases_kwd"]),- Description: firstStringValue(c["description_with_weight"]),- Type: pageType,+ Name: name,+ Aliases: payloadStrings(payload, "aliases"),+ Description: payloadString(payload, "description"),+ Type: entityType,Weight: w,SourceChunkIDs: toStringSlice(c["source_chunk_ids"]),})@@ -1064,6 +1075,47 @@ func firstStringValue(v interface{}) string {return ""}+// wikiEntityPayload decodes the canvas payload a wiki_entity row stores in+// content_with_weight (Python `_wiki_entity_payload`); the payload is+// authoritative for name/aliases/description/type.+func wikiEntityPayload(row map[string]interface{}) map[string]interface{} {+ raw := firstStringValue(row["content_with_weight"])+ if strings.TrimSpace(raw) == "" {+ return nil+ }+ var payload map[string]interface{}+ if json.Unmarshal([]byte(raw), &payload) != nil {+ return nil+ }+ return payload+}++// payloadString reads a payload field, accepting the legacy Go key too.+func payloadString(payload map[string]interface{}, key string) string {+ if payload == nil {+ return ""+ }+ if v, ok := payload[key].(string); ok {+ return strings.TrimSpace(v)+ }+ // Legacy Go rows named the same values title / page_type / summary.+ legacy := map[string]string{"name": "title", "type": "page_type", "description": "summary"}[key]+ if legacy != "" {+ if v, ok := payload[legacy].(string); ok {+ return strings.TrimSpace(v)+ }+ }+ return ""+}++// payloadStrings returns a string-list payload field.+func payloadStrings(payload map[string]interface{}, key string) []string {+ if payload == nil {+ return []string{}+ }+ return toStringSlice(payload[key])+}+// bareWikiSlug strips only the first path segment from a full wiki slug// ("entity/location/长社" -> "location/长社"). Nested type/name segments are// preserved so distinct typed entities remain distinguishable in graph links.