Skip to content

RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.

fix(knowledge_compile): make compiled rows writable to Infinity (#20208)

qinling0210da98d6267124
diff --git a/internal/engine/infinity/chunk.go b/internal/engine/infinity/chunk.go
index 502c565046..00fa7865ea 100644
--- a/internal/engine/infinity/chunk.go
+++ b/internal/engine/infinity/chunk.go
@@ -25,6 +25,7 @@ import (
"ragflow/internal/common"
"ragflow/internal/engine/types"
"ragflow/internal/utility"
+ "reflect"
"regexp"
"slices"
"sort"
@@ -134,21 +135,15 @@ func (e *Engine) createChunkStoreWithDB(db *infinity.Database, baseName, dataset
common.Info("Successfully added vector column", zap.String("column", vectorColName))
}
- // A metadata-only Wiki MAP write can create the dataset table before a
- // table-parser document arrives. Reconcile the parser-specific column on
- // the existing-table path so creation order does not make table chunks
- // unwritable.
+ // Reconcile the parser-specific column on the existing-table path: the
+ // table may have been created ahead of its first table-parser document.
if parserID == "table" {
chunkDataExists, checkErr := e.columnExists(table, "chunk_data")
if checkErr != nil {
return fmt.Errorf("failed to check chunk_data column: %w", checkErr)
}
- if !chunkDataExists {
- if _, addErr := table.AddColumns(infinity.TableSchema{
- &infinity.ColumnDefinition{Name: "chunk_data", DataType: "json", Default: "{}"},
- }); addErr != nil {
- return fmt.Errorf("failed to add chunk_data column: %w", addErr)
- }
+ if err := e.ensureChunkDataColumn(table, chunkDataExists); err != nil {
+ return err
}
}
} else {
@@ -284,6 +279,32 @@ func (e *Engine) createChunkStoreWithDB(db *infinity.Database, baseName, dataset
return nil
}
+// parserIDFromChunks reports the parser that produced a batch: a chunk carrying
+// chunk_data comes from the table parser (Python's create_idx(parser_id)).
+func parserIDFromChunks(chunks []map[string]interface{}) string {
+ for _, chunk := range chunks {
+ if chunkData, ok := chunk["chunk_data"].(map[string]interface{}); ok && chunkData != nil {
+ return "table"
+ }
+ }
+ return ""
+}
+
+// ensureChunkDataColumn adds the column the table parser writes when a table
+// created ahead of its first table-parser document lacks it. exists is the
+// caller's already-known column existence.
+func (e *Engine) ensureChunkDataColumn(table *infinity.Table, exists bool) error {
+ if exists {
+ return nil
+ }
+ if _, err := table.AddColumns(infinity.TableSchema{
+ &infinity.ColumnDefinition{Name: "chunk_data", DataType: "json", Default: "{}"},
+ }); err != nil {
+ return fmt.Errorf("failed to add chunk_data column: %w", err)
+ }
+ return nil
+}
+
// InsertChunks inserts documents into a dataset table;
// Table name format: {baseName}_{datasetID}
// Auto-create the table if it doesn't exist
@@ -325,14 +346,8 @@ func (e *Engine) InsertChunks(ctx context.Context, chunks []map[string]interface
return nil, fmt.Errorf("cannot infer vector size from chunks")
}
- // Determine parser_id from chunk structure
- parserID := ""
- if chunkData, ok := chunks[0]["chunk_data"].(map[string]interface{}); ok && chunkData != nil {
- parserID = "table"
- }
-
// Create table
- if err := e.createChunkStoreWithDB(db, baseName, datasetID, vectorSize, parserID); err != nil {
+ if err := e.createChunkStoreWithDB(db, baseName, datasetID, vectorSize, parserIDFromChunks(chunks)); err != nil {
return nil, fmt.Errorf("failed to create table: %w", err)
}
@@ -355,11 +370,15 @@ func (e *Engine) InsertChunks(ctx context.Context, chunks []map[string]interface
// ShowColumns returns a result set where Data contains arrays of column values
re := regexp.MustCompile(`Embedding\([a-z]+,(\d+)\)`)
+ hasChunkData := false
if nameArr, ok := result.Data["name"]; ok {
if typeArr, ok := result.Data["type"]; ok {
for i := 0; i < len(nameArr); i++ {
colName, _ := nameArr[i].(string)
colType, _ := typeArr[i].(string)
+ if colName == "chunk_data" {
+ hasChunkData = true
+ }
matches := re.FindStringSubmatch(colType)
if len(matches) >= 2 {
size, _ := strconv.Atoi(matches[1])
@@ -369,6 +388,15 @@ func (e *Engine) InsertChunks(ctx context.Context, chunks []map[string]interface
}
}
+ // Reconcile the parser column here too: an existing table is opened through
+ // GetTable and never reaches createChunkStoreWithDB. Best effort, like Python's
+ // ensure_columns — a column that stays missing surfaces on the insert below.
+ if parserIDFromChunks(chunks) == "table" {
+ if err := e.ensureChunkDataColumn(table, hasChunkData); err != nil {
+ common.Warn("Failed to add chunk_data column", zap.Error(err))
+ }
+ }
+
// Transform chunks using helper function
insertChunks := make([]map[string]interface{}, len(chunks))
for i, chunk := range chunks {
@@ -2438,31 +2466,39 @@ func equivalentConditionToStr(condition map[string]interface{}, tableColumns map
return strings.Join(cond, " AND ")
}
-// decodeJSONFields converts Infinity JSON-column strings back into their Go
-// array/object representation. Legacy list rows used ###-joined strings, so
-// preserve read compatibility for those columns.
+// decodeJSONFields converts Infinity JSON-column values back into their Go
+// representation, mirroring Python's read-side parse_json_list for the list
+// columns (infinity_conn.py:937-947): a slice passes through, nil becomes [], a
+// JSON non-list value is wrapped, a legacy "###" string is split, and object
+// columns such as source_chunk_hashes keep their shape.
func decodeJSONFields(chunk map[string]interface{}) {
for fieldName, value := range chunk {
- fieldNameLower := strings.ToLower(fieldName)
- if !fieldJSON(fieldNameLower) {
+ if !fieldJSON(fieldName) {
continue
}
+ listField := fieldJSONList(fieldName)
text, ok := value.(string)
if !ok {
+ if listField {
+ chunk[fieldName] = asJSONList(value)
+ }
continue
}
if text == "" {
- if fieldJSONList(fieldNameLower) {
+ if listField {
chunk[fieldName] = []interface{}{}
}
continue
}
var decoded interface{}
if err := json.Unmarshal([]byte(text), &decoded); err == nil {
+ if listField {
+ decoded = asJSONList(decoded)
+ }
chunk[fieldName] = decoded
continue
}
- if fieldJSONList(fieldNameLower) {
+ if listField {
parts := strings.Split(text, "###")
decodedList := make([]interface{}, 0, len(parts))
for _, part := range parts {
@@ -2475,6 +2511,19 @@ func decodeJSONFields(chunk map[string]interface{}) {
}
}
+// asJSONList normalizes a list-column value the way Python's parse_json_list
+// does (infinity_conn.py:938-944): a slice passes through unchanged, a nil value
+// becomes [] and any other scalar is wrapped in a single-element list.
+func asJSONList(value interface{}) interface{} {
+ if value == nil {
+ return []interface{}{}
+ }
+ if rv := reflect.ValueOf(value); rv.Kind() == reflect.Slice || rv.Kind() == reflect.Array {
+ return value
+ }
+ return []interface{}{value}
+}
+
// calculateScores calculates _score = score_column + pagerank
func calculateScores(chunks []map[string]interface{}, scoreColumn, pagerankField string) []map[string]interface{} {
for i := range chunks {
@@ -2637,9 +2686,11 @@ func numericSlice(v interface{}) ([]interface{}, bool) {
// Also handles:
// - kb_id: extracts first element if it's a list
// - position_int, page_num_int, top_int: converts arrays to hex strings
+// - fieldJSON / fieldJSONList: json column -> JSON string (Infinity json columns)
// - tag_kwd: joins with ### separator
// - question_kwd: joins with newline separator
-// - chunk_data: dict -> JSON string
+// - other *_kwd keyword fields: lists join with ###
+// - chunk_data / extra: dict -> JSON string (chunk-table varchar columns)
// - Missing embeddings filled with zeros if embeddingCols provided
func transformChunkFields(chunk map[string]interface{}, embeddingCols [][2]interface{}) map[string]interface{} {
d := make(map[string]interface{})
@@ -2744,17 +2795,27 @@ func transformChunkFields(chunk map[string]interface{}, embeddingCols [][2]inter
}
case "chunk_data":
d["chunk_data"] = utility.ConvertMapToJSONString(v)
+ case "extra":
+ // Python json-encodes a dict-valued `extra` (infinity_conn.py:572-581);
+ // raw, the SDK reads the map as a sparse-vector literal and rejects the
+ // string keys. meta_fields is not a chunk column — InsertMetadata
+ // encodes it on the doc-meta path (infinity/metadata.go).
+ d[k] = utility.ConvertMapToJSONString(v)
default:
if fieldJSON(k) {
- d[k] = encodeJSONField(v)
+ d[k] = encodeJSONField(k, v)
} else if strings.HasSuffix(k, "_feas") {
jsonBytes, _ := json.Marshal(v)
d[k] = string(jsonBytes)
} else if fieldKeyword(k) {
- // keyword fields with list values -> ### joined
- if list, ok := v.([]interface{}); ok {
+ // keyword fields with list values -> ### joined; accept the
+ // Go-native []string as well as []interface{}.
+ switch list := v.(type) {
+ case []string:
+ d[k] = strings.Join(list, "###")
+ case []interface{}:
d[k] = strings.Join(utility.ConvertToStringSlice(list), "###")
- } else {
+ default:
d[k] = v
}
} else {
@@ -2789,13 +2850,34 @@ func transformChunkFields(chunk map[string]interface{}, embeddingCols [][2]inter
return d
}
-func encodeJSONField(value interface{}) interface{} {
- if text, ok := value.(string); ok && json.Valid([]byte(text)) {
+// encodeJSONField renders a json column value as the string Infinity stores. The
+// Go SDK cannot encode a []string constant (3058), so a list is JSON-dumped; a
+// non-list value falls back to the column default ("[]" for the json-list columns,
+// infinity_conn.py:557-558; "{}" for the object one). Strings pass through, so
+// re-transforming a row stays idempotent.
+func encodeJSONField(fieldName string, value interface{}) interface{} {
+ listField := fieldJSONList(fieldName)
+ fallback := "{}"
+ if listField {
+ fallback = "[]"
+ }
+ if text, ok := value.(string); ok {
+ if text == "" {
+ return fallback
+ }
return text
}
+ if value == nil {
+ return fallback
+ }
+ if listField {
+ if rv := reflect.ValueOf(value); rv.Kind() != reflect.Slice && rv.Kind() != reflect.Array {
+ return fallback
+ }
+ }
encoded, err := json.Marshal(value)
if err != nil {
- return value
+ return fallback
}
return string(encoded)
}
diff --git a/internal/engine/infinity/chunk_insert_test.go b/internal/engine/infinity/chunk_insert_test.go
new file mode 100644
index 0000000000..d4c6622237
--- /dev/null
+++ b/internal/engine/infinity/chunk_insert_test.go
@@ -0,0 +1,50 @@
+// Copyright 2026 The InfiniFlow Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package infinity
+
+import "testing"
+
+// TestParserIDFromChunks pins the table-parser detection that gates the
+// chunk_data column reconcile: only a batch that carries chunk_data needs the
+// parser-specific column, and any position in the batch counts (a mixed batch
+// still writes chunk_data on the rows that have it).
+func TestParserIDFromChunks(t *testing.T) {
+ var nilMap map[string]interface{}
+
+ tests := []struct {
+ name string
+ chunks []map[string]interface{}
+ want string
+ }{
+ {name: "no chunks", chunks: nil, want: ""},
+ {name: "plain chunk", chunks: []map[string]interface{}{{"id": "c1"}}, want: ""},
+ {name: "nil chunk_data", chunks: []map[string]interface{}{{"chunk_data": nil}}, want: ""},
+ {name: "typed nil chunk_data", chunks: []map[string]interface{}{{"chunk_data": nilMap}}, want: ""},
+ {name: "empty chunk_data", chunks: []map[string]interface{}{{"chunk_data": map[string]interface{}{}}}, want: "table"},
+ {
+ name: "table chunk after a plain chunk",
+ chunks: []map[string]interface{}{{"id": "c1"}, {"chunk_data": map[string]interface{}{"rows": []interface{}{}}}},
+ want: "table",
+ },
+ }
+
+ for _, tc := range tests {
+ t.Run(tc.name, func(t *testing.T) {
+ if got := parserIDFromChunks(tc.chunks); got != tc.want {
+ t.Fatalf("parserIDFromChunks() = %q, want %q", got, tc.want)
+ }
+ })
+ }
+}
diff --git a/internal/engine/infinity/chunk_transform_test.go b/internal/engine/infinity/chunk_transform_test.go
index a0193cee78..5c38ca22b1 100644
--- a/internal/engine/infinity/chunk_transform_test.go
+++ b/internal/engine/infinity/chunk_transform_test.go
@@ -15,8 +15,16 @@
package infinity
import (
+ "encoding/json"
+ "os"
"reflect"
+ "regexp"
+ "strings"
"testing"
+
+ "ragflow/internal/utility"
+
+ infinitysdk "github.com/infiniflow/infinity-go-sdk"
)
// TestTransformChunkFields_IngestionShape is the T0 unit-tier read-back
@@ -144,33 +152,6 @@ func TestTransformChunkFieldsJoinsNativeKeywordSlice(t *testing.T) {
}
}
-// TestTransformChunkFieldsJSONSlices serializes Go-native slices for Infinity
-// JSON columns. Passing []string directly to the SDK is rejected as an
-// unsupported slice element type.
-func TestTransformChunkFieldsJSONSlices(t *testing.T) {
- got := transformChunkFields(map[string]interface{}{
- "source_chunk_ids": []string{"chunk-1", "chunk-2"},
- "source_doc_ids": []string{"doc-1"},
- }, nil)
-
- if want := `["chunk-1","chunk-2"]`; got["source_chunk_ids"] != want {
- t.Errorf("source_chunk_ids = %#v, want %q", got["source_chunk_ids"], want)
- }
- if want := `["doc-1"]`; got["source_doc_ids"] != want {
- t.Errorf("source_doc_ids = %#v, want %q", got["source_doc_ids"], want)
- }
-}
-
-func TestTransformChunkFieldsKeepsEncodedJSON(t *testing.T) {
- got := transformChunkFields(map[string]interface{}{
- "source_doc_ids": `["doc-1"]`,
- }, nil)
-
- if want := `["doc-1"]`; got["source_doc_ids"] != want {
- t.Errorf("source_doc_ids = %#v, want %q", got["source_doc_ids"], want)
- }
-}
-
func TestDecodeJSONFields(t *testing.T) {
row := map[string]interface{}{
"source_doc_ids": `["doc-1","doc-2"]`,
@@ -207,3 +188,318 @@ func TestTransformChunkFieldsDropsTenantRoutingField(t *testing.T) {
t.Fatalf("kb_id = %#v, want %q", got["kb_id"], "kb-1")
}
}
+
+// compiledRowStringLists are the string-list values the compile path hands over:
+// Python's _JSON_LIST_FIELDS columns plus the keyword column children_kwd.
+var compiledRowStringLists = map[string]interface{}{
+ "source_chunk_ids": []string{"c1", "c2"},
+ "source_doc_ids": []string{"d1"},
+ "compilation_template_ids": []string{"t1"},
+ "doc_ids_kwd": []string{"d1"},
+ "entity_names_kwd": []string{"alpha", "玄德"},
+ "outlinks_kwd": []string{"entity/beta"},
+ "related_kb_pages_kwd": []string{"entity/beta"},
+ "rechunked_from_chunk_ids": []string{"c9"},
+ "children_kwd": []string{"r1", "r2"},
+}
+
+// compiledRowForTransform builds a post-indexdoc compiled chunk row.
+func compiledRowForTransform() map[string]interface{} {
+ row := map[string]interface{}{
+ "id": "c1",
+ "doc_id": "d1",
+ "kb_id": "kb1",
+ "content_with_weight": "compiled body",
+ "compile_kwd": "tree",
+ "raptor_kwd": "root",
+ "raptor_layer_int": 2,
+ "md_with_weight": "# page",
+ "extra": map[string]interface{}{"raptor_method": "gmm"},
+ "q_3_vec": []float64{0.1, 0.2, 0.3},
+ }
+ for k, v := range compiledRowStringLists {
+ row[k] = v
+ }
+ return row
+}
+
+// TestInsertValuesAreEncodableConstants is the regression test for the reported
+// failure
+//
+// insert chunk batch 0-32 after 3 attempts: failed to insert chunks to
+// dataset: InfinityException(3058, Unsupported slice element type: string)
+//
+// The SDK encodes every value as a constant and rejects a Go-native []string
+// (expression_parser.go parseSliceConstantValue), so the raw values must NOT
+// encode while everything transformChunkFields emits must. If a future SDK
+// accepts string slices the repro is gone, so the test skips instead of failing.
+func TestInsertValuesAreEncodableConstants(t *testing.T) {
+ for k, v := range compiledRowStringLists {
+ if _, err := infinitysdk.ParseConstantValue(v); err == nil {
+ t.Skipf("SDK now supports string slices; the 3058 repro is gone (raw %s=%#v is encodable)", k, v)
+ }
+ }
+
+ transformed := transformChunkFields(compiledRowForTransform(), nil)
+ for k, v := range transformed {
+ if _, err := infinitysdk.ParseConstantValue(v); err != nil {
+ t.Errorf("transformed %s = %#v cannot be encoded for the insert: %v", k, v, err)
+ }
+ }
+ // The json columns travel as JSON arrays (Python json.dumps) and the
+ // keyword column as the ### join — strings either way, never a Go slice.
+ for _, k := range []string{
+ "source_chunk_ids", "source_doc_ids", "compilation_template_ids", "doc_ids_kwd",
+ "entity_names_kwd", "outlinks_kwd", "related_kb_pages_kwd", "rechunked_from_chunk_ids",
+ } {
+ if s, ok := transformed[k].(string); !ok || !strings.HasPrefix(s, "[") {
+ t.Errorf("transformed %s = %#v, want a JSON array string", k, transformed[k])
+ }
+ }
+ if want := "r1###r2"; transformed["children_kwd"] != want {
+ t.Errorf("children_kwd = %#v, want %q", transformed["children_kwd"], want)
+ }
+ if _, ok := transformed["extra"].(string); !ok {
+ t.Errorf("extra = %#v, want a JSON string", transformed["extra"])
+ }
+}
+
+// TestTransformedCompiledRowNamesOnlyMappingColumns covers the failure that
+// follows 3058: Infinity rejects a column the table does not have, which a
+// Go-only field such as tenant_id triggered.
+func TestTransformedCompiledRowNamesOnlyMappingColumns(t *testing.T) {
+ columns := chunkMappingColumns(t)
+ vectorColumn := regexp.MustCompile(`^q_\d+_vec$`)
+ for k := range transformChunkFields(compiledRowForTransform(), nil) {
+ if columns[k] || vectorColumn.MatchString(k) {
+ continue
+ }
+ t.Errorf("transformed key %q is not a column of conf/infinity_mapping.json", k)
+ }
+}
+
+// TestFieldJSONMatchesJSONMappingColumns keeps the write side's encoding set in
+// step with the schema: a column conf/infinity_mapping.json declares as json must
+// be encoded by fieldJSON before the insert (Infinity answers 3058 for a raw Go
+// slice), and a column fieldJSON encodes must be declared json (a JSON string in
+// a typed column is just as wrong).
+func TestFieldJSONMatchesJSONMappingColumns(t *testing.T) {
+ jsonColumns := 0
+ for column, columnType := range mappingColumnTypes(t) {
+ isJSON := columnType == "json"
+ if isJSON {
+ jsonColumns++
+ }
+ if encoded := fieldJSON(column); encoded != isJSON {
+ t.Errorf("fieldJSON(%q) = %v, but conf/infinity_mapping.json declares type %q", column, encoded, columnType)
+ }
+ }
+ if jsonColumns == 0 {
+ t.Fatal("conf/infinity_mapping.json parsed with no json column; the mapping read is broken")
+ }
+}
+
+// chunkMappingColumns reads the column set the chunk table is created from.
+func chunkMappingColumns(t *testing.T) map[string]bool {
+ columns := mappingColumnTypes(t)
+ out := make(map[string]bool, len(columns))
+ for name := range columns {
+ out[name] = true
+ }
+ return out
+}
+
+// mappingColumnTypes reads conf/infinity_mapping.json as column -> declared type.
+func mappingColumnTypes(t *testing.T) map[string]string {
+ t.Helper()
+ path, err := utility.FindConfFileInProject("infinity_mapping.json")
+ if err != nil || path == nil {
+ t.Fatalf("locate infinity_mapping.json: %v", err)
+ }
+ data, err := os.ReadFile(*path)
+ if err != nil {
+ t.Fatalf("read %s: %v", *path, err)
+ }
+ var columns map[string]struct {
+ Type string `json:"type"`
+ }
+ if err := json.Unmarshal(data, &columns); err != nil {
+ t.Fatalf("parse %s: %v", *path, err)
+ }
+ out := make(map[string]string, len(columns))
+ for name, def := range columns {
+ out[name] = def.Type
+ }
+ return out
+}
+
+// TestTransformChunkFieldsSerializesJSONListColumns is the regression for the
+// reported 3058: the compile path hands over Go-native []string for the
+// json-typed provenance columns, which the Go SDK cannot encode. Python
+// json-encodes them (_JSON_LIST_FIELDS).
+func TestTransformChunkFieldsSerializesJSONListColumns(t *testing.T) {
+ got := transformChunkFields(map[string]interface{}{
+ "source_chunk_ids": []string{"c1", "c2"},
+ "source_doc_ids": []interface{}{"d1"},
+ "compilation_template_ids": []string{"t1"},
+ "entity_names_kwd": []string{"刘备", "玄德"},
+ "outlinks_kwd": []interface{}{"concept/三国演义"},
+ "related_kb_pages_kwd": []string{"entity/person/刘备"},
+ }, nil)
+
+ want := map[string]string{
+ "source_chunk_ids": `["c1","c2"]`,
+ "source_doc_ids": `["d1"]`,
+ "compilation_template_ids": `["t1"]`,
+ "entity_names_kwd": `["刘备","玄德"]`,
+ "outlinks_kwd": `["concept/三国演义"]`,
+ "related_kb_pages_kwd": `["entity/person/刘备"]`,
+ }
+ for field, expected := range want {
+ if got[field] != expected {
+ t.Errorf("%s = %#v, want %q", field, got[field], expected)
+ }
+ }
+}
+
+// TestTransformChunkFieldsJSONListColumnEdgeCases: empty list -> "[]", an
+// already serialized value survives, nil -> "[]".
+func TestTransformChunkFieldsJSONListColumnEdgeCases(t *testing.T) {
+ got := transformChunkFields(map[string]interface{}{
+ "source_chunk_ids": []string{},
+ "source_doc_ids": `["already"]`,
+ "outlinks_kwd": nil,
+ }, nil)
+
+ if want := "[]"; got["source_chunk_ids"] != want {
+ t.Errorf("empty source_chunk_ids = %#v, want %q", got["source_chunk_ids"], want)
+ }
+ if want := `["already"]`; got["source_doc_ids"] != want {
+ t.Errorf("serialized source_doc_ids = %#v, want %q", got["source_doc_ids"], want)
+ }
+ if want := "[]"; got["outlinks_kwd"] != want {
+ t.Errorf("nil outlinks_kwd = %#v, want %q", got["outlinks_kwd"], want)
+ }
+}
+
+// TestTransformChunkFieldsJoinsGenericKeywordSlices covers the keyword branch
+// beyond important_kwd: the compile path emits children_kwd / entity_names as
+// plain []string, which must be joined like the JSON-decoded []interface{} form.
+func TestTransformChunkFieldsJoinsGenericKeywordSlices(t *testing.T) {
+ got := transformChunkFields(map[string]interface{}{
+ "children_kwd": []string{"c1", "c2"},
+ "raptor_kwd": "root",
+ }, nil)
+
+ if want := "c1###c2"; got["children_kwd"] != want {
+ t.Errorf("children_kwd = %#v, want %q", got["children_kwd"], want)
+ }
+ if want := "root"; got["raptor_kwd"] != want {
+ t.Errorf("raptor_kwd = %#v, want %q", got["raptor_kwd"], want)
+ }
+}
+
+// TestTransformChunkFieldsSerializesMapColumns pins the dict-valued varchar
+// column `extra` (infinity_conn.py:572-581).
+func TestTransformChunkFieldsSerializesMapColumns(t *testing.T) {
+ got := transformChunkFields(map[string]interface{}{
+ "extra": map[string]interface{}{"raptor_method": "gmm"},
+ }, nil)
+
+ if want := `{"raptor_method":"gmm"}`; got["extra"] != want {
+ t.Errorf("extra = %#v, want %q", got["extra"], want)
+ }
+}
+
+// TestTransformChunkFieldsSerializesObjectJSONColumn pins the object-typed json
+// column (source_chunk_hashes): a map is dumped and a nil value falls back to
+// the mapping default "{}".
+func TestTransformChunkFieldsSerializesObjectJSONColumn(t *testing.T) {
+ got := transformChunkFields(map[string]interface{}{
+ "source_chunk_hashes": map[string]interface{}{"c1": "h1"},
+ }, nil)
+ if want := `{"c1":"h1"}`; got["source_chunk_hashes"] != want {
+ t.Errorf("source_chunk_hashes = %#v, want %q", got["source_chunk_hashes"], want)
+ }
+
+ got = transformChunkFields(map[string]interface{}{"source_chunk_hashes": nil}, nil)
+ if want := "{}"; got["source_chunk_hashes"] != want {
+ t.Errorf("nil source_chunk_hashes = %#v, want %q", got["source_chunk_hashes"], want)
+ }
+}
+
+// TestDecodeJSONFieldsNormalizesListColumns pins the parse_json_list cases that
+// do not arrive as JSON strings (infinity_conn.py:938-944): a nil value is [],
+// a JSON non-list value is wrapped, an object column keeps its shape.
+func TestDecodeJSONFieldsNormalizesListColumns(t *testing.T) {
+ row := map[string]interface{}{
+ "related_kb_pages_kwd": nil,
+ "outlinks_kwd": float64(3.5),
+ "source_doc_ids": `[]`,
+ "entity_names_kwd": []interface{}{"e1"},
+ "source_chunk_hashes": nil,
+ "content_with_weight": `{"x":1}`,
+ "docnm_kwd": "plain",
+ }
+
+ decodeJSONFields(row)
+
+ if want := []interface{}{}; !reflect.DeepEqual(row["related_kb_pages_kwd"], want) {
+ t.Errorf("nil related_kb_pages_kwd = %#v, want %#v", row["related_kb_pages_kwd"], want)
+ }
+ if want := []interface{}{float64(3.5)}; !reflect.DeepEqual(row["outlinks_kwd"], want) {
+ t.Errorf("scalar outlinks_kwd = %#v, want %#v", row["outlinks_kwd"], want)
+ }
+ if want := []interface{}{}; !reflect.DeepEqual(row["source_doc_ids"], want) {
+ t.Errorf("empty source_doc_ids = %#v, want %#v", row["source_doc_ids"], want)
+ }
+ if want := []interface{}{"e1"}; !reflect.DeepEqual(row["entity_names_kwd"], want) {
+ t.Errorf("decoded entity_names_kwd = %#v, want %#v", row["entity_names_kwd"], want)
+ }
+ if row["source_chunk_hashes"] != nil {
+ t.Errorf("object column source_chunk_hashes = %#v, want it untouched", row["source_chunk_hashes"])
+ }
+ if row["content_with_weight"] != `{"x":1}` {
+ t.Errorf("content_with_weight should be untouched, got %#v", row["content_with_weight"])
+ }
+ if row["docnm_kwd"] != "plain" {
+ t.Errorf("docnm_kwd should be untouched, got %#v", row["docnm_kwd"])
+ }
+}
+
+// TestApplyFieldMappingsDecodesJSONList confirms the search-path post-processor
+// normalizes JSON-string columns into slices, so get chunk / search / get fields
+// all hand readers the same shape.
+func TestApplyFieldMappingsDecodesJSONList(t *testing.T) {
+ chunks := []map[string]interface{}{{
+ "source_doc_ids": `["d1","d2"]`,
+ "source_chunk_ids": `["c1"]`,
+ "entity_names_kwd": `["e1","e2"]`,
+ "outlinks_kwd": `["o1"]`,
+ "doc_ids_kwd": `["x1"]`,
+ }}
+
+ applyFieldMappings(chunks)
+
+ for _, key := range []string{"source_doc_ids", "source_chunk_ids", "entity_names_kwd", "outlinks_kwd", "doc_ids_kwd"} {
+ if _, ok := chunks[0][key].([]interface{}); !ok {
+ t.Errorf("%s not decoded into []interface{}, got %#v", key, chunks[0][key])
+ }
+ }
+}
+
+// TestJSONColumnsMatchCaseInsensitively pins that the write path
+// (transformChunkFields) and the read path (decodeJSONFields) agree on a json
+// column name regardless of its spelling.
+func TestJSONColumnsMatchCaseInsensitively(t *testing.T) {
+ got := transformChunkFields(map[string]interface{}{"Source_Doc_IDs": []string{"d1"}}, nil)
+ if want := `["d1"]`; got["Source_Doc_IDs"] != want {
+ t.Errorf("Source_Doc_IDs = %#v, want %q", got["Source_Doc_IDs"], want)
+ }
+
+ row := map[string]interface{}{"Source_Doc_IDs": nil}
+ decodeJSONFields(row)
+ if want := []interface{}{}; !reflect.DeepEqual(row["Source_Doc_IDs"], want) {
+ t.Errorf("nil Source_Doc_IDs = %#v, want %#v", row["Source_Doc_IDs"], want)
+ }
+}
diff --git a/internal/engine/infinity/common.go b/internal/engine/infinity/common.go
index 71c4561356..eb28450bc9 100644
--- a/internal/engine/infinity/common.go
+++ b/internal/engine/infinity/common.go
@@ -159,9 +159,11 @@ func fieldKeyword(fieldName string) bool {
}
// fieldJSON reports fields stored as Infinity JSON columns. The Infinity Go
-// SDK expects JSON columns as encoded strings, not Go slices or maps.
+// SDK expects JSON columns as encoded strings, not Go slices or maps. Column
+// names are matched case-insensitively so the write path (transformChunkFields)
+// and the read path (decodeJSONFields) agree on every spelling.
func fieldJSON(fieldName string) bool {
- switch fieldName {
+ switch strings.ToLower(fieldName) {
case "source_chunk_ids", "source_doc_ids", "compilation_template_ids",
"doc_ids_kwd", "entity_names_kwd", "entity_names", "outlinks_kwd",
"related_kb_pages_kwd", "claims", "page_ids", "source_chunk_hashes",
@@ -175,7 +177,7 @@ func fieldJSON(fieldName string) bool {
// fieldJSONList reports JSON columns whose value is an array and whose filter
// semantics are membership rather than whole-document equality.
func fieldJSONList(fieldName string) bool {
- switch fieldName {
+ switch strings.ToLower(fieldName) {
case "source_chunk_ids", "source_doc_ids", "compilation_template_ids",
"doc_ids_kwd", "entity_names_kwd", "entity_names", "outlinks_kwd",
"related_kb_pages_kwd", "claims", "page_ids",
diff --git a/internal/ingestion/component/knowledge_compiler/component.go b/internal/ingestion/component/knowledge_compiler/component.go
index 832d40713f..7eb1113b6e 100644
--- a/internal/ingestion/component/knowledge_compiler/component.go
+++ b/internal/ingestion/component/knowledge_compiler/component.go
@@ -392,12 +392,9 @@ var variantCompileKWD = map[common.Variant]string{
}
// productsToChunkDocs converts the internal compiled Product rows into
-// schema.ChunkDoc values aligned to conf/infinity_mapping.json (lines 1–77).
-// The compile_kwd discriminator marks them as compiled knowledge units
-// (distinct from plain chunks); variant-specific columns are populated from
-// Product.Meta using stable keys (see each variant's build site for the
-// contract). The original kind/level/name/size meta is also carried under
-// "kc_"-prefixed Extra keys so no information is lost.
+// schema.ChunkDoc values aligned to conf/infinity_mapping.json. Product.Meta is
+// not persisted: only engine columns are written, because Infinity rejects an
+// insert naming an unknown column.
func productsToChunkDocs(products []common.Product) ([]schema.ChunkDoc, error) {
docs := make([]schema.ChunkDoc, 0, len(products))
for _, p := range products {
@@ -437,11 +434,6 @@ func productsToChunkDocs(products []common.Product) ([]schema.ChunkDoc, error) {
return nil, err
}
}
- if p.TenantID != "" {
- if err := doc.SetExtraValue("tenant_id", p.TenantID); err != nil {
- return nil, err
- }
- }
compileKWD := variantCompileKWD[p.Variant]
// A variant may pin a finer-grained compile_kwd per row via Meta
// (structure stamps the inferred compile kind — list/set/hypergraph —
@@ -503,16 +495,10 @@ func productsToChunkDocs(products []common.Product) ([]schema.ChunkDoc, error) {
return nil, err
}
}
- // Per-variant fine-grained columns (conf/infinity_mapping.json §45–77).
+ // Per-variant fine-grained columns (conf/infinity_mapping.json §45–97).
if err := applyVariantColumns(&doc, p); err != nil {
return nil, err
}
- // Preserve the raw Product.Meta under kc_* for round-trip fidelity.
- for k, v := range p.Meta {
- if err := doc.SetExtraValue("kc_"+k, v); err != nil {
- return nil, err
- }
- }
docs = append(docs, doc)
}
return docs, nil
@@ -548,9 +534,6 @@ func applyVariantColumns(doc *schema.ChunkDoc, p common.Product) error {
if err := doc.SetExtraValue("slug_kwd", fullSlug); err != nil {
return err
}
- if err := doc.SetExtraValue("artifact_slug_kwd", fullSlug); err != nil {
- return err
- }
}
}
if v := metaString(p.Meta, "title"); v != "" {
@@ -574,18 +557,17 @@ func applyVariantColumns(doc *schema.ChunkDoc, p common.Product) error {
return err
}
}
- // Section rows also carry level/index so a retriever can scope to a
- // sub-section of a wiki page.
- if v, ok := metaInt(p.Meta, "section_level"); ok {
- if err := doc.SetExtraValue("section_level_int", v); err != nil {
- return err
- }
- if err := doc.SetExtraValue("depth_int", v); err != nil {
+ // md_with_weight is the page-body column Python writes with the same
+ // value as content_with_weight (wiki_incremental.py:2190-2191), i.e. the
+ // rendered body; section rows have no page body of their own.
+ if metaString(p.Meta, "kind") == "page" && p.Content != "" {
+ if err := doc.SetExtraValue("md_with_weight", p.Content); err != nil {
return err
}
}
- if v, ok := metaInt(p.Meta, "section_index"); ok {
- if err := doc.SetExtraValue("section_index_int", v); err != nil {
+ // Section depth goes to depth_int; there is no section_level_int column.
+ if v, ok := metaInt(p.Meta, "section_level"); ok {
+ if err := doc.SetExtraValue("depth_int", v); err != nil {
return err
}
}
@@ -732,20 +714,15 @@ func metaString(m map[string]any, key string) string {
return v
}
-// setTitleTokens populates the ChunkDoc title_tks / title_sm_tks fields from a
-// title string, mirroring how the chunker/tokenizer components tokenize titles
-// (coarse → TitleTks, fine-grained → TitleSmTks). Errors are ignored: when the
-// tokenizer pool is uninitialised (no-CGo test path) the fields stay empty,
-// matching the chunker's graceful-degrade behaviour.
+// setTitleTokens populates ChunkDoc.title_tks. Python's page row has no
+// title_sm_tks, and the Infinity writer folds title_kwd / title_sm_tks into
+// `docnm`, so the twin would make docnm map-order dependent.
func setTitleTokens(doc *schema.ChunkDoc, title string) {
if title == "" {
return
}
if tks, err := tokenizer.Tokenize(title); err == nil && tks != "" {
doc.TitleTks = tks
- if sm, err := tokenizer.FineGrainedTokenize(tks); err == nil && sm != "" {
- doc.TitleSmTks = sm
- }
}
}
diff --git a/internal/ingestion/component/knowledge_compiler/component_test.go b/internal/ingestion/component/knowledge_compiler/component_test.go
index ea7d31905f..58a8807d5c 100644
--- a/internal/ingestion/component/knowledge_compiler/component_test.go
+++ b/internal/ingestion/component/knowledge_compiler/component_test.go
@@ -229,7 +229,8 @@ func TestKnowledgeCompiler_Structure_EndToEnd(t *testing.T) {
if !ok {
continue
}
- if kind, _ := cm["kc_kind"].(string); kind == "graph" {
+ // knowledge_graph_kwd discriminates the graph row.
+ if kind, _ := cm["knowledge_graph_kwd"].(string); kind == "graph" {
graphChunks++
}
}
@@ -349,10 +350,10 @@ func runVariant(t *testing.T, variant string, extra map[string]any) []map[string
func TestKnowledgeCompiler_Wiki_EndToEnd(t *testing.T) {
installProseDeps(t)
chunks := runVariant(t, "wiki", nil)
- // wiki produces a "page" chunk (kind stored as kc_kind).
+ // wiki produces a "page" chunk (kind carried by compile_kwd=wiki_page).
foundPage := false
for _, c := range chunks {
- if kind, _ := c["kc_kind"].(string); kind == "page" {
+ if kind, _ := c["compile_kwd"].(string); kind == "wiki_page" {
foundPage = true
}
}
@@ -430,7 +431,7 @@ func invokeWikiCompiler(t *testing.T, inputs map[string]any) []map[string]any {
func hasWikiPageChunk(chunks []map[string]any) bool {
for _, cm := range chunks {
- if kind, _ := cm["kc_kind"].(string); kind == "page" {
+ if kind, _ := cm["compile_kwd"].(string); kind == "wiki_page" {
return true
}
}
@@ -535,7 +536,7 @@ func TestKnowledgeCompiler_Tree_EndToEnd(t *testing.T) {
chunks := runVariant(t, "tree", nil)
foundRoot := false
for _, c := range chunks {
- if kind, _ := c["kc_kind"].(string); kind == "root" {
+ if kind, _ := c["raptor_kwd"].(string); kind == "root" {
foundRoot = true
}
}
@@ -551,7 +552,7 @@ func TestKnowledgeCompiler_Tree_EndToEnd(t *testing.T) {
}
foundRootCoarse := false
for _, c := range chunksCoarse {
- if kind, _ := c["kc_kind"].(string); kind == "root" {
+ if kind, _ := c["raptor_kwd"].(string); kind == "root" {
foundRootCoarse = true
}
}
@@ -570,7 +571,7 @@ func TestKnowledgeCompiler_Mindmap_EndToEnd(t *testing.T) {
chunks := runVariant(t, "mindmap", nil)
entityCount := 0
for _, c := range chunks {
- kind, _ := c["kc_kind"].(string)
+ kind, _ := c["knowledge_graph_kwd"].(string)
if kind == "entity" {
entityCount++
if _, ok := c["name_kwd"]; !ok {
@@ -764,7 +765,7 @@ func TestKnowledgeCompiler_Tree_DegenerateNoInfiniteLoop(t *testing.T) {
foundRoot := false
for _, r := range raw {
if cm, ok := r.(map[string]any); ok {
- if k, _ := cm["kc_kind"].(string); k == "root" {
+ if k, _ := cm["raptor_kwd"].(string); k == "root" {
foundRoot = true
}
}
@@ -876,7 +877,7 @@ func TestKnowledgeCompiler_Wiki_UpdateMergesExistingPage(t *testing.T) {
if !ok {
continue
}
- if cm["compile_kwd"] == "wiki_page" && cm["kc_kind"] == "page" && cm["slug_kwd"] == "entity/person/alpha" {
+ if cm["compile_kwd"] == "wiki_page" && cm["slug_kwd"] == "entity/person/alpha" {
page = cm
break
}
@@ -1441,9 +1442,8 @@ func TestKnowledgeCompiler_BuildInputsAcceptsMapSliceChunks(t *testing.T) {
}
// TestProductsToChunkDocs_PageVsSectionCompileKWD locks the page/section
-// discriminator: a wiki page product is stamped compile_kwd="wiki_page" and a
-// wiki section product compile_kwd="wiki_section", so a page search on
-// compile_kwd="wiki_page" (engine_service / kcWikiPageStore) returns pages only.
+// discriminator (compile_kwd) and the schema-column contract: the page body goes
+// to md_with_weight and no Go-only field (kc_*, tenant_id) is emitted.
func TestProductsToChunkDocs_PageVsSectionCompileKWD(t *testing.T) {
page := common.Product{
ID: "page-id", DocID: "d1", TenantID: "t1", Variant: common.VariantWiki,
@@ -1460,17 +1460,33 @@ func TestProductsToChunkDocs_PageVsSectionCompileKWD(t *testing.T) {
t.Fatalf("productsToChunkDocs: %v", err)
}
var pageKWD, sectionKWD string
- var sectionParent string
+ var sectionParent, pageBody, sectionBody string
for _, d := range docs {
- // Product.Meta is preserved under the kc_* round-trip keys; the page/
- // section kind lives at "kc_kind".
- kind, _ := d.GetExtraString("kc_kind")
- if kind == "page" {
- pageKWD, _ = d.GetExtraString("compile_kwd")
+ // compile_kwd IS the page/section discriminator (wiki_page /
+ // wiki_section); there is no separate kind column.
+ kind, _ := d.GetExtraString("compile_kwd")
+ if kind == "wiki_page" {
+ pageKWD = kind
+ pageBody, _ = d.GetExtraString("md_with_weight")
+ // Python's page row has only title_tks; the Infinity writer folds
+ // title_sm_tks into docnm.
+ if _, ok := d.ToMap()["title_sm_tks"]; ok {
+ t.Errorf("page row must not carry title_sm_tks (docnm would become map-order dependent)")
+ }
}
- if kind == "section" {
- sectionKWD, _ = d.GetExtraString("compile_kwd")
+ if kind == "wiki_section" {
+ sectionKWD = kind
sectionParent, _ = d.GetExtraString("parent_kwd")
+ sectionBody, _ = d.GetExtraString("md_with_weight")
+ }
+ // No Go-only field: Infinity rejects an unknown column.
+ for k := range d.Extra {
+ if strings.HasPrefix(k, "kc_") {
+ t.Errorf("compiled row carries the non-schema field %q", k)
+ }
+ }
+ if _, ok := d.GetExtraString("tenant_id"); ok {
+ t.Errorf("compiled row must not carry tenant_id (not a chunk column)")
}
}
if pageKWD != "wiki_page" {
@@ -1482,6 +1498,12 @@ func TestProductsToChunkDocs_PageVsSectionCompileKWD(t *testing.T) {
if sectionParent != "page-id" {
t.Errorf("section parent_kwd = %q, want page-id", sectionParent)
}
+ if pageBody != "# Alpha\n\nBody" {
+ t.Errorf("page md_with_weight = %q, want the rendered page body", pageBody)
+ }
+ if sectionBody != "" {
+ t.Errorf("section md_with_weight = %q, want empty (only page rows carry the page body)", sectionBody)
+ }
}
// TestMain installs the stub resolvers for the variant unit tests.
diff --git a/internal/ingestion/component/knowledge_compiler/golden/golden_test.go b/internal/ingestion/component/knowledge_compiler/golden/golden_test.go
index 8c55674021..5e1d4a522e 100644
--- a/internal/ingestion/component/knowledge_compiler/golden/golden_test.go
+++ b/internal/ingestion/component/knowledge_compiler/golden/golden_test.go
@@ -48,20 +48,20 @@ func TestAnalyzeTreeProducts_TreeShape(t *testing.T) {
vector := json.RawMessage(`[0.1,0.2,0.3]`)
chunks := []schema.ChunkDoc{
{Text: "root summary", Extra: mustExtras(t, map[string]any{
- "id": "r1", "doc_id": "d1", "tenant_id": "t1", "compile_kwd": "tree",
- "kc_kind": "root", "kc_level": float64(-1), "q_3_vec": vector,
+ "id": "r1", "doc_id": "d1", "compile_kwd": "tree",
+ "raptor_kwd": "root", "raptor_layer_int": float64(-1), "q_3_vec": vector,
})},
{Text: "leaf A", Extra: mustExtras(t, map[string]any{
- "id": "a1", "doc_id": "d1", "tenant_id": "t1", "compile_kwd": "tree",
- "kc_kind": "summary", "kc_level": float64(0), "parent_kwd": "r1", "q_3_vec": vector,
+ "id": "a1", "doc_id": "d1", "compile_kwd": "tree",
+ "raptor_kwd": "summary", "raptor_layer_int": float64(0), "parent_kwd": "r1", "q_3_vec": vector,
})},
{Text: "leaf B", Extra: mustExtras(t, map[string]any{
- "id": "b1", "doc_id": "d1", "tenant_id": "t1", "compile_kwd": "tree",
- "kc_kind": "summary", "kc_level": float64(0), "parent_kwd": "r1", "q_3_vec": vector,
+ "id": "b1", "doc_id": "d1", "compile_kwd": "tree",
+ "raptor_kwd": "summary", "raptor_layer_int": float64(0), "parent_kwd": "r1", "q_3_vec": vector,
})},
{Text: "mid A", Extra: mustExtras(t, map[string]any{
- "id": "m1", "doc_id": "d1", "tenant_id": "t1", "compile_kwd": "tree",
- "kc_kind": "summary", "kc_level": float64(1), "parent_kwd": "r1", "q_3_vec": vector,
+ "id": "m1", "doc_id": "d1", "compile_kwd": "tree",
+ "raptor_kwd": "summary", "raptor_layer_int": float64(1), "parent_kwd": "r1", "q_3_vec": vector,
})},
}
m := AnalyzeTreeProducts(chunks)
@@ -85,12 +85,12 @@ func TestAnalyzeTreeProducts_TreeShape(t *testing.T) {
func TestAnalyzeTreeProducts_DetectsDanglingParent(t *testing.T) {
chunks := []schema.ChunkDoc{
{Text: "root", Extra: mustExtras(t, map[string]any{
- "id": "r1", "doc_id": "d1", "tenant_id": "t1", "compile_kwd": "tree",
- "kc_kind": "root",
+ "id": "r1", "doc_id": "d1", "compile_kwd": "tree",
+ "raptor_kwd": "root",
})},
{Text: "orphan", Extra: mustExtras(t, map[string]any{
- "id": "o1", "doc_id": "d1", "tenant_id": "t1", "compile_kwd": "tree",
- "kc_kind": "summary", "kc_level": float64(0), "parent_kwd": "missing-parent",
+ "id": "o1", "doc_id": "d1", "compile_kwd": "tree",
+ "raptor_kwd": "summary", "raptor_layer_int": float64(0), "parent_kwd": "missing-parent",
})},
}
m := AnalyzeTreeProducts(chunks)
@@ -134,12 +134,12 @@ func TestAnalyzeTreeProducts_IgnoresUnknownSourceIDs(t *testing.T) {
vector := json.RawMessage(`[0.1,0.2,0.3]`)
chunks := []schema.ChunkDoc{
{Text: "root", Extra: mustExtras(t, map[string]any{
- "id": "r1", "doc_id": "d1", "tenant_id": "t1", "compile_kwd": "tree",
- "kc_kind": "root", "kc_level": float64(-1), "q_3_vec": vector,
+ "id": "r1", "doc_id": "d1", "compile_kwd": "tree",
+ "raptor_kwd": "root", "raptor_layer_int": float64(-1), "q_3_vec": vector,
})},
{Text: "leaf", Extra: mustExtras(t, map[string]any{
- "id": "a1", "doc_id": "d1", "tenant_id": "t1", "compile_kwd": "tree",
- "kc_kind": "summary", "kc_level": float64(0), "parent_kwd": "r1", "q_3_vec": vector,
+ "id": "a1", "doc_id": "d1", "compile_kwd": "tree",
+ "raptor_kwd": "summary", "raptor_layer_int": float64(0), "parent_kwd": "r1", "q_3_vec": vector,
"source_chunk_ids": []string{"chunk-01", "chunk-02", "leaked-unknown-id"},
})},
}
@@ -153,8 +153,8 @@ func TestAnalyzeTreeProducts_IgnoresUnknownSourceIDs(t *testing.T) {
}
func TestExtraFloat_Roundtrip(t *testing.T) {
- doc := schema.ChunkDoc{Extra: mustExtras(t, map[string]any{"kc_level": float64(3)})}
- if v, ok := extraFloat(doc, "kc_level"); !ok || v != 3 {
+ doc := schema.ChunkDoc{Extra: mustExtras(t, map[string]any{"raptor_layer_int": float64(3)})}
+ if v, ok := extraFloat(doc, "raptor_layer_int"); !ok || v != 3 {
t.Errorf("extraFloat = %v/%v, want 3/true", v, ok)
}
if _, ok := extraFloat(doc, "missing"); ok {
diff --git a/internal/ingestion/component/knowledge_compiler/golden/metrics.go b/internal/ingestion/component/knowledge_compiler/golden/metrics.go
index 27224bb628..02bec14b27 100644
--- a/internal/ingestion/component/knowledge_compiler/golden/metrics.go
+++ b/internal/ingestion/component/knowledge_compiler/golden/metrics.go
@@ -50,7 +50,8 @@ func AnalyzeTreeProducts(chunks []schema.ChunkDoc, validSourceIDs ...string) Tre
m := TreeMetrics{AllParented: true, VectorOK: true, SchemaOK: true, covered: make(map[string]bool)}
maxLevel := -1
for _, c := range chunks {
- kind, _ := c.GetExtraString("kc_kind")
+ // raptor_kwd / raptor_layer_int are the tree row's kind and depth.
+ kind, _ := c.GetExtraString("raptor_kwd")
// Auxiliary tree-graph rows (kind entity/relation/graph, from the
// structure-graph projection) are not part of the RAPTOR tree structure,
// so they are excluded from the structural metrics (ProductCount,
@@ -60,7 +61,7 @@ func AnalyzeTreeProducts(chunks []schema.ChunkDoc, validSourceIDs ...string) Tre
}
m.ProductCount++
level := 0
- if lf, ok := extraFloat(c, "kc_level"); ok {
+ if lf, ok := extraFloat(c, "raptor_layer_int"); ok {
level = int(lf)
}
switch kind {
@@ -102,9 +103,8 @@ func AnalyzeTreeProducts(chunks []schema.ChunkDoc, validSourceIDs ...string) Tre
}
id, _ := c.GetExtraString("id")
docID, _ := c.GetExtraString("doc_id")
- tenant, _ := c.GetExtraString("tenant_id")
ck, _ := c.GetExtraString("compile_kwd")
- if id == "" || docID == "" || tenant == "" || c.Text == "" || ck == "" {
+ if id == "" || docID == "" || c.Text == "" || ck == "" {
m.SchemaOK = false
}
}
diff --git a/internal/ingestion/component/knowledge_compiler/golden_test.go b/internal/ingestion/component/knowledge_compiler/golden_test.go
index 544ab7cfaa..cbde245b01 100644
--- a/internal/ingestion/component/knowledge_compiler/golden_test.go
+++ b/internal/ingestion/component/knowledge_compiler/golden_test.go
@@ -153,8 +153,7 @@ func TestGolden_Structure_ProductCount(t *testing.T) {
for _, p := range prods {
id, _ := p.GetExtraString("id")
docID, _ := p.GetExtraString("doc_id")
- tenant, _ := p.GetExtraString("tenant_id")
- if id == "" || docID == "" || tenant == "" || p.Text == "" {
+ if id == "" || docID == "" || p.Text == "" {
t.Fatalf("structure: product missing schema fields: %+v", p)
}
if !chunkHasVector(p) {
@@ -166,7 +165,8 @@ func TestGolden_Structure_ProductCount(t *testing.T) {
// compact graph blob was removed by #19474).
graphCount, nodeCount, edgeCount := 0, 0, 0
for _, p := range prods {
- kind, _ := p.GetExtraString("kc_kind")
+ // knowledge_graph_kwd discriminates the structure graph rows.
+ kind, _ := p.GetExtraString("knowledge_graph_kwd")
switch kind {
case "graph":
graphCount++
@@ -199,8 +199,7 @@ func TestGolden_Wiki_ProductCount(t *testing.T) {
for _, p := range prods {
id, _ := p.GetExtraString("id")
docID, _ := p.GetExtraString("doc_id")
- tenant, _ := p.GetExtraString("tenant_id")
- if id == "" || docID == "" || tenant == "" || p.Text == "" {
+ if id == "" || docID == "" || p.Text == "" {
t.Fatalf("wiki: product missing schema fields: %+v", p)
}
if !chunkHasVector(p) {
@@ -210,7 +209,8 @@ func TestGolden_Wiki_ProductCount(t *testing.T) {
foundPage := false
for _, p := range prods {
- if kind, _ := p.GetExtraString("kc_kind"); kind == "page" {
+ // compile_kwd=wiki_page is the page/section discriminator.
+ if kwd, _ := p.GetExtraString("compile_kwd"); kwd == "wiki_page" {
foundPage = true
}
}
diff --git a/internal/ingestion/knowledge_compile/reader.go b/internal/ingestion/knowledge_compile/reader.go
index 43c2a4df9c..81123c81f5 100644
--- a/internal/ingestion/knowledge_compile/reader.go
+++ b/internal/ingestion/knowledge_compile/reader.go
@@ -97,24 +97,17 @@ type engineReader struct {
eng engine.DocEngine
}
-// compiledSelectFields are the columns needed to reconstruct a Product from a
-// stored compiled chunk document.
-//
-// wiki_incremental port: the list also selects `kc_kind` and
-// `create_timestamp_flt` (+`create_time`) so the reader can round-trip the
-// wiki product kind (page/section) and the original creation timestamp without
-// re-deriving them (see productFromChunkMap). Without these in the SELECT list,
-// the stored values are invisible to the reader and every merged row would fall
-// back to the compile_kwd-derived kind / a fresh now() timestamp — which both
-// breaks the page/section filter and re-stamps creation time on every rebuild.
+// compiledSelectFields are the columns needed to reconstruct a Product. Every
+// name must exist in the target engine schema (Infinity rejects a SELECT of an
+// unknown column); payload = content_with_weight, kind = compile_kwd.
var compiledSelectFields = []string{
- "id", "doc_id", "tenant_id", "compile_kwd",
+ "id", "doc_id", "compile_kwd",
"available_int",
- "content_with_weight", "kc_payload",
+ "content_with_weight",
"source_chunk_ids", "source_doc_ids",
"name_kwd", "entity_type_kwd", "from_entity_kwd", "to_entity_kwd",
- "slug_kwd", "type",
- "kc_kind", "create_timestamp_flt", "create_time",
+ "slug_kwd",
+ "create_timestamp_flt", "create_time",
"compilation_template_kind_kwd", "compilation_template_ids",
// The product kind discriminator for structure/tree: the component stores it
// under knowledge_graph_kwd (structure: graph/entity/relation) / raptor_kwd
@@ -141,7 +134,7 @@ var compiledSelectFields = []string{
var wikiSelectFields = []string{
"page_type_kwd", "topic_kwd", "plan_group_kwd", "generation_kwd", "title_kwd",
"entity_names_kwd", "summary_with_weight",
- "related_kb_pages_kwd", "outlinks_kwd", "section_level_int",
+ "related_kb_pages_kwd", "outlinks_kwd", "depth_int",
"q_*_vec",
}
@@ -331,8 +324,8 @@ func (r engineReader) LoadDocumentWikiPagesBySlugs(ctx context.Context, tenant,
}
// productFromChunkMap reconstructs a kccommon.Product from a stored compiled
-// chunk document. It reads the payload from kc_payload (falling back to
-// content_with_weight) and the embedding from the q_<dim>_vec column.
+// chunk document. It reads the payload from content_with_weight and the
+// embedding from the q_<dim>_vec column.
//
// expect is the variant the caller is querying for. The stored compile_kwd is
// reverse-mapped via KwdToVariant and compared against expect; a mismatch (or
@@ -341,10 +334,7 @@ func (r engineReader) LoadDocumentWikiPagesBySlugs(ctx context.Context, tenant,
// the plan: we never rely on the raw string equality alone, so unknown kinds
// are rejected consistently rather than leaking into the wrong bucket.
func productFromChunkMap(c map[string]interface{}, tenant string, expect kccommon.Variant) (kccommon.Product, bool) {
- content, _ := c["kc_payload"].(string)
- if content == "" {
- content, _ = c["content_with_weight"].(string)
- }
+ content, _ := c["content_with_weight"].(string)
if content == "" {
return kccommon.Product{}, false
}
@@ -421,11 +411,13 @@ func productFromChunkMap(c map[string]interface{}, tenant string, expect kccommo
if v := metaStringSlice(c, "outlinks_kwd"); len(v) > 0 {
meta["outlinks"] = v
}
- if v, ok := metaInt(c, "section_level_int"); ok {
- meta["section_level"] = v
- }
- if v, ok := c["type"].(string); ok && v != "" {
- meta["type"] = v
+ // Section depth comes from depth_int. Only the wiki variant consumes
+ // Meta["section_level"] (applyVariantColumns); tree rows also stamp depth_int,
+ // but they keep their depth in raptor_layer_int.
+ if expect == kccommon.VariantWiki {
+ if v, ok := metaInt(c, "depth_int"); ok {
+ meta["section_level"] = v
+ }
}
// Restore the structure/tree product kind. The component stores it under
// knowledge_graph_kwd (structure: graph/entity/relation) / raptor_kwd (tree:
@@ -442,16 +434,10 @@ func productFromChunkMap(c map[string]interface{}, tenant string, expect kccommo
meta["kind"] = "entity"
}
}
- // wiki_incremental port: round-trip the wiki product kind so the
- // dataset-level merge can reliably distinguish pages from sections. The
- // merged writer stores kc_kind; when present it is authoritative. Without
- // it (legacy rows), derive from compile_kwd: wiki_page -> "page",
- // wiki_section -> "section". This fix is what stops the processBatch
- // "Meta.kind==page" filter from deleting every wiki page (previously kind
- // was empty for wiki pages that had no entity/relation endpoint).
- if v := asString(c["kc_kind"]); v != "" {
- meta["kind"] = v
- } else if variant == compileKwdWikiPage {
+ // compile_kwd IS the page/section discriminator (there is no kind column):
+ // wiki_page -> "page", wiki_section -> "section". Without it the
+ // processBatch "Meta.kind==page" filter would drop every wiki page.
+ if variant == compileKwdWikiPage {
meta["kind"] = "page"
} else if variant == compileKwdWikiSection {
meta["kind"] = "section"
@@ -525,10 +511,10 @@ func (r engineReader) SearchSimilar(ctx context.Context, tenant, kb string, vari
IndexNames: []string{fmt.Sprintf("ragflow_%s", tenant)},
KbIDs: []string{kb},
Limit: topN,
- SelectFields: append([]string{"id", "doc_id", "kb_id", "content_with_weight", "kc_payload",
+ SelectFields: append([]string{"id", "doc_id", "kb_id", "content_with_weight",
"name_kwd", "entity_type_kwd", "from_entity_kwd", "to_entity_kwd", "slug_kwd",
- "type", "source_chunk_ids", "source_doc_ids", "available_int", "compile_kwd",
- "kc_kind", "create_timestamp_flt", "create_time"},
+ "source_chunk_ids", "source_doc_ids", "available_int", "compile_kwd",
+ "create_timestamp_flt", "create_time"},
wikiSelectFields...),
Filter: map[string]interface{}{
"available_int": 1,
diff --git a/internal/ingestion/knowledge_compile/reader_test.go b/internal/ingestion/knowledge_compile/reader_test.go
index 17da6c06e3..ae62dffa0a 100644
--- a/internal/ingestion/knowledge_compile/reader_test.go
+++ b/internal/ingestion/knowledge_compile/reader_test.go
@@ -51,7 +51,7 @@ func TestWikiReadersFilterDisabledDocuments(t *testing.T) {
eng := &fakeEngine{searchChunks: []map[string]interface{}{{
"id": "page-1", "doc_id": "active-doc", "compile_kwd": "wiki_page",
- "available_int": 0, "kc_payload": "page content", "slug_kwd": "entity/page",
+ "available_int": 0, "content_with_weight": "page content", "slug_kwd": "entity/page",
"page_type_kwd": "entity", "title_kwd": "page", "source_doc_ids": []string{"active-doc"},
}}}
r := engineReader{eng: eng}
@@ -82,21 +82,20 @@ func TestSearchSimilarFiltersByVariant(t *testing.T) {
searchChunks: []map[string]interface{}{
// dirty row: wrong variant (must be skipped by the in-memory guard).
{
- "id": "dirty",
- "doc_id": "kb",
- "available_int": 1,
- "compile_kwd": "wiki_section", // not wiki_page
- "kc_payload": "{\"c\":1}",
- "_score": 0.99,
+ "id": "dirty",
+ "doc_id": "kb",
+ "available_int": 1,
+ "compile_kwd": "wiki_section", // not wiki_page
+ "content_with_weight": "{\"c\":1}",
+ "_score": 0.99,
},
- // good row: correct variant + kc_kind round-trip.
+ // good row: correct variant (wiki_page => kind "page").
{
"id": "page1",
"doc_id": "kb",
"available_int": 1,
"compile_kwd": "wiki_page",
- "kc_kind": "page",
- "kc_payload": "{\"c\":1}",
+ "content_with_weight": "{\"c\":1}",
"create_timestamp_flt": 1700000000.0,
"create_time": "2023-11-14T22:13:20Z",
"_score": 0.95,
@@ -113,7 +112,7 @@ func TestSearchSimilarFiltersByVariant(t *testing.T) {
t.Fatalf("expected the wiki_page row to win, got %q", p.ID)
}
if p.Meta["kind"] != "page" {
- t.Fatalf("expected kc_kind round-trip to 'page', got %v", p.Meta["kind"])
+ t.Fatalf("expected compile_kwd=wiki_page to map to kind 'page', got %v", p.Meta["kind"])
}
if p.Merged != true {
t.Fatalf("expected merged=true for available_int=1 row")
@@ -136,7 +135,7 @@ func TestSearchSimilarFiltersByVariant(t *testing.T) {
t.Fatalf("expected compile_kwd=%q filter, got %v", compileKwdWikiPage, eng.lastSearchReq.Filter["compile_kwd"])
}
// SelectFields must include the round-trip columns added in the 8th review.
- for _, want := range []string{"kc_kind", "create_timestamp_flt", "create_time"} {
+ for _, want := range []string{"compile_kwd", "content_with_weight", "create_timestamp_flt", "create_time"} {
found := false
for _, f := range eng.lastSearchReq.SelectFields {
if f == want {
@@ -156,12 +155,12 @@ func TestSearchSimilarSkipsNonMerged(t *testing.T) {
eng := &fakeEngine{
searchChunks: []map[string]interface{}{
{
- "id": "doc1",
- "doc_id": "doc",
- "available_int": 0,
- "compile_kwd": "wiki_page",
- "kc_payload": "{\"c\":1}",
- "_score": 0.99,
+ "id": "doc1",
+ "doc_id": "doc",
+ "available_int": 0,
+ "compile_kwd": "wiki_page",
+ "content_with_weight": "{\"c\":1}",
+ "_score": 0.99,
},
},
}
diff --git a/internal/ingestion/knowledge_compile/structure_merge_test.go b/internal/ingestion/knowledge_compile/structure_merge_test.go
index 586b73ef06..8da59fe663 100644
--- a/internal/ingestion/knowledge_compile/structure_merge_test.go
+++ b/internal/ingestion/knowledge_compile/structure_merge_test.go
@@ -338,7 +338,7 @@ func TestProductFromChunkMap_RestoresStructureTreeKind(t *testing.T) {
"id": "s1",
"doc_id": "d1",
"compile_kwd": "list",
- "kc_payload": `{"entities":[]}`,
+ "content_with_weight": `{"entities":[]}`,
"knowledge_graph_kwd": "graph",
"name_kwd": "engine",
"source_chunk_ids": []interface{}{"c1"},
@@ -363,7 +363,7 @@ func TestProductFromChunkMap_RestoresStructureTreeKind(t *testing.T) {
"id": "t1",
"doc_id": "d1",
"compile_kwd": "tree",
- "kc_payload": "overall summary",
+ "content_with_weight": "overall summary",
"raptor_kwd": "root",
"source_chunk_ids": []interface{}{"c1"},
"source_doc_ids": []interface{}{"d1"},
diff --git a/internal/ingestion/knowledge_compile/wiki_contribution.go b/internal/ingestion/knowledge_compile/wiki_contribution.go
index ebca83b223..ff3ba9fa8e 100644
--- a/internal/ingestion/knowledge_compile/wiki_contribution.go
+++ b/internal/ingestion/knowledge_compile/wiki_contribution.go
@@ -99,7 +99,6 @@ func (s *engineWikiContributionStore) Put(ctx context.Context, tenantID, dataset
row := map[string]any{
"id": wikiContributionStateID(datasetID, contribution.DocumentID),
"doc_id": "wiki_contribution:" + contribution.DocumentID,
- "tenant_id": tenantID,
"kb_id": datasetID,
"compile_kwd": wikiContributionCompileKWD,
"scope_kwd": "doc",
diff --git a/internal/ingestion/knowledge_compile/wiki_graph_test.go b/internal/ingestion/knowledge_compile/wiki_graph_test.go
index deb3c537e8..fe0d516a88 100644
--- a/internal/ingestion/knowledge_compile/wiki_graph_test.go
+++ b/internal/ingestion/knowledge_compile/wiki_graph_test.go
@@ -65,12 +65,20 @@ func TestProjectWikiGraphRowsWritesSearchContent(t *testing.T) {
if entity == nil {
t.Fatal("missing entity")
}
- for _, field := range []string{"title_tks", "title_sm_tks", "content_ltks", "content_sm_ltks"} {
+ // Python tokenizes slug + description into content_ltks and writes no title
+ // token column on the entity row (dataset_wiki_generator.py:750-773); the
+ // row's title stays in title_kwd / the payload.
+ for _, field := range []string{"content_ltks", "content_sm_ltks"} {
value, ok := entity[field].(string)
if !ok || strings.TrimSpace(value) == "" {
t.Fatalf("entity[%q] = %#v, want non-empty search content", field, entity[field])
}
}
+ for _, field := range []string{"title_tks", "title_sm_tks"} {
+ if _, ok := entity[field]; ok {
+ t.Fatalf("entity row must not carry %q (Python writes no title token column)", field)
+ }
+ }
}
// TestProjectWikiGraphRowsCrossBatchEdgeSurvives verifies a relation whose two
@@ -217,9 +225,9 @@ func TestProjectWikiGraphRowsDropsDanglingAndSelfLoop(t *testing.T) {
}
}
-// TestProjectWikiGraphRowsFieldMapping verifies the reader-facing display
-// columns are written (aliases_kwd from entity_names_kwd, description_with_weight
-// from summary_with_weight), and the id is the xxhash colon-namespaced form.
+// TestProjectWikiGraphRowsFieldMapping verifies the entity row matches Python's
+// shape (dataset_wiki_generator.py:759-776): display fields in the
+// content_with_weight payload, schema columns only, xxhash id.
func TestProjectWikiGraphRowsFieldMapping(t *testing.T) {
w := engineWriter{}
pages := []wikiPageProjection{
@@ -244,12 +252,6 @@ func TestProjectWikiGraphRowsFieldMapping(t *testing.T) {
if e == nil {
t.Fatalf("missing entity")
}
- if e["aliases_kwd"] == nil {
- t.Fatalf("expected aliases_kwd (mapped from entity_names_kwd) to be set")
- }
- if e["description_with_weight"] != "the alpha page" {
- t.Fatalf("expected description_with_weight=summary, got %v", e["description_with_weight"])
- }
if e["entity_type_kwd"] != "wiki_concept" {
t.Fatalf("expected entity_type_kwd=wiki_concept, got %v", e["entity_type_kwd"])
}
@@ -263,14 +265,27 @@ func TestProjectWikiGraphRowsFieldMapping(t *testing.T) {
if id != wantID {
t.Fatalf("id mismatch: got %q want %q", id, wantID)
}
- // content_with_weight JSON must round-trip the slug/page_type/title/aliases/summary/weight.
+ // content_with_weight JSON is the canvas payload (Python keys), the
+ // authoritative source for name/aliases/description/type.
raw, _ := e["content_with_weight"].(string)
var decoded map[string]any
if err := json.Unmarshal([]byte(raw), &decoded); err != nil {
t.Fatalf("content_with_weight not valid JSON: %v", err)
}
- if decoded["slug"] != "concept/alpha" || decoded["page_type"] != "concept" {
- t.Fatalf("content_with_weight missing slug/page_type: %v", decoded)
+ if decoded["slug"] != "concept/alpha" || decoded["type"] != "concept" {
+ t.Fatalf("content_with_weight missing slug/type: %v", decoded)
+ }
+ if decoded["name"] != "Alpha" || decoded["description"] != "the alpha page" {
+ t.Fatalf("content_with_weight missing name/description: %v", decoded)
+ }
+ if aliases, _ := decoded["aliases"].([]any); len(aliases) != 2 || aliases[0] != "A" {
+ t.Fatalf("content_with_weight aliases = %v, want [A Alpha Prime]", decoded["aliases"])
+ }
+ // No Go-only field: none of these exists in an engine mapping.
+ for _, forbidden := range []string{"aliases_kwd", "description_with_weight", "tenant_id", "from_id", "to_id"} {
+ if _, ok := e[forbidden]; ok {
+ t.Fatalf("entity row must not carry the non-schema field %q", forbidden)
+ }
}
}
diff --git a/internal/ingestion/knowledge_compile/wiki_map_store.go b/internal/ingestion/knowledge_compile/wiki_map_store.go
index 8ac6bc9bd3..6d72e3e843 100644
--- a/internal/ingestion/knowledge_compile/wiki_map_store.go
+++ b/internal/ingestion/knowledge_compile/wiki_map_store.go
@@ -76,7 +76,6 @@ func (s *wikiMapVersionStore) PutWikiMapActiveState(ctx context.Context, state k
row := map[string]interface{}{
"id": state.Key,
"doc_id": "wiki_map_active:" + state.DocumentID,
- "tenant_id": state.TenantID,
"kb_id": state.DatasetID,
"compile_kwd": wikiMapActiveCompileKWD,
"scope_kwd": "doc",
@@ -88,13 +87,6 @@ func (s *wikiMapVersionStore) PutWikiMapActiveState(ctx context.Context, state k
return err
}
-// NewWikiMapVersionStore returns the DocStore-backed immutable Wiki MAP cache.
-// Its rows remain non-searchable through available_int=0 and the compile
-// discriminator, while preserving every chunk/hash version for reuse.
-func NewWikiMapVersionStore(docEngine engine.DocEngine) kccommon.WikiMapVersionStore {
- return &wikiMapVersionStore{engine: docEngine}
-}
-
// NewWikiMapVersionStoreWithVectorSizeResolver creates a Wiki MAP store that
// can initialize an Infinity chunk table before inserting metadata-only MAP
// rows. Elasticsearch does not need this resolver because it can create its
@@ -236,7 +228,6 @@ func wikiMapVersionRow(version kccommon.WikiMapVersion) map[string]interface{} {
// Keep immutable MAP history in a separate document namespace so source
// document deletion cannot remove a reusable chunk/hash version.
"doc_id": wikiMapCacheDocID(version.DocumentID),
- "tenant_id": version.TenantID,
"kb_id": version.DatasetID,
"compile_kwd": wikiMapExtractCompileKWD,
"scope_kwd": "doc",
diff --git a/internal/ingestion/knowledge_compile/wiki_map_store_test.go b/internal/ingestion/knowledge_compile/wiki_map_store_test.go
index 6b8d521f9e..1d777c231f 100644
--- a/internal/ingestion/knowledge_compile/wiki_map_store_test.go
+++ b/internal/ingestion/knowledge_compile/wiki_map_store_test.go
@@ -120,7 +120,7 @@ func (e *wikiMapStoreEngine) InsertChunks(_ context.Context, chunks []map[string
func TestWikiMapVersionStoreUsesNonSearchableDocStoreRows(t *testing.T) {
engine := &wikiMapStoreEngine{rows: map[string]map[string]interface{}{}}
- store := NewWikiMapVersionStore(engine)
+ store := NewWikiMapVersionStoreWithVectorSizeResolver(engine, nil)
version := kccommon.WikiMapVersion{
Key: "version-a",
TenantID: "tenant-1",
@@ -228,7 +228,7 @@ func TestWikiMapVersionStoreCreatesMissingInfinityStore(t *testing.T) {
func TestWikiMapActiveStateRequiresResolverOnlyWhenInfinityStoreMissing(t *testing.T) {
engine := &wikiMapStoreEngine{rows: map[string]map[string]interface{}{}, engineType: "infinity"}
- store := NewWikiMapVersionStore(engine).(kccommon.WikiMapActiveStateStore)
+ store := NewWikiMapVersionStoreWithVectorSizeResolver(engine, nil).(kccommon.WikiMapActiveStateStore)
state := kccommon.WikiMapActiveState{
Key: "active-a", TenantID: "tenant-1", DatasetID: "kb-1",
DocumentID: "doc-1", Payload: []byte(`{}`),
diff --git a/internal/ingestion/knowledge_compile/writer.go b/internal/ingestion/knowledge_compile/writer.go
index b5a13744c6..3efbae47c2 100644
--- a/internal/ingestion/knowledge_compile/writer.go
+++ b/internal/ingestion/knowledge_compile/writer.go
@@ -538,13 +538,11 @@ func (w engineWriter) WriteMergedStructure(ctx context.Context, tenant, kb strin
// relation identity plus the raw compile kind.
"id": bid,
"doc_id": kb,
- "tenant_id": tenant,
"kb_id": kb,
"available_int": 1,
"compile_kwd": ckwd,
"scope_kwd": "dataset",
"content_with_weight": payload,
- "kc_payload": payload,
"source_doc_ids": b.SourceDocIDs,
"source_chunk_ids": b.SourceChunkIDs,
"create_time": now.Format("2006-01-02 15:04:05"),
@@ -601,7 +599,6 @@ func (w engineWriter) WriteMergedStructure(ctx context.Context, tenant, kb strin
rows = append(rows, map[string]interface{}{
"id": "dataset_build_meta_" + hashStr(tenant+"\x00"+kb+"\x00"+ckwd),
"doc_id": kb,
- "tenant_id": tenant,
"kb_id": kb,
"available_int": 0,
"compile_kwd": ckwd,
@@ -658,7 +655,7 @@ func mergeExistingStructureBuckets(ctx context.Context, eng engine.DocEngine, ba
SelectFields: []string{
"id", "compile_kwd", "compilation_template_ids", "compilation_template_kind_kwd",
"knowledge_graph_kwd", "name_kwd", "entity_type_kwd", "from_entity_kwd", "to_entity_kwd",
- "content_with_weight", "kc_payload", "source_doc_ids", "source_chunk_ids",
+ "content_with_weight", "source_doc_ids", "source_chunk_ids",
},
Filter: map[string]interface{}{
"kb_id": kb,
@@ -765,11 +762,9 @@ func structureRowIdentity(row map[string]interface{}) string {
}
func structureRowPayload(row map[string]interface{}) map[string]interface{} {
- for _, field := range []string{"kc_payload", "content_with_weight"} {
- value := structureString(row[field])
- if value == "" {
- continue
- }
+ // The payload is content_with_weight verbatim (Python json.dumps's it there);
+ // there is no separate payload column.
+ if value := structureString(row["content_with_weight"]); value != "" {
var payload map[string]interface{}
if json.Unmarshal([]byte(value), &payload) == nil {
return payload
@@ -959,7 +954,6 @@ func mergedChunkMap(tenant, kb, runID, inputHash string, now time.Time, p kccomm
m := map[string]interface{}{
"id": datasetLevelID(tenant, kb, p),
"doc_id": kb,
- "tenant_id": tenant,
"kb_id": kb,
"available_int": 1,
// scope_kwd marks this row as dataset-level (O1=B). It is the unified
@@ -969,7 +963,6 @@ func mergedChunkMap(tenant, kb, runID, inputHash string, now time.Time, p kccomm
"scope_kwd": "dataset",
"compile_kwd": compileKwdForVariant(p.Variant),
"content_with_weight": p.Content,
- "kc_payload": p.Content, // raw payload, for Reader reconstruction
"source_doc_ids": srcDocIDs,
"source_chunk_ids": srcChunkIDs,
"plan_kwd": runID,
@@ -977,16 +970,9 @@ func mergedChunkMap(tenant, kb, runID, inputHash string, now time.Time, p kccomm
"create_time": now.Format("2006-01-02 15:04:05"),
"create_timestamp_flt": float64(now.Unix()),
}
- // wiki_incremental port: persist the product kind so the Reader can round-trip
- // page vs section without re-deriving it from compile_kwd. The merged writer
- // carries the authoritative kc_kind; legacy rows without it are derived in
- // productFromChunkMap (compile_kwd wiki_page -> "page", wiki_section ->
- // "section"). Without this, the dataset-level merge could not distinguish a
- // wiki page from a section and the processBatch "Meta.kind==page" filter would
- // be unreliable.
- if kind := metaString(p.Meta, "kind"); kind != "" {
- m["kc_kind"] = kind
- }
+ // The product kind (page vs section) is carried by compile_kwd
+ // (wiki_page / wiki_section), which is what Python stores and what
+ // productFromChunkMap derives Meta["kind"] from — there is no kind column.
// wiki_incremental port: preserve the original creation timestamp across a
// page merge. If the incoming merged product already carries
// created_at_unix (restored by the Reader from create_timestamp_flt), reuse
@@ -1015,7 +1001,6 @@ func mergedChunkMap(tenant, kb, runID, inputHash string, now time.Time, p kccomm
fullSlug = pageType + "/" + slug
}
m["slug_kwd"] = fullSlug
- m["artifact_slug_kwd"] = fullSlug
}
if v := metaString(p.Meta, "title"); v != "" {
m["title_kwd"] = v
@@ -1035,6 +1020,11 @@ func mergedChunkMap(tenant, kb, runID, inputHash string, now time.Time, p kccomm
if v := metaString(p.Meta, "summary"); v != "" {
m["summary_with_weight"] = v
}
+ // The merged wiki page keeps the page body in md_with_weight (the column
+ // GetWikiPage / the artifact API read), as Python does; sections have none.
+ if p.Variant == kccommon.VariantWiki && metaString(p.Meta, "kind") == "page" && p.Content != "" {
+ m["md_with_weight"] = p.Content
+ }
if v := metaStringSlice(p.Meta, "entity_names"); len(v) > 0 {
m["entity_names_kwd"] = v
}
@@ -1299,7 +1289,7 @@ func metaFloat(m map[string]any, key string) (float64, bool) {
// KwdToVariant is the inverse of compileKwdForVariant: it maps a stored
// compile_kwd back to its compiler Variant. Both wiki_page and wiki_section
// map to VariantWiki (same product family); the page/section distinction is
-// carried by the kc_kind field, not the variant. Returns an error for an
+// carried by compile_kwd itself, not the variant. Returns an error for an
// unknown kwd so callers can reject dirty/foreign rows. Structure products
// stamp the inferred compile kind verbatim (list/set/hypergraph), which are NOT
// in the KindToVariant whitelist, so they are mapped to VariantStructure
@@ -1386,24 +1376,15 @@ func wikiGraphBareKey(slug string) string {
return strings.TrimSpace(s)
}
-// tokenizeWikiGraphContent prepares the lexical fields used by wiki graph
-// keyword search. Keep the raw text as a fallback so graph rows remain
-// searchable when the tokenizer pool is unavailable during startup or tests.
-func tokenizeWikiGraphContent(title, slug, description string) (string, string, string, string) {
- title = strings.TrimSpace(title)
+// tokenizeWikiGraphContent tokenizes Python's entity search text,
+// `slug + " " + description` (dataset_wiki_generator.py:750-773); the entity row
+// gets no title token column.
+func tokenizeWikiGraphContent(slug, description string) (string, string) {
content := strings.TrimSpace(strings.Join([]string{slug, description}, " "))
- if title == "" && content == "" {
- return "", "", "", ""
+ if content == "" {
+ return "", ""
}
- titleLTKS, err := tokenizer.Tokenize(title)
- if err != nil || titleLTKS == "" {
- titleLTKS = title
- }
- titleSMLTKS, err := tokenizer.FineGrainedTokenize(titleLTKS)
- if err != nil || titleSMLTKS == "" {
- titleSMLTKS = titleLTKS
- }
contentLTKS, err := tokenizer.Tokenize(content)
if err != nil || contentLTKS == "" {
contentLTKS = content
@@ -1412,7 +1393,7 @@ func tokenizeWikiGraphContent(title, slug, description string) (string, string,
if err != nil || contentSMLTKS == "" {
contentSMLTKS = contentLTKS
}
- return titleLTKS, titleSMLTKS, contentLTKS, contentSMLTKS
+ return contentLTKS, contentSMLTKS
}
// wikiPageProjection is the subset of a merged wiki_page row that the graph
@@ -1829,39 +1810,37 @@ func (w engineWriter) projectWikiGraphRows(_ context.Context, tenant, kb string,
seen[entityID] = true
weight := len(p.Outlinks) // raw outlink count; 0 is allowed (Python parity)
+ // Python's canvas payload (dataset_wiki_generator.py:751-758): aliases /
+ // description / type live inside it. No such columns exist, and Infinity
+ // rejects an insert naming one.
content, err := json.Marshal(map[string]any{
- "slug": p.Slug,
- "page_type": p.PageType,
- "title": p.Title,
- "aliases": p.Aliases,
- "summary": p.Summary,
- "weight": weight,
+ "slug": p.Slug,
+ "name": p.Title,
+ "aliases": p.Aliases,
+ "description": p.Summary,
+ "type": p.PageType,
+ "weight": weight,
})
if err != nil {
return nil, err
}
- titleLTKS, titleSMLTKS, contentLTKS, contentSMLTKS := tokenizeWikiGraphContent(p.Title, p.Slug, p.Summary)
+ contentLTKS, contentSMLTKS := tokenizeWikiGraphContent(p.Slug, p.Summary)
rows = append(rows, map[string]interface{}{
- "id": entityID,
- "doc_id": kb,
- "tenant_id": tenant,
- "kb_id": kb,
- "available_int": 1,
- "compile_kwd": compileKwdWikiEntity,
- "type_kwd": "wiki_" + p.PageType,
- "entity_type_kwd": "wiki_" + p.PageType,
- "slug_kwd": p.Slug,
- "title_kwd": p.Title,
- "aliases_kwd": p.Aliases,
- "description_with_weight": p.Summary,
- "weight_int": weight,
- "source_chunk_ids": p.SourceChunkIDs,
- "source_doc_ids": capSourceDocs(p.SourceDocIDs),
- "title_tks": titleLTKS,
- "title_sm_tks": titleSMLTKS,
- "content_ltks": contentLTKS,
- "content_sm_ltks": contentSMLTKS,
- "content_with_weight": string(content),
+ "id": entityID,
+ "doc_id": kb,
+ "kb_id": kb,
+ "available_int": 1,
+ "compile_kwd": compileKwdWikiEntity,
+ "type_kwd": "wiki_" + p.PageType,
+ "entity_type_kwd": "wiki_" + p.PageType,
+ "slug_kwd": p.Slug,
+ "title_kwd": p.Title,
+ "weight_int": weight,
+ "source_chunk_ids": p.SourceChunkIDs,
+ "source_doc_ids": capSourceDocs(p.SourceDocIDs),
+ "content_ltks": contentLTKS,
+ "content_sm_ltks": contentSMLTKS,
+ "content_with_weight": string(content),
})
// Relations: one edge per outlink whose target page exists in this
@@ -1905,13 +1884,10 @@ func (w engineWriter) projectWikiGraphRows(_ context.Context, tenant, kb string,
rows = append(rows, map[string]interface{}{
"id": relID,
"doc_id": kb,
- "tenant_id": tenant,
"kb_id": kb,
"available_int": 1,
"compile_kwd": compileKwdWikiRelation,
"type_kwd": compileKwdWikiRelation,
- "from_id": entityID,
- "to_id": wikiGraphXXHash("wiki_entity", kb, tgt),
"from_kwd": p.Slug,
"to_kwd": tgt,
"source_doc_ids": srcDocs,
diff --git a/internal/ingestion/knowledge_compile/writer_test.go b/internal/ingestion/knowledge_compile/writer_test.go
index 861def5426..69e2fb657e 100644
--- a/internal/ingestion/knowledge_compile/writer_test.go
+++ b/internal/ingestion/knowledge_compile/writer_test.go
@@ -2,6 +2,7 @@ package knowledge_compile
import (
"context"
+ "strings"
"testing"
"time"
@@ -69,6 +70,7 @@ func TestMergedChunkMapKeepsWikiFields(t *testing.T) {
Content: "# Alpha\n\nBody",
Vector: []float32{0.1, 0.2, 0.3},
Meta: map[string]any{
+ "kind": "page",
"slug": "entity/alpha",
"title": "Alpha",
"page_type": "entity",
@@ -85,13 +87,16 @@ func TestMergedChunkMapKeepsWikiFields(t *testing.T) {
cases := map[string]string{
"slug_kwd": "entity/alpha",
- "artifact_slug_kwd": "entity/alpha",
"title_kwd": "Alpha",
"page_type_kwd": "entity",
"topic_kwd": "Knowledge/Core/Alpha",
"summary_with_weight": "A page about Alpha",
- "plan_kwd": "run-abc",
- "input_hash_kwd": "hash-123",
+ // The wiki page body goes to md_with_weight, the column Python writes
+ // and reads (wiki_incremental.py:2190); artifact_slug_kwd was a Go-only
+ // column that exists in no engine mapping.
+ "md_with_weight": "# Alpha\n\nBody",
+ "plan_kwd": "run-abc",
+ "input_hash_kwd": "hash-123",
}
// The wall-clock audit fields must be stamped from `now`, not omitted.
if m["create_time"] != now.Format("2006-01-02 15:04:05") {
@@ -111,8 +116,12 @@ func TestMergedChunkMapKeepsWikiFields(t *testing.T) {
if m["doc_id"] != "kb1" || m["available_int"] != 1 {
t.Errorf("merged flags wrong: doc_id=%v available_int=%v", m["doc_id"], m["available_int"])
}
- if _, ok := m["kc_merged"]; ok {
- t.Errorf("merged row must not persist undefined kc_merged field")
+ // No kc_* key may be persisted: they exist in no engine mapping, and
+ // Infinity rejects an insert that carries an unknown column.
+ for k := range m {
+ if strings.HasPrefix(k, "kc_") {
+ t.Errorf("merged row must not persist the non-schema field %q", k)
+ }
}
if m["q_3_vec"] == nil {
t.Errorf("vector column missing")
@@ -127,8 +136,7 @@ func TestProductFromChunkMapRestoresWikiFields(t *testing.T) {
"id": "wiki/1",
"doc_id": "d1",
"compile_kwd": "wiki_page",
- "content_with_weight": "# Alpha",
- "kc_payload": "# Alpha\n\nBody",
+ "content_with_weight": "# Alpha\n\nBody",
"slug_kwd": "entity/alpha",
"page_type_kwd": "entity",
"topic_kwd": "Alpha",
@@ -136,7 +144,7 @@ func TestProductFromChunkMapRestoresWikiFields(t *testing.T) {
"summary_with_weight": "A page about Alpha",
"entity_names_kwd": []interface{}{"Alpha"},
"related_kb_pages_kwd": []interface{}{"entity/beta"},
- "section_level_int": float64(2),
+ "depth_int": float64(2),
}
p, ok := productFromChunkMap(c, "t1", kccommon.VariantWiki)
if !ok {
diff --git a/internal/ingestion/task/knowledge_compiler_wiring.go b/internal/ingestion/task/knowledge_compiler_wiring.go
index 770e37ae36..ca2b8ba66e 100644
--- a/internal/ingestion/task/knowledge_compiler_wiring.go
+++ b/internal/ingestion/task/knowledge_compiler_wiring.go
@@ -404,7 +404,12 @@ func putWikiActiveStates(ctx context.Context, docEngine engine.DocEngine, states
if len(states) == 0 {
return nil
}
- store, ok := knowledge_compile.NewWikiMapVersionStore(docEngine).(kc.WikiMapActiveStateStore)
+ // Every state comes from one document's compile, so they share a tenant and
+ // dataset: the resolver only runs when the Infinity chunk table is missing.
+ store, ok := knowledge_compile.NewWikiMapVersionStoreWithVectorSizeResolver(
+ docEngine,
+ wikiMapVectorSizeResolver(states[0].TenantID, states[0].DatasetID),
+ ).(kc.WikiMapActiveStateStore)
if !ok {
return fmt.Errorf("Wiki dirty compile: active MAP store is unavailable")
}
@@ -416,6 +421,52 @@ func putWikiActiveStates(ctx context.Context, docEngine engine.DocEngine, states
return nil
}
+// wikiMapVectorSizeResolver resolves, lazily, the vector size a missing Infinity
+// chunk table needs. The document-level compile path holds no embedder, so it
+// binds the dataset's — kb.tenant_embd_id, else kb.embd_id — like a compile task.
+func wikiMapVectorSizeResolver(tenantID, datasetID string) func(context.Context) (int, error) {
+ svc := service.NewModelProviderService()
+ modelSolver := service.NewModelSolver()
+ return func(ctx context.Context) (int, error) {
+ kb, err := dao.NewKnowledgebaseDAO().GetByID(ctx, dao.DB, datasetID)
+ if err != nil {
+ return 0, fmt.Errorf("Wiki MAP chunk store: load dataset %s: %w", datasetID, err)
+ }
+ embedder := &kcEmbedder{svc: svc, solver: modelSolver, tenantID: tenantID, embdID: datasetEmbeddingID(kb)}
+ return embedderVectorSize(ctx, embedder)
+ }
+}
+
+// datasetEmbeddingID mirrors Python's embedding binding for a compile task:
+// kb.tenant_embd_id wins, else kb.embd_id, and an empty id leaves the embedder to
+// its tenant-default fallback (rag/svr/task_executor.py:1454-1487).
+func datasetEmbeddingID(kb *entity.Knowledgebase) string {
+ if kb == nil {
+ return ""
+ }
+ if kb.TenantEmbdID != nil && strings.TrimSpace(*kb.TenantEmbdID) != "" {
+ return *kb.TenantEmbdID
+ }
+ return kb.EmbdID
+}
+
+// embedderVectorSize reports the vector size a missing Infinity chunk table must
+// be created with: an unset dimension is measured with a one-input probe (the
+// signal Python sizes the table with in create_idx, task_executor.py:737-740).
+func embedderVectorSize(ctx context.Context, embed kc.Embedder) (int, error) {
+ if dimension := embed.Dimensions(); dimension > 0 {
+ return dimension, nil
+ }
+ vectors, err := embed.Encode(ctx, []string{"ok"})
+ if err != nil {
+ return 0, err
+ }
+ if len(vectors) == 0 || len(vectors[0]) == 0 {
+ return 0, fmt.Errorf("embedding returned an empty vector")
+ }
+ return len(vectors[0]), nil
+}
+
func copyStringAnyMap(source map[string]any) map[string]any {
result := make(map[string]any, len(source))
for key, value := range source {
@@ -541,17 +592,7 @@ func newKnowledgeCompilerDepsResolver() kc.DepsResolver {
embedder := &kcEmbedder{svc: svc, solver: modelSolver, tenantID: tenantID, embdID: embeddingModel}
mapStore := knowledge_compile.NewWikiMapVersionStoreWithVectorSizeResolver(engine.Get(), func(ctx context.Context) (int, error) {
- if dimension := embedder.Dimensions(); dimension > 0 {
- return dimension, nil
- }
- vectors, err := embedder.Encode(ctx, []string{"ok"})
- if err != nil {
- return 0, err
- }
- if len(vectors) == 0 || len(vectors[0]) == 0 {
- return 0, fmt.Errorf("embedding returned an empty vector")
- }
- return len(vectors[0]), nil
+ return embedderVectorSize(ctx, embedder)
})
return kc.Deps{
@@ -806,7 +847,7 @@ func (s *kcWikiPageStore) FindSimilarPages(ctx context.Context, tenantID, datase
IndexNames: []string{fmt.Sprintf("ragflow_%s", tenantID)},
KbIDs: []string{datasetID},
Limit: k,
- SelectFields: []string{"id", "slug_kwd", "title_kwd", "page_type_kwd", "topic_kwd", "plan_group_kwd", "summary_with_weight", "content_with_weight", "entity_names_kwd", "related_kb_pages_kwd", "outlinks_kwd", "source_chunk_ids", "kc_content_md_raw", "_score"},
+ SelectFields: []string{"id", "slug_kwd", "title_kwd", "page_type_kwd", "topic_kwd", "plan_group_kwd", "summary_with_weight", "content_with_weight", "md_with_weight", "entity_names_kwd", "related_kb_pages_kwd", "outlinks_kwd", "source_chunk_ids", "_score"},
// compile_kwd="wiki_page" is the schema-backed discriminator for wiki
// pages (sections carry compile_kwd="wiki_section"); there is no
// "kc_kind" column in the chunk schema, so filtering on it would return
@@ -842,7 +883,7 @@ func (s *kcWikiPageStore) GetPageBySlug(ctx context.Context, tenantID, datasetID
IndexNames: []string{fmt.Sprintf("ragflow_%s", tenantID)},
KbIDs: []string{datasetID},
Limit: 1,
- SelectFields: []string{"id", "slug_kwd", "title_kwd", "page_type_kwd", "topic_kwd", "plan_group_kwd", "summary_with_weight", "content_with_weight", "entity_names_kwd", "related_kb_pages_kwd", "outlinks_kwd", "source_chunk_ids", "kc_content_md_raw", "_score"},
+ SelectFields: []string{"id", "slug_kwd", "title_kwd", "page_type_kwd", "topic_kwd", "plan_group_kwd", "summary_with_weight", "content_with_weight", "md_with_weight", "entity_names_kwd", "related_kb_pages_kwd", "outlinks_kwd", "source_chunk_ids", "_score"},
Filter: map[string]interface{}{
"compile_kwd": "wiki_page",
"slug_kwd": slug,
@@ -864,7 +905,7 @@ func (s *kcWikiPageStore) FindPagesBySourceChunks(ctx context.Context, tenantID,
IndexNames: []string{fmt.Sprintf("ragflow_%s", tenantID)},
KbIDs: []string{datasetID},
Limit: k,
- SelectFields: []string{"id", "slug_kwd", "title_kwd", "page_type_kwd", "topic_kwd", "summary_with_weight", "content_with_weight", "entity_names_kwd", "related_kb_pages_kwd", "outlinks_kwd", "source_chunk_ids", "kc_content_md_raw", "_score"},
+ SelectFields: []string{"id", "slug_kwd", "title_kwd", "page_type_kwd", "topic_kwd", "summary_with_weight", "content_with_weight", "md_with_weight", "entity_names_kwd", "related_kb_pages_kwd", "outlinks_kwd", "source_chunk_ids", "_score"},
Filter: map[string]interface{}{
"compile_kwd": "wiki_page",
"source_chunk_ids": chunkIDs,
@@ -887,15 +928,18 @@ func (s *kcWikiPageStore) FindPagesBySourceChunks(ctx context.Context, tenantID,
func wikiPageCandidateFromRow(row map[string]interface{}) kc.WikiPageCandidate {
return kc.WikiPageCandidate{
- ID: strings.TrimSpace(anyString(row["id"])),
- Slug: strings.TrimSpace(anyString(row["slug_kwd"])),
- Title: strings.TrimSpace(anyString(row["title_kwd"])),
- PageType: strings.TrimSpace(anyString(row["page_type_kwd"])),
- Topic: strings.TrimSpace(anyString(row["topic_kwd"])),
- PlanGroup: strings.TrimSpace(anyString(row["plan_group_kwd"])),
- Summary: strings.TrimSpace(anyString(row["summary_with_weight"])),
- ContentMD: strings.TrimSpace(anyString(row["content_with_weight"])),
- ContentMDRaw: strings.TrimSpace(anyString(row["kc_content_md_raw"])),
+ ID: strings.TrimSpace(anyString(row["id"])),
+ Slug: strings.TrimSpace(anyString(row["slug_kwd"])),
+ Title: strings.TrimSpace(anyString(row["title_kwd"])),
+ PageType: strings.TrimSpace(anyString(row["page_type_kwd"])),
+ Topic: strings.TrimSpace(anyString(row["topic_kwd"])),
+ PlanGroup: strings.TrimSpace(anyString(row["plan_group_kwd"])),
+ Summary: strings.TrimSpace(anyString(row["summary_with_weight"])),
+ ContentMD: strings.TrimSpace(anyString(row["content_with_weight"])),
+ // md_with_weight is the page-body column Python writes and reads
+ // (wiki_incremental.py:2190/:2251); page rows fall back to
+ // content_with_weight when it was not stamped.
+ ContentMDRaw: firstNonEmptyString(row["md_with_weight"], row["content_with_weight"]),
EntityNames: anyStrings(row["entity_names_kwd"]),
RelatedKBPages: anyStrings(row["related_kb_pages_kwd"]),
Outlinks: anyStrings(row["outlinks_kwd"]),
@@ -904,6 +948,16 @@ func wikiPageCandidateFromRow(row map[string]interface{}) kc.WikiPageCandidate {
}
}
+// firstNonEmptyString returns the first candidate that renders non-empty.
+func firstNonEmptyString(values ...interface{}) string {
+ for _, v := range values {
+ if s := strings.TrimSpace(anyString(v)); s != "" {
+ return s
+ }
+ }
+ return ""
+}
+
func anyString(v interface{}) string {
switch x := v.(type) {
case string:
diff --git a/internal/ingestion/task/knowledge_compiler_wiring_test.go b/internal/ingestion/task/knowledge_compiler_wiring_test.go
index ab4a81abf4..99b565bdd6 100644
--- a/internal/ingestion/task/knowledge_compiler_wiring_test.go
+++ b/internal/ingestion/task/knowledge_compiler_wiring_test.go
@@ -20,6 +20,7 @@ import (
"testing"
"ragflow/internal/agent/runtime"
+ "ragflow/internal/entity"
)
// TestKnowledgeCompilerRegisteredByWiring locks the composition-root contract:
@@ -41,3 +42,38 @@ func TestKnowledgeCompilerRegisteredByWiring(t *testing.T) {
t.Fatal("component \"Compiler\" registered with a nil factory")
}
}
+
+// TestDatasetEmbeddingIDBindsTheDatasetModel mirrors Python's compile-task
+// binding: the tenant-level model wins over the dataset's own, a blank tenant id
+// falls back to the dataset model, and an empty id defers to the tenant default.
+func TestDatasetEmbeddingIDBindsTheDatasetModel(t *testing.T) {
+ tenantModel := "tenant-model-1"
+ blank := " "
+
+ tests := []struct {
+ name string
+ kb *entity.Knowledgebase
+ want string
+ }{
+ {name: "nil knowledgebase", kb: nil, want: ""},
+ {name: "dataset model", kb: &entity.Knowledgebase{EmbdID: "bge-large"}, want: "bge-large"},
+ {
+ name: "tenant model wins",
+ kb: &entity.Knowledgebase{EmbdID: "bge-large", TenantEmbdID: &tenantModel},
+ want: "tenant-model-1",
+ },
+ {
+ name: "blank tenant model falls back",
+ kb: &entity.Knowledgebase{EmbdID: "bge-large", TenantEmbdID: &blank},
+ want: "bge-large",
+ },
+ }
+
+ for _, tc := range tests {
+ t.Run(tc.name, func(t *testing.T) {
+ if got := datasetEmbeddingID(tc.kb); got != tc.want {
+ t.Fatalf("datasetEmbeddingID() = %q, want %q", got, tc.want)
+ }
+ })
+ }
+}
diff --git a/internal/service/dataset_artifact_keyword_integration_test.go b/internal/service/dataset_artifact_keyword_integration_test.go
index 0f3f1728dc..8ea9870d14 100644
--- a/internal/service/dataset_artifact_keyword_integration_test.go
+++ b/internal/service/dataset_artifact_keyword_integration_test.go
@@ -23,7 +23,6 @@ func wikiIntRow(id, pageType, slug, title, topic, summary string, outlinks int)
"scope_kwd": "dataset",
"compile_kwd": CompileKwdWikiPage,
"slug_kwd": fullSlug,
- "artifact_slug_kwd": fullSlug,
"title_kwd": title,
"page_type_kwd": pageType,
"topic_kwd": topic,
diff --git a/internal/service/dataset_artifact_service.go b/internal/service/dataset_artifact_service.go
index ef20e5161b..4aee8ab937 100644
--- a/internal/service/dataset_artifact_service.go
+++ b/internal/service/dataset_artifact_service.go
@@ -615,7 +615,7 @@ func (s *DatasetArtifactService) GetWikiGraph(ctx context.Context, tenantID, dat
orderBy = (&types.OrderByExpr{}).Desc("weight_int")
}
entityChunks, _, err := s.searchCompiledWithMatch(ctx, tenantID, datasetID, entityFilter,
- []string{"id", "slug_kwd", "title_kwd", "aliases_kwd", "description_with_weight", "entity_type_kwd", "weight_int", "source_chunk_ids"},
+ []string{"id", "slug_kwd", "title_kwd", "entity_type_kwd", "weight_int", "source_chunk_ids", "content_with_weight"},
0, limit, orderBy, matchExprs)
if err != nil {
return nil, err
@@ -645,12 +645,23 @@ func (s *DatasetArtifactService) GetWikiGraph(ctx context.Context, tenantID, dat
pageType = bareSlug[:idx]
bareSlug = bareSlug[idx+1:]
}
+ // name / aliases / description live in the content_with_weight payload
+ // (as Python writes them); the columns they replaced exist in no mapping.
+ payload := wikiEntityPayload(c)
+ name := firstStringValue(c["title_kwd"])
+ if v := payloadString(payload, "name"); v != "" {
+ name = v
+ }
+ entityType := pageType
+ if v := payloadString(payload, "type"); v != "" {
+ entityType = v
+ }
graph.Entities = append(graph.Entities, WikiGraphEntity{
Slug: bareSlug,
- Name: firstStringValue(c["title_kwd"]),
- Aliases: toStringSlice(c["aliases_kwd"]),
- Description: firstStringValue(c["description_with_weight"]),
- Type: pageType,
+ Name: name,
+ Aliases: payloadStrings(payload, "aliases"),
+ Description: payloadString(payload, "description"),
+ Type: entityType,
Weight: w,
SourceChunkIDs: toStringSlice(c["source_chunk_ids"]),
})
@@ -1064,6 +1075,47 @@ func firstStringValue(v interface{}) string {
return ""
}
+// wikiEntityPayload decodes the canvas payload a wiki_entity row stores in
+// content_with_weight (Python `_wiki_entity_payload`); the payload is
+// authoritative for name/aliases/description/type.
+func wikiEntityPayload(row map[string]interface{}) map[string]interface{} {
+ raw := firstStringValue(row["content_with_weight"])
+ if strings.TrimSpace(raw) == "" {
+ return nil
+ }
+ var payload map[string]interface{}
+ if json.Unmarshal([]byte(raw), &payload) != nil {
+ return nil
+ }
+ return payload
+}
+
+// payloadString reads a payload field, accepting the legacy Go key too.
+func payloadString(payload map[string]interface{}, key string) string {
+ if payload == nil {
+ return ""
+ }
+ if v, ok := payload[key].(string); ok {
+ return strings.TrimSpace(v)
+ }
+ // Legacy Go rows named the same values title / page_type / summary.
+ legacy := map[string]string{"name": "title", "type": "page_type", "description": "summary"}[key]
+ if legacy != "" {
+ if v, ok := payload[legacy].(string); ok {
+ return strings.TrimSpace(v)
+ }
+ }
+ return ""
+}
+
+// payloadStrings returns a string-list payload field.
+func payloadStrings(payload map[string]interface{}, key string) []string {
+ if payload == nil {
+ return []string{}
+ }
+ return toStringSlice(payload[key])
+}
+
// bareWikiSlug strips only the first path segment from a full wiki slug
// ("entity/location/长社" -> "location/长社"). Nested type/name segments are
// preserved so distinct typed entities remain distinguishable in graph links.