RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.
Go agent-harnessagent-native-databaseagentic-aiagentic-retrievalagentic-searchaiai-agentscontext-enginecontext-engineeringcontext-managementharness-engineeringknowledge-compilationragretrieval-augmented-generationretrieval-harnesssearch-harness
Refactor: knowledge compilation field name refactor (#20587)
Wang Qif307aba34fa6
diff --git a/conf/infinity_mapping.json b/conf/infinity_mapping.jsonindex 15ba437143..424a4ace26 100644--- a/conf/infinity_mapping.json+++ b/conf/infinity_mapping.json@@ -26,7 +26,6 @@"rank_int": {"type": "integer", "default": 0},"rank_flt": {"type": "float", "default": 0},"available_int": {"type": "integer", "default": 1, "index_type": {"type": "secondary", "cardinality": "low"}},- "knowledge_graph_kwd": {"type": "varchar", "default": ""},"entities_kwd": {"type": "varchar", "default": "", "analyzer": "whitespace-#"},"pagerank_fea": {"type": "integer", "default": 0},"tag_feas": {"type": "varchar", "default": "", "analyzer": "rankfeatures"},@@ -51,7 +50,6 @@"source_chunk_ids": {"type": "json", "default": "[]", "index_type": {"type": "secondary", "cardinality": "low"}},"source_doc_ids": {"type": "json", "default": "[]", "index_type": {"type": "secondary", "cardinality": "low"}},"compilation_template_ids": {"type": "json", "default": "[]", "index_type": {"type": "secondary", "cardinality": "low"}},- "compilation_template_kind_kwd": {"type": "varchar", "default": "", "analyzer": "whitespace-#"},"chunk_hash_kwd": {"type": "varchar", "default": "", "analyzer": "whitespace-#"},"input_hash_kwd": {"type": "varchar", "default": "", "analyzer": "whitespace-#"},"wiki_slug_kwd": {"type": "varchar", "default": "", "analyzer": "whitespace-#"},@@ -66,7 +64,6 @@"topic_kwd": {"type": "varchar", "default": "", "analyzer": "whitespace-#"},"plan_group_kwd": {"type": "varchar", "default": "", "analyzer": "whitespace-#"},"generation_kwd": {"type": "varchar", "default": "", "analyzer": "whitespace-#"},- "page_type_kwd": {"type": "varchar", "default": "", "analyzer": "whitespace-#"},"entity_names_kwd": {"type": "json", "default": "[]", "index_type": {"type": "secondary", "cardinality": "low"}},"entity_names": {"type": "json", "default": "[]", "enabled": false},"outlinks_kwd": {"type": "json", "default": "[]", "index_type": {"type": "secondary", "cardinality": "low"}},diff --git a/internal/engine/elasticsearch/chunk.go b/internal/engine/elasticsearch/chunk.goindex d987389572..eeff56e0b0 100644--- a/internal/engine/elasticsearch/chunk.go+++ b/internal/engine/elasticsearch/chunk.go@@ -709,34 +709,34 @@ func conditionTermsList(value interface{}) ([]interface{}, bool) {return out, true}-// mustNotExistsClauses renders the `must_not` exclusions a condition carries.-// The only form RAGFlow produces is must_not={"exists": <field>} (e.g.-// hybrid_search excluding compiled products); any other shape yields nothing.+// mustNotExistsClauses renders field and existence exclusions.func mustNotExistsClauses(value interface{}) []map[string]interface{} {exclusions, ok := value.(map[string]interface{})if !ok {return nil}- clauses := make([]map[string]interface{}, 0, len(exclusions))- for field, value := range exclusions {- if field != "exists" {- continue- }- clauses = append(clauses, map[string]interface{}{- "exists": map[string]interface{}{"field": value},- })+ if field, ok := exclusions["exists"]; ok && len(exclusions) == 1 {+ return []map[string]interface{}{{"exists": map[string]interface{}{"field": field}}}}- return clauses+ if query := buildBoolQueryFromConditionRaw(exclusions, nil, false, false); query != nil {+ return []map[string]interface{}{query}+ }+ return nil}// updateChunksByQuery handles multi-document updatefunc (e *Engine) updateChunksByQuery(ctx context.Context, indexName string, condition map[string]interface{}, newValue map[string]interface{}) error {common.Debug("ElasticsearchConnection.updateChunksByQuery called", zap.String("indexName", indexName))+ condition = types.CompilationFilter(condition)// Build bool query from conditionvar mustClauses []map[string]interface{}var mustNotClauses []map[string]interface{}for k, v := range condition {+ if k == "and" || k == "or" {+ mustClauses = append(mustClauses, buildBoolQueryFromConditionRaw(map[string]interface{}{k: v}, nil, false, false))+ continue+ }if k == "exists" {mustClauses = append(mustClauses, map[string]interface{}{"exists": map[string]interface{}{"field": v},@@ -1010,6 +1010,7 @@ func (e *Engine) DeleteChunks(ctx context.Context, condition map[string]interfacreturn 0, nil}+ condition = types.CompilationFilter(condition)// Build bool query from conditionvar mustClauses []map[string]interface{}var filterClauses []map[string]interface{}@@ -1045,6 +1046,10 @@ func (e *Engine) DeleteChunks(ctx context.Context, condition map[string]interfac// Add all other conditions as filters/must/must_notfor k, v := range condition {+ if k == "and" || k == "or" {+ filterClauses = append(filterClauses, buildBoolQueryFromConditionRaw(map[string]interface{}{k: v}, nil, false, false))+ continue+ }if k == "id" || k == "kb_id" {continue // Already handled above}@@ -2138,6 +2143,10 @@ func memoryMessageStatusBool(value interface{}) bool {// Skill indexes use status, regular chunk indexes use kb_id, and memory// message indexes use memory_id plus message-specific storage fields.func buildBoolQueryFromCondition(filter map[string]interface{}, kbIDs []string, isSkillIndex, isMemoryIndex bool) map[string]interface{} {+ return buildBoolQueryFromConditionRaw(types.CompilationFilter(filter), kbIDs, isSkillIndex, isMemoryIndex)+}++func buildBoolQueryFromConditionRaw(filter map[string]interface{}, kbIDs []string, isSkillIndex, isMemoryIndex bool) map[string]interface{} {var mustClauses []interface{}var mustNotClauses []interface{}var filterClauses []interface{}@@ -2168,6 +2177,20 @@ func buildBoolQueryFromCondition(filter map[string]interface{}, kbIDs []string,}for k, v := range filter {+ if k == "and" || k == "or" {+ var children []interface{}+ for _, child := range types.FilterClauses(v) {+ children = append(children, buildBoolQueryFromConditionRaw(child, nil, false, isMemoryIndex))+ }+ if len(children) > 0 {+ clause := map[string]interface{}{"filter": children}+ if k == "or" {+ clause = map[string]interface{}{"should": children, "minimum_should_match": 1}+ }+ filterClauses = append(filterClauses, map[string]interface{}{"bool": clause})+ }+ continue+ }if isMemoryIndex {k = mapMemoryMessageESField(k, false)}@@ -2215,13 +2238,15 @@ func buildBoolQueryFromCondition(filter map[string]interface{}, kbIDs []string,continue}if k == "must_not" {- if condition, ok := v.(map[string]interface{}); ok {- if field, ok := condition["exists"].(string); ok && field != "" {- mustNotClauses = append(mustNotClauses, map[string]interface{}{"exists": map[string]interface{}{"field": field}})- }+ for _, clause := range mustNotExistsClauses(v) {+ mustNotClauses = append(mustNotClauses, clause)}continue}+ if k == "exists" {+ filterClauses = append(filterClauses, map[string]interface{}{"exists": map[string]interface{}{"field": v}})+ continue+ }if k == "knowledge_graph_kwd" {var graphTypeClauses []interface{}for _, field := range []string{"type_kwd", "knowledge_graph_kwd"} {diff --git a/internal/engine/elasticsearch/chunk_helpers_test.go b/internal/engine/elasticsearch/chunk_helpers_test.goindex d28e630866..238c8f0aa2 100644--- a/internal/engine/elasticsearch/chunk_helpers_test.go+++ b/internal/engine/elasticsearch/chunk_helpers_test.go@@ -195,8 +195,8 @@ func TestDeleteChunksPreservesStringSliceCondition(t *testing.T) {}engine := &Engine{client: client}deleted, err := engine.DeleteChunks(ctx, map[string]interface{}{- "kb_id": "kb-1",- "compile_kwd": []string{"wiki_entity", "wiki_relation"},+ "kb_id": "kb-1",+ "source_doc_ids": []string{"doc-a", "doc-b"},}, "ragflow_tenant", "kb-1")if err != nil {t.Fatalf("DeleteChunks: %v", err)@@ -218,7 +218,7 @@ func TestDeleteChunksPreservesStringSliceCondition(t *testing.T) {t.Fatalf("delete query must=%#v, want one terms clause", boolQuery["must"])}terms := must[0].(map[string]interface{})["terms"].(map[string]interface{})- assertEqual(t, terms["compile_kwd"], []interface{}{"wiki_entity", "wiki_relation"})+ assertEqual(t, terms["source_doc_ids"], []interface{}{"doc-a", "doc-b"})}// TestDeleteChunksIDStringSlice guards against the id filter being dropped whendiff --git a/internal/engine/elasticsearch/compilation_test.go b/internal/engine/elasticsearch/compilation_test.gonew file mode 100644index 0000000000..e5132e81bb--- /dev/null+++ b/internal/engine/elasticsearch/compilation_test.go@@ -0,0 +1,162 @@+package elasticsearch++import (+ "encoding/json"+ "fmt"+ "net/http"+ "net/http/httptest"+ "sync/atomic"+ "testing"++ "github.com/elastic/go-elasticsearch/v8"+ "ragflow/internal/common"+ "ragflow/internal/engine/types"+)++func TestCompilationSearchReadsBothGenerationsInOneRequest(t *testing.T) {+ if err := common.InitLogger("info", common.FileOutput{}, "compilation_test"); err != nil {+ t.Fatal(err)+ }+ rows := []map[string]interface{}{+ {"id": "new", "kb_id": "kb1", "compile_kwd": "wiki", "type_kwd": "wiki_page"},+ {"id": "old", "kb_id": "kb1", "compile_kwd": "wiki_page"},+ {"id": "section", "kb_id": "kb1", "compile_kwd": "wiki", "type_kwd": "wiki_section"},+ }+ var requests atomic.Int32+ server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {+ requests.Add(1)+ var body map[string]interface{}+ if err := json.NewDecoder(r.Body).Decode(&body); err != nil {+ t.Error(err)+ http.Error(w, "bad query", 400)+ return+ }+ var hits []map[string]interface{}+ for _, row := range rows {+ if matchesCompilationQuery(body["query"].(map[string]interface{}), row) {+ hits = append(hits, map[string]interface{}{"_id": row["id"], "_source": row})+ }+ }+ w.Header().Set("X-Elastic-Product", "Elasticsearch")+ w.Header().Set("Content-Type", "application/json")+ if err := json.NewEncoder(w).Encode(map[string]interface{}{"hits": map[string]interface{}{"total": map[string]interface{}{"value": len(hits)}, "hits": hits}}); err != nil {+ t.Error(err)+ }+ }))+ defer server.Close()+ client, err := elasticsearch.NewClient(elasticsearch.Config{Addresses: []string{server.URL}})+ if err != nil {+ t.Fatal(err)+ }+ eng := &Engine{client: client}+ result, err := eng.Search(t.Context(), &types.SearchRequest{IndexNames: []string{"ragflow_t1"}, KbIDs: []string{"kb1"}, Limit: 10, Filter: map[string]interface{}{"type_kwd": "wiki_page"}})+ if err != nil {+ t.Fatal(err)+ }+ if requests.Load() != 1 || result.Total != 2 || len(result.Chunks) != 2 {+ t.Fatalf("requests=%d, result=%+v; want one request returning new and old pages", requests.Load(), result)+ }+}++func TestCompilationQueryMatchesNewAndOldRows(t *testing.T) {+ for _, test := range []struct {+ name string+ filter, row map[string]interface{}+ want bool+ }{+ {"new page", map[string]interface{}{"compile_kwd": "wiki_page"}, map[string]interface{}{"kb_id": "kb1", "compile_kwd": "wiki", "type_kwd": "wiki_page"}, true},+ {"old page", map[string]interface{}{"compile_kwd": "wiki_page"}, map[string]interface{}{"kb_id": "kb1", "compile_kwd": "wiki_page"}, true},+ {"section excluded", map[string]interface{}{"compile_kwd": "wiki_page"}, map[string]interface{}{"kb_id": "kb1", "compile_kwd": "wiki", "type_kwd": "wiki_section"}, false},+ {"tenant dataset scope", map[string]interface{}{"compile_kwd": "wiki_page"}, map[string]interface{}{"kb_id": "kb2", "compile_kwd": "wiki", "type_kwd": "wiki_page"}, false},+ {"old graph", map[string]interface{}{"compile_kwd": "graph"}, map[string]interface{}{"kb_id": "kb1", "compile_kwd": "hypergraph", "compilation_template_kind_kwd": "knowledge_graph"}, true},+ {"canonical row", map[string]interface{}{"compilation_template_kind_kwd": "timeline"}, map[string]interface{}{"kb_id": "kb1", "compile_kwd": "graph"}, false},+ {"old inferred timeline preserves graph", map[string]interface{}{"compile_kwd": "graph"}, map[string]interface{}{"kb_id": "kb1", "compile_kwd": "timeline", "compilation_template_kind_kwd": "knowledge_graph"}, true},+ {"old inferred timeline is not a timeline template", map[string]interface{}{"compile_kwd": "timeline"}, map[string]interface{}{"kb_id": "kb1", "compile_kwd": "timeline", "compilation_template_kind_kwd": "knowledge_graph"}, false},+ {"blank stamp", map[string]interface{}{"compile_kwd": "graph"}, map[string]interface{}{"kb_id": "kb1", "compile_kwd": "graph", "compilation_template_kind_kwd": ""}, true},+ {"navigation cleanup", map[string]interface{}{"compile_kwd": "dataset_nav"}, map[string]interface{}{"kb_id": "kb1", "compile_kwd": "page_index", "type_kwd": "nav_cluster"}, true},+ {"structure survives nav cleanup", map[string]interface{}{"compile_kwd": "dataset_nav"}, map[string]interface{}{"kb_id": "kb1", "compile_kwd": "page_index", "type_kwd": "entity"}, false},+ {"wiki graph cleanup", map[string]interface{}{"compile_kwd": []string{"wiki_entity", "wiki_relation"}}, map[string]interface{}{"kb_id": "kb1", "compile_kwd": "wiki", "type_kwd": "wiki_relation"}, true},+ {"page survives graph cleanup", map[string]interface{}{"compile_kwd": []string{"wiki_entity", "wiki_relation"}}, map[string]interface{}{"kb_id": "kb1", "compile_kwd": "wiki", "type_kwd": "wiki_page"}, false},+ {"mind map type reads old spelling", map[string]interface{}{"entity_type_kwd": "mind_map"}, map[string]interface{}{"kb_id": "kb1", "entity_type_kwd": "mindmap"}, true},+ {"mind map type reads new spelling", map[string]interface{}{"entity_type_kwd": "mindmap"}, map[string]interface{}{"kb_id": "kb1", "entity_type_kwd": "mind_map"}, true},+ {"mind map type remains scoped", map[string]interface{}{"entity_type_kwd": "mind_map"}, map[string]interface{}{"kb_id": "kb2", "entity_type_kwd": "mindmap"}, false},+ } {+ t.Run(test.name, func(t *testing.T) {+ query := buildBoolQueryFromCondition(test.filter, []string{"kb1"}, false, false)+ if got := matchesCompilationQuery(query, test.row); got != test.want {+ t.Fatalf("query %v matched = %v, want %v", query, got, test.want)+ }+ })+ }+}++func matchesCompilationQuery(query, row map[string]interface{}) bool {+ if clause, ok := query["bool"].(map[string]interface{}); ok {+ for _, field := range []string{"must", "filter"} {+ for _, child := range compilationQueryChildren(clause[field]) {+ if !matchesCompilationQuery(child, row) {+ return false+ }+ }+ }+ for _, child := range compilationQueryChildren(clause["must_not"]) {+ if matchesCompilationQuery(child, row) {+ return false+ }+ }+ minimum := 0+ switch value := clause["minimum_should_match"].(type) {+ case int:+ minimum = value+ case float64:+ minimum = int(value)+ }+ if minimum > 0 {+ matched := 0+ for _, child := range compilationQueryChildren(clause["should"]) {+ if matchesCompilationQuery(child, row) {+ matched+++ }+ }+ if matched < minimum {+ return false+ }+ }+ return true+ }+ if clause, ok := query["exists"].(map[string]interface{}); ok {+ return row[clause["field"].(string)] != nil+ }+ for _, operator := range []string{"term", "terms"} {+ if clause, ok := query[operator].(map[string]interface{}); ok {+ for field, expected := range clause {+ if items, ok := conditionTermsList(expected); ok {+ matched := false+ for _, value := range items {+ matched = matched || fmt.Sprint(row[field]) == fmt.Sprint(value)+ }+ if !matched {+ return false+ }+ } else if fmt.Sprint(row[field]) != fmt.Sprint(expected) {+ return false+ }+ }+ return true+ }+ }+ return false+}++func compilationQueryChildren(value interface{}) []map[string]interface{} {+ if children, ok := value.([]map[string]interface{}); ok {+ return children+ }+ var out []map[string]interface{}+ if children, ok := value.([]interface{}); ok {+ for _, child := range children {+ out = append(out, child.(map[string]interface{}))+ }+ }+ return out+}diff --git a/internal/engine/infinity/chunk.go b/internal/engine/infinity/chunk.goindex b3d5507360..83f9035d09 100644--- a/internal/engine/infinity/chunk.go+++ b/internal/engine/infinity/chunk.go@@ -2536,6 +2536,13 @@ func buildGraphTypeFilterCondition(value interface{}) string {func equivalentConditionToStr(condition map[string]interface{}, tableColumns map[string]struct {Type stringDefault interface{}+}) string {+ return equivalentConditionToStrRaw(types.CompilationFilter(condition), tableColumns)+}++func equivalentConditionToStrRaw(condition map[string]interface{}, tableColumns map[string]struct {+ Type string+ Default interface{}}) string {if len(condition) == 0 {return ""@@ -2544,6 +2551,22 @@ func equivalentConditionToStr(condition map[string]interface{}, tableColumns mapvar cond []stringfor k, v := range condition {+ if k == "and" || k == "or" {+ var children []string+ for _, child := range types.FilterClauses(v) {+ if expr := equivalentConditionToStrRaw(child, tableColumns); expr != "" {+ children = append(children, "("+expr+")")+ }+ }+ operator := " AND "+ if k == "or" {+ operator = " OR "+ }+ if len(children) > 0 {+ cond = append(cond, joinBalanced(children, operator))+ }+ continue+ }if k == "_id" || utility.IsEmpty(v) {continue}@@ -2551,11 +2574,8 @@ func equivalentConditionToStr(condition map[string]interface{}, tableColumns map// Handle must_not speciallyif k == "must_not" {if m, ok := v.(map[string]interface{}); ok {- for kk, vv := range m {- if kk == "exists" {- // For must_not exists, use !='' since we don't have table schema- cond = append(cond, fmt.Sprintf("NOT (%v!='')", vv))- }+ if expr := equivalentConditionToStrRaw(m, tableColumns); expr != "" {+ cond = append(cond, "NOT ("+expr+")")}}continuediff --git a/internal/engine/infinity/common.go b/internal/engine/infinity/common.goindex 8d648214d1..edeb53036e 100644--- a/internal/engine/infinity/common.go+++ b/internal/engine/infinity/common.go@@ -22,6 +22,7 @@ import ("encoding/json""fmt""ragflow/internal/common"+ "ragflow/internal/engine/types""strings""unicode"@@ -375,10 +376,33 @@ func existsCondition(field string, tableColumns map[string]struct {func buildFilterFromCondition(condition map[string]interface{}, tableColumns map[string]struct {Type stringDefault interface{}+}) string {+ return buildFilterFromConditionRaw(types.CompilationFilter(condition), tableColumns)+}++func buildFilterFromConditionRaw(condition map[string]interface{}, tableColumns map[string]struct {+ Type string+ Default interface{}}) string {var conditions []stringfor k, v := range condition {+ if k == "and" || k == "or" {+ var children []string+ for _, child := range types.FilterClauses(v) {+ if expr := buildFilterFromConditionRaw(child, tableColumns); expr != "" {+ children = append(children, "("+expr+")")+ }+ }+ operator := " AND "+ if k == "or" {+ operator = " OR "+ }+ if len(children) > 0 {+ conditions = append(conditions, joinBalanced(children, operator))+ }+ continue+ }if v == nil {continue}@@ -388,18 +412,21 @@ func buildFilterFromCondition(condition map[string]interface{}, tableColumns map// Handle must_not conditions -> NOT (...)if k == "must_not" {- if mustNotMap, ok := v.(map[string]interface{}); ok {- for kk, vv := range mustNotMap {- if kk == "exists" {- if existsField, ok := vv.(string); ok {- conditions = append(conditions, fmt.Sprintf("NOT (%s)", existsCondition(existsField, tableColumns)))- }- }+ if m, ok := v.(map[string]interface{}); ok {+ if expr := buildFilterFromConditionRaw(m, tableColumns); expr != "" {+ conditions = append(conditions, "NOT ("+expr+")")}}continue}+ if k == "exists" {+ if field, ok := v.(string); ok {+ conditions = append(conditions, existsCondition(field, tableColumns))+ }+ continue+ }+if k == "knowledge_graph_kwd" {if graphTypeCondition := buildGraphTypeFilterCondition(v); graphTypeCondition != "" {conditions = append(conditions, graphTypeCondition)diff --git a/internal/engine/infinity/sql_test.go b/internal/engine/infinity/sql_test.goindex f66c51987a..495d62eaf5 100644--- a/internal/engine/infinity/sql_test.go+++ b/internal/engine/infinity/sql_test.go@@ -716,6 +716,16 @@ func TestKeywordFilterConditionExactForWhitespaceValues(t *testing.T) {// would write a doc's parent_kwd while the cluster's own update matched nothing.func TestKeywordFilterRenderingBothPaths(t *testing.T) {const name = "Imperial Memorial and Governance Advice 59cbfbef"+ for _, alias := range []string{"mind_map", "mindmap"} {+ condition := map[string]interface{}{"entity_type_kwd": alias}+ for _, query := range []string{equivalentConditionToStr(condition, nil), buildFilterFromCondition(condition, nil)} {+ for _, value := range []string{"mind_map", "mindmap"} {+ if !strings.Contains(query, "filter_fulltext('entity_type_kwd', '"+value+"')") {+ t.Errorf("entity type filter %q does not match %q", query, value)+ }+ }+ }+ }wantParent := `(parent_kwd = '` + name + `')`parentCond := map[string]interface{}{"parent_kwd": []string{name}}diff --git a/internal/engine/types/compilation.go b/internal/engine/types/compilation.gonew file mode 100644index 0000000000..20aa641ce5--- /dev/null+++ b/internal/engine/types/compilation.go@@ -0,0 +1,237 @@+package types++import "strings"++// CanonicalCompilationKind returns the persisted knowledge compilation kind.+func CanonicalCompilationKind(kind string) string {+ kind = strings.ToLower(strings.ReplaceAll(strings.TrimSpace(kind), "-", "_"))+ switch kind {+ case "knowledge_graph", "knowledgegraph", "structure", "hypergraph", "list", "set":+ return "graph"+ case "mindmap":+ return "mind_map"+ case "pageindex":+ return "page_index"+ case "wiki_page", "wiki_section", "wiki_entity", "wiki_relation", "wiki_contribution_state", "wiki_map_active", "wiki_map_extract":+ return "wiki"+ }+ return kind+}++func compilationString(value any) string {+ switch value := value.(type) {+ case string:+ return strings.TrimSpace(value)+ case []string:+ if len(value) > 0 {+ return strings.TrimSpace(value[0])+ }+ case []any:+ if len(value) > 0 {+ return compilationString(value[0])+ }+ }+ return ""+}++func isCompilationKind(kind string) bool {+ switch kind {+ case "graph", "mind_map", "page_index", "tree", "timeline", "wiki", "session_graph", "session_essence":+ return true+ }+ return false+}++// CompilationKind reads the canonical field before the stored template kind.+func CompilationKind(row map[string]any) string {+ kind := compilationString(row["compile_kwd"])+ template := compilationString(row["compilation_template_kind_kwd"])+ if isCompilationKind(kind) && template == "" {+ return kind+ }+ if template != "" {+ return CanonicalCompilationKind(template)+ }+ return CanonicalCompilationKind(kind)+}++func CompilationRowType(row map[string]any) string {+ kind := compilationString(row["compile_kwd"])+ if kind == "wiki" {+ return compilationString(row["type_kwd"])+ }+ if strings.HasPrefix(kind, "wiki_") {+ return kind+ }+ return compilationString(row["type_kwd"])+}++func WikiPageCategory(row map[string]any) string {+ if compilationString(row["compile_kwd"]) == "wiki" {+ if category := compilationString(row["entity_type_kwd"]); category != "" {+ return category+ }+ }+ return compilationString(row["page_type_kwd"])+}++func IsNavigationRow(row map[string]any) bool {+ role := compilationString(row["type_kwd"])+ return role == "nav_doc" || role == "nav_cluster" || compilationString(row["compile_kwd"]) == "dataset_nav"+}++func compilationValues(value any) []string {+ switch value := value.(type) {+ case string:+ return []string{value}+ case []string:+ return value+ case []any:+ var out []string+ for _, v := range value {+ if s, ok := v.(string); ok {+ out = append(out, s)+ }+ }+ return out+ }+ return nil+}++// CompilationFilter builds predicates for mixed stored generations in one query.+// Each old-field branch is guarded so it cannot override a canonical value.+func CompilationFilter(filter map[string]any) map[string]any {+ out := make(map[string]any, len(filter))+ var clauses []map[string]any+ for field, value := range filter {+ var alternatives []map[string]any+ switch field {+ case "compilation_template_kind_kwd":+ for _, kind := range compilationValues(value) {+ alternatives = append(alternatives, compilationKindFilter(kind))+ }+ if len(alternatives) == 0 {+ out[field] = value+ }+ case "compile_kwd":+ changed := false+ for _, kind := range compilationValues(value) {+ switch {+ case kind == "dataset_nav":+ changed = true+ alternatives = append(alternatives, map[string]any{"type_kwd": []string{"nav_doc", "nav_cluster"}})+ case strings.HasPrefix(kind, "wiki_") && CanonicalCompilationKind(kind) == "wiki":+ changed = true+ alternatives = append(alternatives, wikiRoleFilter(kind))+ case isCompilationKind(CanonicalCompilationKind(kind)):+ changed = true+ alternatives = append(alternatives, compilationKindFilter(kind))+ default:+ alternatives = append(alternatives, map[string]any{field: kind})+ }+ }+ if !changed {+ alternatives = nil+ out[field] = value+ }+ case "type_kwd":+ changed := false+ for _, role := range compilationValues(value) {+ if strings.HasPrefix(role, "wiki_") && CanonicalCompilationKind(role) == "wiki" {+ changed = true+ alternatives = append(alternatives, wikiRoleFilter(role))+ } else {+ alternatives = append(alternatives, map[string]any{field: role})+ }+ }+ if !changed {+ alternatives = nil+ out[field] = value+ }+ case "entity_type_kwd":+ out[field] = value+ for _, entityType := range compilationValues(value) {+ if entityType == "mind_map" || entityType == "mindmap" {+ entityTypes := append([]string(nil), compilationValues(value)...)+ entityTypes = append(entityTypes, "mind_map", "mindmap")+ out[field] = entityTypes+ break+ }+ }+ case "page_type_kwd":+ alternatives = append(alternatives,+ map[string]any{"compile_kwd": "wiki", "entity_type_kwd": value},+ map[string]any{"page_type_kwd": value, "must_not": map[string]any{"compile_kwd": "wiki", "exists": "entity_type_kwd"}})+ case "and", "or":+ var children []map[string]any+ for _, child := range FilterClauses(value) {+ children = append(children, CompilationFilter(child))+ }+ out[field] = children+ default:+ out[field] = value+ }+ if len(alternatives) > 0 {+ clauses = append(clauses, map[string]any{"or": alternatives})+ }+ }+ if len(clauses) > 0 {+ if len(out) > 0 {+ clauses = append(clauses, out)+ }+ return map[string]any{"and": clauses}+ }+ return out+}++func wikiRoleFilter(role string) map[string]any {+ return map[string]any{"or": []map[string]any{+ {"compile_kwd": "wiki", "type_kwd": role},+ {"compile_kwd": role},+ }}+}++func compilationKindFilter(kind string) map[string]any {+ canonical := CanonicalCompilationKind(kind)+ aliases := []string{canonical}+ switch canonical {+ case "graph":+ aliases = []string{"graph", "knowledge_graph", "knowledgegraph", "structure"}+ case "mind_map":+ aliases = []string{"mind_map", "mindmap"}+ case "page_index":+ aliases = []string{"page_index", "pageindex"}+ }+ oldKeywords := append([]string(nil), aliases...)+ if canonical == "graph" {+ oldKeywords = append(oldKeywords, "hypergraph", "list", "set")+ }+ if canonical == "wiki" {+ oldKeywords = append(oldKeywords, "wiki_page", "wiki_section", "wiki_entity", "wiki_relation", "wiki_contribution_state", "wiki_map_active", "wiki_map_extract")+ }+ missingTemplateKind := []map[string]any{+ {"must_not": map[string]any{"exists": "compilation_template_kind_kwd"}},+ {"compilation_template_kind_kwd": []string{""}},+ }+ return map[string]any{"or": []map[string]any{+ {"compile_kwd": canonical, "or": missingTemplateKind},+ {"compilation_template_kind_kwd": aliases},+ {"compile_kwd": oldKeywords, "or": missingTemplateKind},+ }}+}++func FilterClauses(value any) []map[string]any {+ switch value := value.(type) {+ case []map[string]any:+ return value+ case []any:+ var out []map[string]any+ for _, child := range value {+ if child, ok := child.(map[string]any); ok {+ out = append(out, child)+ }+ }+ return out+ }+ return nil+}diff --git a/internal/engine/types/compilation_test.go b/internal/engine/types/compilation_test.gonew file mode 100644index 0000000000..694afa5b2a--- /dev/null+++ b/internal/engine/types/compilation_test.go@@ -0,0 +1,97 @@+package types++import "testing"++func TestCompilationKindPrefersCanonicalValue(t *testing.T) {+ for _, test := range []struct {+ row map[string]any+ want string+ }{+ {map[string]any{"compile_kwd": "graph"}, "graph"},+ {map[string]any{"compile_kwd": "timeline", "compilation_template_kind_kwd": "knowledge_graph"}, "graph"},+ {map[string]any{"compile_kwd": "list", "compilation_template_kind_kwd": "page_index"}, "page_index"},+ {map[string]any{"compile_kwd": "hypergraph"}, "graph"},+ {map[string]any{"compile_kwd": []string{"mindmap"}}, "mind_map"},+ {map[string]any{"compile_kwd": "wiki_section"}, "wiki"},+ } {+ if got := CompilationKind(test.row); got != test.want {+ t.Errorf("CompilationKind(%v) = %q, want %q", test.row, got, test.want)+ }+ }+}++func TestCompilationFilterReadsMixedGenerations(t *testing.T) {+ for _, test := range []struct {+ name string+ filter, row map[string]any+ want bool+ }{+ {"new graph", map[string]any{"compile_kwd": "graph", "kb_id": "kb1"}, map[string]any{"compile_kwd": "graph", "kb_id": "kb1"}, true},+ {"old graph", map[string]any{"compile_kwd": "graph"}, map[string]any{"compile_kwd": "hypergraph", "compilation_template_kind_kwd": "knowledge_graph"}, true},+ {"old inferred graph", map[string]any{"compile_kwd": "graph"}, map[string]any{"compile_kwd": "hypergraph"}, true},+ {"old page index inferred list", map[string]any{"compile_kwd": "page_index"}, map[string]any{"compile_kwd": "list", "compilation_template_kind_kwd": "page_index"}, true},+ {"no category mixing", map[string]any{"compile_kwd": "graph"}, map[string]any{"compile_kwd": "list", "compilation_template_kind_kwd": "page_index"}, false},+ {"canonical row has no old stamp", map[string]any{"compilation_template_kind_kwd": "timeline"}, map[string]any{"compile_kwd": "graph"}, false},+ {"old stamp preserves original kind", map[string]any{"compile_kwd": "graph"}, map[string]any{"compile_kwd": "timeline", "compilation_template_kind_kwd": "knowledge_graph"}, true},+ {"old subtype is not original kind", map[string]any{"compile_kwd": "timeline"}, map[string]any{"compile_kwd": "timeline", "compilation_template_kind_kwd": "knowledge_graph"}, false},+ {"dataset stays scoped", map[string]any{"compile_kwd": "graph", "kb_id": "kb1"}, map[string]any{"compile_kwd": "graph", "kb_id": "kb2"}, false},+ {"new wiki page", map[string]any{"compile_kwd": "wiki_page"}, map[string]any{"compile_kwd": "wiki", "type_kwd": "wiki_page"}, true},+ {"old wiki page", map[string]any{"type_kwd": "wiki_page"}, map[string]any{"compile_kwd": "wiki_page"}, true},+ {"section excluded", map[string]any{"compile_kwd": "wiki_page"}, map[string]any{"compile_kwd": "wiki", "type_kwd": "wiki_section"}, false},+ {"state excluded", map[string]any{"compile_kwd": "wiki_page"}, map[string]any{"compile_kwd": "wiki", "type_kwd": "wiki_map_active"}, false},+ {"new navigation", map[string]any{"compile_kwd": "dataset_nav"}, map[string]any{"compile_kwd": "page_index", "type_kwd": "nav_doc"}, true},+ {"structure excluded", map[string]any{"compile_kwd": "dataset_nav"}, map[string]any{"compile_kwd": "page_index", "type_kwd": "entity"}, false},+ {"new category", map[string]any{"page_type_kwd": "concept"}, map[string]any{"compile_kwd": "wiki", "entity_type_kwd": "concept"}, true},+ {"old category", map[string]any{"page_type_kwd": "concept"}, map[string]any{"compile_kwd": "wiki_page", "page_type_kwd": "concept"}, true},+ {"category wins", map[string]any{"page_type_kwd": "entity"}, map[string]any{"compile_kwd": "wiki", "entity_type_kwd": "concept", "page_type_kwd": "entity"}, false},+ {"mind map type reads old value", map[string]any{"entity_type_kwd": "mind_map"}, map[string]any{"entity_type_kwd": "mindmap"}, true},+ {"old mind map type reads new value", map[string]any{"entity_type_kwd": []string{"mindmap"}}, map[string]any{"entity_type_kwd": "mind_map"}, true},+ {"mind map type excludes other types", map[string]any{"entity_type_kwd": []any{"mind_map"}}, map[string]any{"entity_type_kwd": "person"}, false},+ } {+ t.Run(test.name, func(t *testing.T) {+ if got := matchesCompilationFilter(test.row, CompilationFilter(test.filter)); got != test.want {+ t.Fatalf("matched = %v, want %v", got, test.want)+ }+ })+ }+}++func matchesCompilationFilter(row, filter map[string]any) bool {+ for field, value := range filter {+ switch field {+ case "and":+ for _, child := range FilterClauses(value) {+ if !matchesCompilationFilter(row, child) {+ return false+ }+ }+ case "or":+ matched := false+ for _, child := range FilterClauses(value) {+ matched = matched || matchesCompilationFilter(row, child)+ }+ if !matched {+ return false+ }+ case "must_not":+ if matchesCompilationFilter(row, value.(map[string]any)) {+ return false+ }+ case "exists":+ if compilationString(row[value.(string)]) == "" {+ return false+ }+ default:+ matched := false+ for _, actual := range compilationValues(row[field]) {+ for _, expected := range compilationValues(value) {+ matched = matched || actual == expected+ }+ }+ if !matched {+ return false+ }+ }+ }+ return true+}diff --git a/internal/ingestion/component/knowledge_compiler/common/types.go b/internal/ingestion/component/knowledge_compiler/common/types.goindex a7070a7471..34c2136bb6 100644--- a/internal/ingestion/component/knowledge_compiler/common/types.go+++ b/internal/ingestion/component/knowledge_compiler/common/types.go@@ -127,10 +127,7 @@ type Product struct {DocID stringTenantID stringVariant Variant- // Kind is the original compilation_template.kind (e.g. "page_index"),- // distinct from Variant (the collapsed Go strategy, e.g. "structure"). It- // is stamped onto compilation_template_kind_kwd so the document-structure- // endpoint can group rows by the true template kind.+ // Kind identifies the producing template's knowledge compilation kind.Kind string// TemplateID is the compilation template that produced this row.TemplateID stringdiff --git a/internal/ingestion/component/knowledge_compiler/component.go b/internal/ingestion/component/knowledge_compiler/component.goindex 892a352488..49cbeae434 100644--- a/internal/ingestion/component/knowledge_compiler/component.go+++ b/internal/ingestion/component/knowledge_compiler/component.go@@ -6,12 +6,14 @@ package knowledge_compilerimport ("context"+ "encoding/json""fmt""sort""strings""ragflow/internal/agent/runtime"clog "ragflow/internal/common"+ enginetypes "ragflow/internal/engine/types""ragflow/internal/ingestion/component/globals""ragflow/internal/ingestion/component/knowledge_compiler/common""ragflow/internal/ingestion/component/knowledge_compiler/mindmap"@@ -367,9 +369,8 @@ func overlayTemplateConfig(param *common.Param, cfg map[string]any) {}}-// kindOrVariant returns the original template kind when present (the true-// compilation_template.kind, e.g. "page_index"), otherwise the collapsed Go-// variant. It drives the compilation_template_kind_kwd stamp.+// kindOrVariant returns the producing template kind, or the execution variant+// when a product has no template kind.func kindOrVariant(p common.Product) string {if p.Kind != "" {return p.Kind@@ -377,20 +378,6 @@ func kindOrVariant(p common.Product) string {return string(p.Variant)}-// variantCompileKWD maps each Go variant to the compile_kwd discriminator value-// Python writes into ES (rag/advanced_rag/knowlege_compile). It is the primary-// key that distinguishes compiled knowledge units from ordinary chunks and-// routes retrieval-side filters. The wiki value MUST be "wiki_page" (Python's-// canonical WIKI_PAGE_COMPILE_KWD in wiki.py:1661 / wiki_incremental.py:44 /-// dataset_wiki_generator.py:108) so Go-produced wiki pages are visible to the-// artifact API (dataset_artifact_service.go reads compile_kwd="wiki_page").-var variantCompileKWD = map[common.Variant]string{- common.VariantStructure: "structure",- common.VariantWiki: "wiki_page",- common.VariantTree: "tree",- common.VariantMindmap: "mindmap",-}-// productsToChunkDocs converts the internal compiled Product rows into// schema.ChunkDoc values aligned to conf/infinity_mapping.json. Product.Meta is// not persisted: only engine columns are written, because Infinity rejects an@@ -434,29 +421,20 @@ func productsToChunkDocs(products []common.Product) ([]schema.ChunkDoc, error) {return nil, err}}- compileKWD := variantCompileKWD[p.Variant]- // A variant may pin a finer-grained compile_kwd per row via Meta- // (structure stamps the inferred compile kind — list/set/hypergraph —- // mirroring Python's per-row autotype stamp).- if v := metaString(p.Meta, "compile_kwd"); v != "" {- compileKWD = v- }- // Wiki sub-parts: sections get their own compile_kwd so that a page- // search on compile_kwd="wiki_page" returns pages only (page.go emits- // both kind:"page" and kind:"section" rows under VariantWiki). This is- // the schema-backed page/section discriminator: "wiki_page" == page,- // "wiki_section" == a page sub-section.- if p.Variant == common.VariantWiki && metaString(p.Meta, "kind") == "section" && compileKWD == "wiki_page" {- compileKWD = "wiki_section"- }- if compileKWD == "" {- compileKWD = string(p.Variant)- }+ compileKWD := enginetypes.CanonicalCompilationKind(kindOrVariant(p))if err := doc.SetExtraValue("compile_kwd", compileKWD); err != nil {return nil, err}- if err := doc.SetExtraValue("compilation_template_kind_kwd", kindOrVariant(p)); err != nil {- return nil, err+ if p.Variant == common.VariantStructure {+ if inferred := metaString(p.Meta, "compile_kwd"); inferred != "" {+ extra, err := json.Marshal(map[string]string{"compile_type": inferred})+ if err != nil {+ return nil, err+ }+ if err := doc.SetExtraValue("extra", string(extra)); err != nil {+ return nil, err+ }+ }}// scope_kwd marks doc/dataset-level rows (B8/O1=B). A doc-level compiled// product is always a per-document (scope="doc") input to the dataset-level@@ -516,7 +494,12 @@ func applyVariantColumns(doc *schema.ChunkDoc, p common.Product) error {return applyStructureGraphColumns(doc, p, kind)case common.VariantWiki:- // One artifact_page row per wiki page; section rows reuse the same+ if kind != "" {+ if err := doc.SetExtraValue("type_kwd", "wiki_"+kind); err != nil {+ return err+ }+ }+ // Page and section roles reuse the same// page-level columns so retrieval-side filters work uniformly.// Match the Python writer contract (api/db/db_models.py slug_kwd):// slug_kwd stores the full "<page_type>/<slug>" form, so retrieval@@ -543,7 +526,7 @@ func applyVariantColumns(doc *schema.ChunkDoc, p common.Product) error {setTitleTokens(doc, v)}if v := metaString(p.Meta, "page_type"); v != "" {- if err := doc.SetExtraValue("page_type_kwd", v); err != nil {+ if err := doc.SetExtraValue("entity_type_kwd", v); err != nil {return err}}diff --git a/internal/ingestion/component/knowledge_compiler/component_test.go b/internal/ingestion/component/knowledge_compiler/component_test.goindex e26a3bcc1f..6a784fcc86 100644--- a/internal/ingestion/component/knowledge_compiler/component_test.go+++ b/internal/ingestion/component/knowledge_compiler/component_test.go@@ -353,7 +353,7 @@ func TestKnowledgeCompiler_Wiki_EndToEnd(t *testing.T) {// wiki produces a "page" chunk (kind carried by compile_kwd=wiki_page).foundPage := falsefor _, c := range chunks {- if kind, _ := c["compile_kwd"].(string); kind == "wiki_page" {+ if kind, _ := c["type_kwd"].(string); kind == "wiki_page" {foundPage = true}}@@ -431,7 +431,7 @@ func invokeWikiCompiler(t *testing.T, inputs map[string]any) []map[string]any {func hasWikiPageChunk(chunks []map[string]any) bool {for _, cm := range chunks {- if kind, _ := cm["compile_kwd"].(string); kind == "wiki_page" {+ if kind, _ := cm["type_kwd"].(string); kind == "wiki_page" {return true}}@@ -574,6 +574,9 @@ func TestKnowledgeCompiler_Mindmap_EndToEnd(t *testing.T) {kind, _ := c["type_kwd"].(string)if kind == "entity" {entityCount+++ if c["entity_type_kwd"] != "mind_map" {+ t.Fatalf("mindmap entity_type_kwd = %v, want mind_map", c["entity_type_kwd"])+ }if _, ok := c["name_kwd"]; !ok {t.Fatalf("mindmap entity chunk missing name_kwd: %+v", c)}@@ -629,7 +632,7 @@ func TestKnowledgeCompiler_EmitsChunks(t *testing.T) {cm := r.(map[string]any)// Structure rows carry the inferred compile kind (hypergraph here),// mirroring Python's per-row autotype stamp.- if ck, _ := cm["compile_kwd"].(string); ck == "hypergraph" {+ if ck, _ := cm["compile_kwd"].(string); ck == "graph" {compiled++if cm["id"] == nil || cm["id"] == "" {t.Fatal("compiled chunk missing id")@@ -674,7 +677,7 @@ func TestKnowledgeCompiler_TemplateIDsAndProvenance(t *testing.T) {}for _, r := range out["chunks"].([]any) {cm := r.(map[string]any)- if ck, _ := cm["compile_kwd"].(string); ck != "hypergraph" {+ if ck, _ := cm["compile_kwd"].(string); ck != "graph" {continue}// Every compiled chunk must carry the resolved template id (one per@@ -877,7 +880,7 @@ func TestKnowledgeCompiler_Wiki_UpdateMergesExistingPage(t *testing.T) {if !ok {continue}- if cm["compile_kwd"] == "wiki_page" && cm["slug_kwd"] == "entity/person/alpha" {+ if cm["type_kwd"] == "wiki_page" && cm["slug_kwd"] == "entity/person/alpha" {page = cmbreak}@@ -1203,7 +1206,7 @@ func TestKnowledgeCompiler_GroupIDsResolvedToTemplateIDs(t *testing.T) {cm := r.(map[string]any)// No parser_config is supplied, so InferType returns "list" and the// structure variant stamps compile_kwd="list" (not "structure").- if cm["compile_kwd"] != "list" {+ if cm["compile_kwd"] != "graph" {continue}checked++@@ -1444,7 +1447,7 @@ func TestKnowledgeCompiler_BuildInputsAcceptsMapSliceChunks(t *testing.T) {// TestProductsToChunkDocs_PageVsSectionCompileKWD locks the page/section// discriminator (compile_kwd) and the schema-column contract: the page body goes// to md_with_weight and no Go-only field (kc_*, tenant_id) is emitted.-func TestProductsToChunkDocs_PageVsSectionCompileKWD(t *testing.T) {+func TestProductsToChunkDocs_WikiPageAndSectionRoles(t *testing.T) {page := common.Product{ID: "page-id", DocID: "d1", TenantID: "t1", Variant: common.VariantWiki,Content: "# Alpha\n\nBody", ParentID: "",@@ -1462,9 +1465,17 @@ func TestProductsToChunkDocs_PageVsSectionCompileKWD(t *testing.T) {var pageKWD, sectionKWD stringvar sectionParent, pageBody, sectionBody stringfor _, d := range docs {+ if compiled, _ := d.GetExtraString("compile_kwd"); compiled != "wiki" {+ t.Fatalf("compile_kwd = %q, want wiki", compiled)+ }+ for _, field := range []string{"compilation_template_kind_kwd", "page_type_kwd"} {+ if _, exists := d.Extra[field]; exists {+ t.Fatalf("retired field %s was written", field)+ }+ }// compile_kwd IS the page/section discriminator (wiki_page /// wiki_section); there is no separate kind column.- kind, _ := d.GetExtraString("compile_kwd")+ kind, _ := d.GetExtraString("type_kwd")if kind == "wiki_page" {pageKWD = kindpageBody, _ = d.GetExtraString("md_with_weight")diff --git a/internal/ingestion/component/knowledge_compiler/golden_test.go b/internal/ingestion/component/knowledge_compiler/golden_test.goindex acbd2b621f..88d828fc9c 100644--- a/internal/ingestion/component/knowledge_compiler/golden_test.go+++ b/internal/ingestion/component/knowledge_compiler/golden_test.go@@ -210,7 +210,7 @@ func TestGolden_Wiki_ProductCount(t *testing.T) {foundPage := falsefor _, p := range prods {// compile_kwd=wiki_page is the page/section discriminator.- if kwd, _ := p.GetExtraString("compile_kwd"); kwd == "wiki_page" {+ if kwd, _ := p.GetExtraString("type_kwd"); kwd == "wiki_page" {foundPage = true}}diff --git a/internal/ingestion/component/knowledge_compiler/mindmap/mindmap.go b/internal/ingestion/component/knowledge_compiler/mindmap/mindmap.goindex 2376bb2949..97d527342f 100644--- a/internal/ingestion/component/knowledge_compiler/mindmap/mindmap.go+++ b/internal/ingestion/component/knowledge_compiler/mindmap/mindmap.go@@ -149,7 +149,7 @@ func Run(ctx context.Context, deps common.Deps, param common.Param, inputs commo//// Mapping:// - each node (including the root) → an entity product (kind="entity",-// name = node id, type = "mindmap").+// name = node id, type = "mind_map").// - each parent→child edge → a relation product (kind="relation",// from = parent id, to = child id, type = "related", matching Python's// mindmap structure-graph projection).@@ -165,12 +165,12 @@ func treeToProducts(tenantID, docID string, root *utility.Node, fallbackSourceChseen := map[string]bool{}// Entity: root node.rootSourceChunkIDs := nodeSourceChunkIDs(root, fallbackSourceChunkIDs)- rootPayload := map[string]any{"name": root.ID, "type": "mindmap"}+ rootPayload := map[string]any{"name": root.ID, "type": "mind_map"}rootMeta := map[string]any{"kind": "entity","name": root.ID,- "entity_type": "mindmap",- "compile_kwd": "mindmap",+ "entity_type": "mind_map",+ "compile_kwd": "mind_map",}if len(rootSourceChunkIDs) > 0 {rootPayload["source_chunk_ids"] = rootSourceChunkIDs@@ -203,12 +203,12 @@ func treeToProducts(tenantID, docID string, root *utility.Node, fallbackSourceChif !seen[child.ID] {seen[child.ID] = truechildSourceChunkIDs := nodeSourceChunkIDs(child, fallbackSourceChunkIDs)- childPayload := map[string]any{"name": child.ID, "type": "mindmap"}+ childPayload := map[string]any{"name": child.ID, "type": "mind_map"}childMeta := map[string]any{"kind": "entity","name": child.ID,- "entity_type": "mindmap",- "compile_kwd": "mindmap",+ "entity_type": "mind_map",+ "compile_kwd": "mind_map",}if len(childSourceChunkIDs) > 0 {childPayload["source_chunk_ids"] = childSourceChunkIDs@@ -242,7 +242,7 @@ func treeToProducts(tenantID, docID string, root *utility.Node, fallbackSourceCh"from": p.parent,"to": child.ID,"relation_type": "related",- "compile_kwd": "mindmap",+ "compile_kwd": "mind_map",},})}diff --git a/internal/ingestion/component/knowledge_compiler/mindmap/mindmap_test.go b/internal/ingestion/component/knowledge_compiler/mindmap/mindmap_test.goindex 7afc3944b5..99f3e95ce3 100644--- a/internal/ingestion/component/knowledge_compiler/mindmap/mindmap_test.go+++ b/internal/ingestion/component/knowledge_compiler/mindmap/mindmap_test.go@@ -62,6 +62,9 @@ func TestTreeToProducts_ParentLinks(t *testing.T) {entCount, relCount := 0, 0fromTo := map[string]bool{}for _, p := range products {+ if p.Meta["compile_kwd"] != "mind_map" {+ t.Errorf("product %s compile_kwd = %v, want mind_map", p.ID, p.Meta["compile_kwd"])+ }kind, _ := p.Meta["kind"].(string)switch kind {case "entity":@@ -70,14 +73,11 @@ func TestTreeToProducts_ParentLinks(t *testing.T) {if err := json.Unmarshal([]byte(p.Content), &payload); err != nil {t.Fatalf("entity %q content is not JSON: %v", p.Meta["name"], err)}- if payload["name"] != p.Meta["name"] || payload["type"] != "mindmap" {- t.Errorf("entity payload = %v, want name=%v and type=mindmap", payload, p.Meta["name"])- }- if p.Meta["entity_type"] != "mindmap" {- t.Errorf("entity %v type = %v, want mindmap", p.Meta["name"], p.Meta["entity_type"])+ if payload["name"] != p.Meta["name"] || payload["type"] != "mind_map" {+ t.Errorf("entity payload = %v, want name=%v and type=mind_map", payload, p.Meta["name"])}- if p.Meta["compile_kwd"] != "mindmap" {- t.Errorf("entity %v compile_kwd = %v, want mindmap", p.Meta["name"], p.Meta["compile_kwd"])+ if p.Meta["entity_type"] != "mind_map" {+ t.Errorf("entity %v type = %v, want mind_map", p.Meta["name"], p.Meta["entity_type"])}case "relation":relCount++diff --git a/internal/ingestion/component/knowledge_compiler/storage_test.go b/internal/ingestion/component/knowledge_compiler/storage_test.gonew file mode 100644index 0000000000..1cb27a3fd3--- /dev/null+++ b/internal/ingestion/component/knowledge_compiler/storage_test.go@@ -0,0 +1,54 @@+package knowledge_compiler++import (+ "encoding/json"+ "testing"++ "ragflow/internal/ingestion/component/knowledge_compiler/common"+)++func TestProductsToChunkDocsCanonicalKinds(t *testing.T) {+ for _, test := range []struct {+ kind string+ variant common.Variant+ want string+ }{+ {"knowledge_graph", common.VariantStructure, "graph"},+ {"graph", common.VariantStructure, "graph"},+ {"mind_map", common.VariantMindmap, "mind_map"},+ {"mindmap", common.VariantMindmap, "mind_map"},+ {"page_index", common.VariantStructure, "page_index"},+ {"tree", common.VariantTree, "tree"},+ {"timeline", common.VariantStructure, "timeline"},+ {"wiki", common.VariantWiki, "wiki"},+ } {+ t.Run(test.kind, func(t *testing.T) {+ docs, err := productsToChunkDocs([]common.Product{{ID: "row", Kind: test.kind, Variant: test.variant, Content: "summary",+ Meta: map[string]any{"compile_kwd": "list", "kind": "entity", "page_type": "concept"}}})+ if err != nil {+ t.Fatal(err)+ }+ row := docs[0].ToMap()+ if row["compile_kwd"] != test.want {+ t.Fatalf("compile_kwd = %v, want %s", row["compile_kwd"], test.want)+ }+ for _, field := range []string{"compilation_template_kind_kwd", "page_type_kwd"} {+ if _, exists := row[field]; exists {+ t.Fatalf("retired field %s was written", field)+ }+ }+ if test.variant == common.VariantStructure {+ var extra map[string]string+ if err := json.Unmarshal([]byte(row["extra"].(string)), &extra); err != nil {+ t.Fatal(err)+ }+ if extra["compile_type"] != "list" {+ t.Fatalf("compile type = %v", extra)+ }+ }+ if test.variant == common.VariantWiki && (row["type_kwd"] != "wiki_entity" || row["entity_type_kwd"] != "concept") {+ t.Fatalf("wiki role/category = %v/%v", row["type_kwd"], row["entity_type_kwd"])+ }+ })+ }+}diff --git a/internal/ingestion/knowledge_compile/consumer.go b/internal/ingestion/knowledge_compile/consumer.goindex 08810af73d..279c65371d 100644--- a/internal/ingestion/knowledge_compile/consumer.go+++ b/internal/ingestion/knowledge_compile/consumer.go@@ -28,6 +28,7 @@ import ("ragflow/internal/common""ragflow/internal/dao""ragflow/internal/engine"+ enginetypes "ragflow/internal/engine/types"kccommon "ragflow/internal/ingestion/component/knowledge_compiler/common""ragflow/internal/service/file""ragflow/internal/service/nav"@@ -377,8 +378,8 @@ func (c *Consumer) RebuildDataset(ctx context.Context, tenant, kb, mode string)// carry the doc's compile variants so the consumer can route to the correct// dataset-level path (a nil/empty Variants would fall back to the legacy// unified path and tree nav could not be rebuilt). The variants are recovered- // from the doc-level compiled products' authoritative `compilation_template_kind_kwd`- // (via KindToVariant, O2a whitelist hard-fail), not re-derived from `compile_kwd`.+ // from the doc-level compiled products' canonical compilation kind+ // via the shared variant mapping.docs, err := c.docLister(ctx, tenant, kb)if err != nil {return fmt.Errorf("knowledge_compile: rebuild list docs: %w", err)@@ -421,7 +422,7 @@ func defaultDocLister(ctx context.Context, tenant, kb string) ([]string, error)// recoverDocTypes reconstructs the compile-type and frontend task-type sets a// document produced by reading its persisted doc-level products and mapping-// each product's authoritative `compilation_template_kind_kwd` through the+// each product's canonical compilation kind through the// shared kind mappings.// It is used by RebuildDataset (B1) so the republished completed events carry// the variants needed to route the dataset-level re-compile (tree nav, structure@@ -444,7 +445,7 @@ func (c *Consumer) recoverDocTypes(ctx context.Context, tenant, kb, docID stringseenTaskTypes := map[string]struct{}{}var variants []stringfor _, p := range products {- // Authoritative: product.Kind (compilation_template_kind_kwd) when present.+ // Use the compilation kind restored by the reader.if p.Kind != "" {v, err := kccommon.KindToVariant(p.Kind)if err != nil {@@ -884,7 +885,7 @@ func (c *Consumer) processBatch(ctx context.Context, tenant, kb, token string, efor _, cand := range candidates {v := string(cand.Variant)byVariant[v]++- if v == "wiki_page" {+ if cand.Variant == kccommon.VariantWiki {wikiPageSlugs = append(wikiPageSlugs, candidateIdentity(cand))}}@@ -1523,9 +1524,11 @@ func navInputFromProducts(kb string, products []kccommon.Product) []nav.UpsertDo// otherwise use the folded structure/page-index summary. This makes the// result independent of product iteration order.if strings.TrimSpace(a.treeSummary) != "" {+ a.in.CompileKind = "tree"a.in.Summary = a.treeSummarya.in.Embedd = a.treeEmbedd} else {+ a.in.CompileKind = "page_index"a.in.Summary = strings.Join(a.structureLines, "\n")}if strings.TrimSpace(a.in.Summary) == "" {@@ -1585,16 +1588,9 @@ func (c *Consumer) mergeStructureDataset(ctx context.Context, tenant, kb string,if p.Variant != kccommon.VariantStructure && p.Variant != kccommon.VariantMindmap {continue}- // Dataset rows carry the SAME compile_kwd as the doc rows (the inferred- // compile type / autotype: "hypergraph"/"timeline"/"mindmap"/"list"/…),- // mirroring Python dataset_structure_merger._do_build which passes the doc- // row's compile_kwd through verbatim. The template kind (p.Kind, restored- // from compilation_template_kind_kwd) is stored on the SEPARATE- // compilation_template_kind_kwd field and is what read/delete paths match- // on — NOT compile_kwd. Do not rewrite compile_kwd to the template kind:- // that would diverge from Python (which never does) and split doc vs- // dataset rows.- ckwd := metaString(p.Meta, "compile_kwd")+ // Dataset rows retain the producing compilation kind. Extraction shape+ // is separate metadata and does not determine the stored kind.+ ckwd := enginetypes.CanonicalCompilationKind(kccommon.FirstNonEmpty(p.Kind, metaString(p.Meta, "compile_kwd")))if ckwd == "" {ckwd = compileKwdForVariant(p.Variant)}@@ -1619,7 +1615,7 @@ func (c *Consumer) mergeStructureDataset(ctx context.Context, tenant, kb string,k := rkey{template: template, from: normalizedStructureEntityName(from), typ: structureRelationType(relType), to: normalizedStructureEntityName(to), ckwd: ckwd}b := relByKey[k]if b == nil {- b = &StructureBucket{Name: from + " -> " + to, Type: "relation", FromEntity: from, ToEntity: to, CompileKwd: ckwd, TemplateID: p.TemplateID, TemplateKind: p.Kind, RelationType: relType}+ b = &StructureBucket{Name: from + " -> " + to, Type: "relation", FromEntity: from, ToEntity: to, CompileKwd: ckwd, TemplateID: p.TemplateID, TemplateKind: p.Kind, CompileType: metaString(p.Meta, "compile_type"), RelationType: relType}relByKey[k] = b}appendBucket(b, p)@@ -1637,7 +1633,7 @@ func (c *Consumer) mergeStructureDataset(ctx context.Context, tenant, kb string,k := ekey{template: template, name: normalizedStructureEntityName(name), ckwd: ckwd}b := entByKey[k]if b == nil {- b = &StructureBucket{Name: name, Type: typ, CompileKwd: ckwd, TemplateID: p.TemplateID, TemplateKind: p.Kind}+ b = &StructureBucket{Name: name, Type: typ, CompileKwd: ckwd, TemplateID: p.TemplateID, TemplateKind: p.Kind, CompileType: metaString(p.Meta, "compile_type")}entByKey[k] = b} else {b.Type = preferredStructureEntityType(b.Type, typ)@@ -1675,6 +1671,9 @@ func normalizedStructureEntityName(name string) string {func preferredStructureEntityType(existing, incoming string) string {existing = strings.TrimSpace(existing)incoming = strings.TrimSpace(incoming)+ if existing == "mindmap" && incoming == "mind_map" {+ return incoming+ }if existing == "" || strings.EqualFold(existing, "other") {if incoming != "" && !strings.EqualFold(incoming, "other") {return incoming@@ -1690,7 +1689,7 @@ func structureTemplateIdentity(templateID, templateKind string) string {if templateID = strings.TrimSpace(templateID); templateID != "" {return templateID}- return strings.TrimSpace(templateKind)+ return enginetypes.CanonicalCompilationKind(templateKind)}// appendBucket folds a structure product into a bucket: concatenates itsdiff --git a/internal/ingestion/knowledge_compile/nav_dispatch_test.go b/internal/ingestion/knowledge_compile/nav_dispatch_test.goindex 36e146ba98..9763019af5 100644--- a/internal/ingestion/knowledge_compile/nav_dispatch_test.go+++ b/internal/ingestion/knowledge_compile/nav_dispatch_test.go@@ -60,6 +60,9 @@ func TestNavInputFromProducts_TreeAndPageIndex(t *testing.T) {if !ok {t.Fatal("missing PageIndex nav input for d2")}+ if treeIn.CompileKind != "tree" || pageIndexIn.CompileKind != "page_index" {+ t.Fatalf("navigation kinds = %q/%q", treeIn.CompileKind, pageIndexIn.CompileKind)+ }if _, found := byDoc["d3"]; found {t.Fatal("Graph structure product must not produce navigation input")}diff --git a/internal/ingestion/knowledge_compile/reader.go b/internal/ingestion/knowledge_compile/reader.goindex 374dff8155..fcd401fce6 100644--- a/internal/ingestion/knowledge_compile/reader.go+++ b/internal/ingestion/knowledge_compile/reader.go@@ -102,7 +102,7 @@ type engineReader struct {// unknown column); payload = content_with_weight, kind = compile_kwd.var compiledSelectFields = []string{"id", "doc_id", "compile_kwd",- "available_int",+ "available_int", "extra","content_with_weight","source_chunk_ids", "source_doc_ids","name_kwd", "entity_type_kwd", "from_entity_kwd", "to_entity_kwd",@@ -133,7 +133,7 @@ var compiledSelectFields = []string{// in _source; the Infinity engine expands it against the table's real columns// because Infinity's SQL binder rejects a partial wildcard (3013).var wikiSelectFields = []string{- "page_type_kwd", "topic_kwd", "plan_group_kwd", "generation_kwd", "title_kwd",+ "page_type_kwd", "entity_type_kwd", "type_kwd", "topic_kwd", "plan_group_kwd", "generation_kwd", "title_kwd","entity_names_kwd", "summary_with_weight","related_kb_pages_kwd", "outlinks_kwd", "depth_int","q_*_vec",@@ -172,7 +172,7 @@ func (r engineReader) LoadDocProducts(ctx context.Context, tenant, kb, docID str}for _, c := range res.Chunks {// Only compiled products carry compile_kwd; skip ordinary source chunks.- if _, ok := c["compile_kwd"]; !ok {+ if _, ok := c["compile_kwd"]; !ok || types.IsNavigationRow(c) {continue}// Reverse-map the row's compile_kwd; reject dirty/unknown kinds@@ -246,7 +246,7 @@ func (r engineReader) LoadMergedWikiPages(ctx context.Context, tenant, kb stringif eng == nil {return nil, nil}- filter := map[string]interface{}{"available_int": 1, "scope_kwd": "dataset", "compile_kwd": compileKwdWikiPage}+ filter := map[string]interface{}{"available_int": 1, "scope_kwd": "dataset", "type_kwd": compileKwdWikiPage}var pages []kccommon.Productfor offset := 0; ; offset += loadDocProductsLimit {res, err := eng.Search(ctx, &types.SearchRequest{@@ -291,7 +291,7 @@ func (r engineReader) LoadDocumentWikiPagesBySlugs(ctx context.Context, tenant,}filter := map[string]interface{}{"available_int": 0,- "compile_kwd": compileKwdWikiPage,+ "type_kwd": compileKwdWikiPage,"slug_kwd": slugs,}if statusAvailable {@@ -352,25 +352,32 @@ func productFromChunkMap(c map[string]interface{}, tenant string, expect kccommoif err != nil || mapped != expect {return kccommon.Product{}, false}- // Round-trip the raw compile_kwd (the inferred compile type / autotype:- // list/set/hypergraph/timeline/mindmap/…) so the dataset-level merge can- // stamp the SAME value on the dataset row as the doc row (Python _do_build- // carries the doc row's compile_kwd through verbatim). Without this, the- // merge falls back to compileKwdForVariant ("structure"/"mindmap"), which- // diverges from the doc row's autotype ("hypergraph"/"timeline").+ // Preserve compilation metadata for the dataset merge.merged := isAvailable(c["available_int"])meta := map[string]any{}- // Preserve the raw compile_kwd (autotype) for the dataset merge (see the- // round-trip note above). A non-string scalar from the engine is normalized- // via asString, matching the variant reverse-map at the top of this func.+ if raw := asString(c["extra"]); raw != "" {+ var extra map[string]any+ if err := json.Unmarshal([]byte(raw), &extra); err != nil {+ return kccommon.Product{}, false+ }+ if inferred, ok := extra["compile_type"].(string); ok {+ meta["compile_type"] = inferred+ }+ }if v := asString(c["compile_kwd"]); v != "" {meta["compile_kwd"] = v+ if _, ok := meta["compile_type"]; !ok && types.CanonicalCompilationKind(v) != v {+ meta["compile_type"] = v+ }}if v, ok := c["name_kwd"].(string); ok && v != "" {meta["name"] = v}if v, ok := c["entity_type_kwd"].(string); ok && v != "" {+ if expect == kccommon.VariantMindmap && v == "mindmap" {+ v = "mind_map"+ }meta["entity_type"] = v}if v, ok := c["from_entity_kwd"].(string); ok && v != "" {@@ -391,7 +398,7 @@ func productFromChunkMap(c map[string]interface{}, tenant string, expect kccommo}// Restore wiki page fields so the merged product (and hence the dataset-level// merged row) retains the metadata the artifact API and page renderers read.- if v, ok := c["page_type_kwd"].(string); ok && v != "" {+ if v := types.WikiPageCategory(c); v != "" {meta["page_type"] = v}if v, ok := c["topic_kwd"].(string); ok && v != "" {@@ -437,12 +444,10 @@ func productFromChunkMap(c map[string]interface{}, tenant string, expect kccommometa["kind"] = "entity"}}- // compile_kwd IS the page/section discriminator (there is no kind column):- // wiki_page -> "page", wiki_section -> "section". Without it the- // processBatch "Meta.kind==page" filter would drop every wiki page.- if variant == compileKwdWikiPage {+ // Page and section roles share the wiki compilation kind.+ if types.CompilationRowType(c) == compileKwdWikiPage {meta["kind"] = "page"- } else if variant == compileKwdWikiSection {+ } else if types.CompilationRowType(c) == compileKwdWikiSection {meta["kind"] = "section"}if v := asString(c["plan_group_kwd"]); v != "" {@@ -469,10 +474,8 @@ func productFromChunkMap(c map[string]interface{}, tenant string, expect kccommo}vec, _ := kccommon.VectorFromChunkMap(c, 0)- // Restore the authoritative template kind (compilation_template_kind_kwd)- // so callers (e.g. RebuildDataset's variant recovery, B1a) can map it back- // via KindToVariant instead of re-deriving from the ambiguous compile_kwd.- kind := asString(c["compilation_template_kind_kwd"])+ // Restore the producing compilation kind for routing and merging.+ kind := types.CompilationKind(c)// Restore the compilation template id (from compilation_template_ids) so the// dataset merge can bucket structure rows per template and read/delete paths// can filter by it. A row should carry exactly one template id; if it carries@@ -516,7 +519,7 @@ func (r engineReader) SearchSimilar(ctx context.Context, tenant, kb string, variLimit: topN,SelectFields: append([]string{"id", "doc_id", "kb_id", "content_with_weight","name_kwd", "entity_type_kwd", "from_entity_kwd", "to_entity_kwd", "slug_kwd",- "source_chunk_ids", "source_doc_ids", "available_int", "compile_kwd",+ "source_chunk_ids", "source_doc_ids", "available_int", "compile_kwd", "type_kwd","create_timestamp_flt", "create_time"},wikiSelectFields...),Filter: map[string]interface{}{@@ -533,24 +536,17 @@ func (r engineReader) SearchSimilar(ctx context.Context, tenant, kb string, vari},},}+ if variant == kccommon.VariantWiki {+ req.Filter["type_kwd"] = compileKwdWikiPage+ }res, err := eng.Search(ctx, req)if err != nil {return kccommon.Product{}, 0, err}- // The KNN Filter scopes compile_kwd, but a foreign/legacy row that slipped- // past it must not poison the merge candidate. Reject by the raw compile_kwd- // (wiki_page vs wiki_section are distinct keywords even though both map to- // VariantWiki); the page/section distinction is resolved downstream by- // Meta.kind (see productFromChunkMap).+ // Verify the row family and role even when a backend returns foreign rows.expectKwd := compileKwdForVariant(variant)for _, c := range res.Chunks {- // The KNN Filter already scopes compile_kwd, but a foreign/legacy row that- // slipped past it must not poison the merge candidate. Reject by the- // reverse-mapped variant (the dirty-row contract): productFromChunkMap- // validates KwdToVariant(c) == variant and drops dirty/unknown kinds. The- // raw-keyword check below is a fast pre-filter before the full product- // reconstruction.- if asString(c["compile_kwd"]) != expectKwd {+ if types.CompilationKind(c) != expectKwd || (variant == kccommon.VariantWiki && types.CompilationRowType(c) != compileKwdWikiPage) {continue}p, ok := productFromChunkMap(c, tenant, variant)diff --git a/internal/ingestion/knowledge_compile/reader_test.go b/internal/ingestion/knowledge_compile/reader_test.goindex ae62dffa0a..42a21289aa 100644--- a/internal/ingestion/knowledge_compile/reader_test.go+++ b/internal/ingestion/knowledge_compile/reader_test.go@@ -131,7 +131,7 @@ func TestSearchSimilarFiltersByVariant(t *testing.T) {if eng.lastSearchReq.Filter["available_int"] != 1 {t.Fatalf("expected available_int=1 filter, got %v", eng.lastSearchReq.Filter["available_int"])}- if eng.lastSearchReq.Filter["compile_kwd"] != string(compileKwdWikiPage) {+ if eng.lastSearchReq.Filter["compile_kwd"] != "wiki" || eng.lastSearchReq.Filter["type_kwd"] != compileKwdWikiPage {t.Fatalf("expected compile_kwd=%q filter, got %v", compileKwdWikiPage, eng.lastSearchReq.Filter["compile_kwd"])}// SelectFields must include the round-trip columns added in the 8th review.diff --git a/internal/ingestion/knowledge_compile/storage.md b/internal/ingestion/knowledge_compile/storage.mdnew file mode 100644index 0000000000..8d8f82c1af--- /dev/null+++ b/internal/ingestion/knowledge_compile/storage.md@@ -0,0 +1,42 @@+# Knowledge compilation storage++`compile_kwd` identifies the producing knowledge compilation kind. New rows use+`graph`, `mind_map`, `page_index`, `tree`, `timeline`, or `wiki`. The existing+session kinds remain `session_graph` and `session_essence`.++`type_kwd` identifies the row's role:++| Compilation | Roles |+| --- | --- |+| Graph, mind map, page index | `entity`, `relation`, `kg_build_meta` |+| Tree | Existing entity/relation roles; root/summary metadata remains in `raptor_kwd` |+| Navigation from tree/page index | `nav_doc`, `nav_cluster` |+| Wiki | `wiki_page`, `wiki_section`, `wiki_entity`, `wiki_relation`, `wiki_contribution_state`, `wiki_map_active`, `wiki_map_extract` |++Wiki page categories such as `concept`, `entity`, and `topic` are stored in+`entity_type_kwd`. Inferred structure extraction types such as `list`, `set`,+and `hypergraph` are retained as `compile_type` inside the JSON string in `extra`.+Neither changes the compilation kind.++New writes omit `compilation_template_kind_kwd` and `page_type_kwd`.+Their engine columns remain available for reading existing data.++Navigation document leaves retain the source compilation kind. Topic clusters+retain the kind of the compilation that created them; clusters can contain+documents compiled by both tree and page index. Navigation selection uses the+row role, so these clusters remain accessible across both kinds. Navigation+rows remain hidden from ordinary retrieval with `available_int=0`.++Elasticsearch and Infinity translate compilation predicates into one query+covering both stored generations. For example, selecting the `wiki_page` role+matches either `(compile_kwd=wiki AND type_kwd=wiki_page)` or the old+`compile_kwd=wiki_page`. Compilation-kind reads prefer canonical `compile_kwd`+values on rows without the retired template-kind stamp. Stamped rows retain+their original template kind, including when their inferred type happens to+equal a canonical kind. Wiki category reads prefer `entity_type_kwd` and+otherwise read `page_type_kwd`. Tenant/dataset and document constraints remain+outside the alternative branches.++Rebuild cleanup selects navigation by role and merged structure rows by dataset+scope. It must not delete all rows sharing a compilation kind, since document+inputs and dataset outputs now use the same canonical value.diff --git a/internal/ingestion/knowledge_compile/structure_merge_test.go b/internal/ingestion/knowledge_compile/structure_merge_test.goindex eb30b2d8e2..33e7c839e0 100644--- a/internal/ingestion/knowledge_compile/structure_merge_test.go+++ b/internal/ingestion/knowledge_compile/structure_merge_test.go@@ -252,13 +252,8 @@ func TestMergeStructureDataset_DescriptionFallbacks(t *testing.T) {}// TestMergeStructureDataset_CompileKwdIsAutotypeNotTemplateKind covers the-// option-A alignment: the dataset row's compile_kwd is the doc row's inferred-// compile type (autotype, e.g. "hypergraph"/"mindmap"), NOT the template kind.-// The template kind travels on the separate TemplateKind field (stamped to-// compilation_template_kind_kwd by WriteMergedStructure), which is what read/-// delete paths match on. This mirrors Python _do_build (compile_kwd passes-// through verbatim) + get_dataset_structure (matches template kind).-func TestMergeStructureDataset_CompileKwdIsAutotypeNotTemplateKind(t *testing.T) {+// Dataset rows retain the producing compilation kind, independent of extraction shape.+func TestMergeStructureDataset_CompileKwdIsCanonicalKind(t *testing.T) {c := &Consumer{writer: &fakeWriter{}}products := []kccommon.Product{{Variant: kccommon.VariantStructure, DocID: "d1", Kind: "knowledge_graph", TemplateID: "tpl-graph",@@ -279,8 +274,8 @@ func TestMergeStructureDataset_CompileKwdIsAutotypeNotTemplateKind(t *testing.T)b := fw.buckets[i]switch b.Name {case "Engine":- if b.CompileKwd != "hypergraph" {- t.Errorf("Engine compile_kwd = %q, want autotype \"hypergraph\" (not template kind)", b.CompileKwd)+ if b.CompileKwd != "graph" {+ t.Errorf("Engine compile_kwd = %q, want \"graph\"", b.CompileKwd)}if b.TemplateKind != "knowledge_graph" {t.Errorf("Engine TemplateKind = %q, want \"knowledge_graph\"", b.TemplateKind)@@ -289,8 +284,8 @@ func TestMergeStructureDataset_CompileKwdIsAutotypeNotTemplateKind(t *testing.T)t.Errorf("Engine TemplateID = %q, want \"tpl-graph\"", b.TemplateID)}case "Fuel":- if b.CompileKwd != "mindmap" {- t.Errorf("Fuel compile_kwd = %q, want autotype \"mindmap\" (not template kind)", b.CompileKwd)+ if b.CompileKwd != "mind_map" {+ t.Errorf("Fuel compile_kwd = %q, want \"mind_map\"", b.CompileKwd)}if b.TemplateKind != "mind_map" {t.Errorf("Fuel TemplateKind = %q, want \"mind_map\"", b.TemplateKind)diff --git a/internal/ingestion/knowledge_compile/wiki_contribution.go b/internal/ingestion/knowledge_compile/wiki_contribution.goindex ff3ba9fa8e..ad872543e1 100644--- a/internal/ingestion/knowledge_compile/wiki_contribution.go+++ b/internal/ingestion/knowledge_compile/wiki_contribution.go@@ -68,10 +68,10 @@ func (s *engineWikiContributionStore) Get(ctx context.Context, tenantID, datasetIndexNames: []string{fmt.Sprintf("ragflow_%s", tenantID)},KbIDs: []string{datasetID},Limit: 1,- SelectFields: []string{"id", "compile_kwd", "content_with_weight"},+ SelectFields: []string{"id", "compile_kwd", "type_kwd", "content_with_weight"},Filter: map[string]any{"id": []string{key},- "compile_kwd": wikiContributionCompileKWD,+ "type_kwd": wikiContributionCompileKWD,"available_int": 0,},})@@ -100,7 +100,8 @@ func (s *engineWikiContributionStore) Put(ctx context.Context, tenantID, dataset"id": wikiContributionStateID(datasetID, contribution.DocumentID),"doc_id": "wiki_contribution:" + contribution.DocumentID,"kb_id": datasetID,- "compile_kwd": wikiContributionCompileKWD,+ "compile_kwd": "wiki",+ "type_kwd": wikiContributionCompileKWD,"scope_kwd": "doc","source_doc_ids": []string{contribution.DocumentID},"content_with_weight": string(payload),@@ -121,7 +122,7 @@ func (s *engineWikiContributionStore) Delete(ctx context.Context, tenantID, datafunc (s *engineWikiContributionStore) ClearDataset(ctx context.Context, tenantID, datasetID string) error {_, err := s.engine.DeleteChunks(ctx, map[string]any{"kb_id": datasetID,- "compile_kwd": wikiContributionCompileKWD,+ "type_kwd": wikiContributionCompileKWD,"available_int": 0,}, fmt.Sprintf("ragflow_%s", tenantID), datasetID)return errdiff --git a/internal/ingestion/knowledge_compile/wiki_graph_test.go b/internal/ingestion/knowledge_compile/wiki_graph_test.goindex fe0d516a88..e79e1f4566 100644--- a/internal/ingestion/knowledge_compile/wiki_graph_test.go+++ b/internal/ingestion/knowledge_compile/wiki_graph_test.go@@ -34,7 +34,7 @@ func page(pageType, slug string, outlinks ...string) wikiPageProjection {func findEntity(rows []map[string]interface{}, slug string) map[string]interface{} {for _, r := range rows {- if r["compile_kwd"] == compileKwdWikiEntity && r["slug_kwd"] == slug {+ if r["compile_kwd"] == "wiki" && r["type_kwd"] == compileKwdWikiEntity && r["slug_kwd"] == slug {return r}}@@ -43,7 +43,7 @@ func findEntity(rows []map[string]interface{}, slug string) map[string]interfacefunc findRelation(rows []map[string]interface{}, from, to string) map[string]interface{} {for _, r := range rows {- if r["compile_kwd"] == compileKwdWikiRelation && r["from_kwd"] == from && r["to_kwd"] == to {+ if r["compile_kwd"] == "wiki" && r["type_kwd"] == compileKwdWikiRelation && r["from_kwd"] == from && r["to_kwd"] == to {return r}}@@ -252,8 +252,8 @@ func TestProjectWikiGraphRowsFieldMapping(t *testing.T) {if e == nil {t.Fatalf("missing entity")}- if e["entity_type_kwd"] != "wiki_concept" {- t.Fatalf("expected entity_type_kwd=wiki_concept, got %v", e["entity_type_kwd"])+ if e["entity_type_kwd"] != "concept" {+ t.Fatalf("expected entity_type_kwd=concept, got %v", e["entity_type_kwd"])}// id format: wiki_entity:{kb}:{slug} xxhashed to 16 hex chars.id, _ := e["id"].(string)diff --git a/internal/ingestion/knowledge_compile/wiki_map_store.go b/internal/ingestion/knowledge_compile/wiki_map_store.goindex 6d72e3e843..6c6921008e 100644--- a/internal/ingestion/knowledge_compile/wiki_map_store.go+++ b/internal/ingestion/knowledge_compile/wiki_map_store.go@@ -46,10 +46,10 @@ func (s *wikiMapVersionStore) GetWikiMapActiveState(ctx context.Context, tenantIIndexNames: []string{fmt.Sprintf("ragflow_%s", tenantID)},KbIDs: []string{datasetID},Limit: 1,- SelectFields: []string{"id", "compile_kwd", "content_with_weight"},+ SelectFields: []string{"id", "compile_kwd", "type_kwd", "content_with_weight"},Filter: map[string]interface{}{"id": []string{key},- "compile_kwd": wikiMapActiveCompileKWD,+ "type_kwd": wikiMapActiveCompileKWD,"available_int": 0,},})@@ -77,7 +77,8 @@ func (s *wikiMapVersionStore) PutWikiMapActiveState(ctx context.Context, state k"id": state.Key,"doc_id": "wiki_map_active:" + state.DocumentID,"kb_id": state.DatasetID,- "compile_kwd": wikiMapActiveCompileKWD,+ "compile_kwd": "wiki",+ "type_kwd": wikiMapActiveCompileKWD,"scope_kwd": "doc","source_doc_ids": []string{state.DocumentID},"content_with_weight": string(state.Payload),@@ -147,11 +148,11 @@ func (s *wikiMapVersionStore) GetWikiMapVersions(ctx context.Context, tenantID,KbIDs: []string{datasetID},Limit: end - start,SelectFields: []string{- "id", "compile_kwd", "content_with_weight",+ "id", "compile_kwd", "type_kwd", "content_with_weight",},Filter: map[string]interface{}{"id": keys[start:end],- "compile_kwd": wikiMapExtractCompileKWD,+ "type_kwd": wikiMapExtractCompileKWD,"available_int": 0,},})@@ -162,7 +163,7 @@ func (s *wikiMapVersionStore) GetWikiMapVersions(ctx context.Context, tenantID,continue}for _, row := range result.Chunks {- if mapStoreString(row["compile_kwd"]) != wikiMapExtractCompileKWD {+ if types.CompilationRowType(row) != wikiMapExtractCompileKWD {continue}id := mapStoreString(row["id"])@@ -229,7 +230,8 @@ func wikiMapVersionRow(version kccommon.WikiMapVersion) map[string]interface{} {// document deletion cannot remove a reusable chunk/hash version."doc_id": wikiMapCacheDocID(version.DocumentID),"kb_id": version.DatasetID,- "compile_kwd": wikiMapExtractCompileKWD,+ "compile_kwd": "wiki",+ "type_kwd": wikiMapExtractCompileKWD,"scope_kwd": "doc","source_chunk_ids": []string{version.ChunkID},"source_doc_ids": []string{version.DocumentID},diff --git a/internal/ingestion/knowledge_compile/wiki_map_store_test.go b/internal/ingestion/knowledge_compile/wiki_map_store_test.goindex 1d777c231f..5165608a41 100644--- a/internal/ingestion/knowledge_compile/wiki_map_store_test.go+++ b/internal/ingestion/knowledge_compile/wiki_map_store_test.go@@ -81,21 +81,46 @@ func (e *wikiMapStoreEngine) Search(_ context.Context, req *types.SearchRequest)}func rowMatchesFilter(row map[string]interface{}, filter map[string]interface{}) bool {- if expected, ok := filter["available_int"].(int); ok {- if value, ok := row["available_int"].(int); !ok || value != expected {- return false- }- }- if mustNot, ok := filter["must_not"].(map[string]interface{}); ok {- if exists, ok := mustNot["exists"].(string); ok {- if _, present := row[exists]; present {+ return matchesStoredCompilationRow(row, types.CompilationFilter(filter))+}++func matchesStoredCompilationRow(row, filter map[string]interface{}) bool {+ for field, value := range filter {+ switch field {+ case "and":+ for _, child := range types.FilterClauses(value) {+ if !matchesStoredCompilationRow(row, child) {+ return false+ }+ }+ case "or":+ matched := false+ for _, child := range types.FilterClauses(value) {+ matched = matched || matchesStoredCompilationRow(row, child)+ }+ if !matched {+ return false+ }+ case "must_not":+ if matchesStoredCompilationRow(row, value.(map[string]interface{})) {+ return false+ }+ case "exists":+ if mapStoreString(row[value.(string)]) == "" {+ return false+ }+ default:+ if expected, ok := value.([]string); ok {+ matched := false+ for _, v := range expected {+ matched = matched || mapStoreString(row[field]) == v+ }+ if !matched {+ return false+ }+ } else if !reflect.DeepEqual(row[field], value) {return false}- }- }- for _, field := range []string{"compile_kwd", "type_kwd"} {- if expected, ok := filter[field].(string); ok && mapStoreString(row[field]) != expected {- return false}}return true@@ -143,7 +168,8 @@ func TestWikiMapVersionStoreUsesNonSearchableDocStoreRows(t *testing.T) {"id": "version-a","doc_id": "wiki_map_cache:doc-1","kb_id": "kb-1",- "compile_kwd": wikiMapExtractCompileKWD,+ "compile_kwd": "wiki",+ "type_kwd": wikiMapExtractCompileKWD,"available_int": 0,"chunk_hash_kwd": "hash-a",} {diff --git a/internal/ingestion/knowledge_compile/writer.go b/internal/ingestion/knowledge_compile/writer.goindex 85ec2c764d..24bfd3079e 100644--- a/internal/ingestion/knowledge_compile/writer.go+++ b/internal/ingestion/knowledge_compile/writer.go@@ -95,6 +95,7 @@ type Writer interface {// folded descriptions and the union of source docs/chunks. It is written as a// scope_kwd="dataset" row. Relations carry FromEntity/ToEntity instead of Name.type StructureBucket struct {+ CompileType stringName stringType stringDescription string // folded entity descriptions@@ -104,22 +105,13 @@ type StructureBucket struct {VecCount int // number of vectors folded into Vector (for true mean)FromEntity string // relation onlyToEntity string // relation only- // CompileKwd is the raw compile keyword (the inferred compile type / autotype,- // e.g. "hypergraph", "timeline", "mindmap", "list") that produced this bucket.- // It is stamped on the stored row's compile_kwd so distinct structure kinds- // never collide in the same dataset namespace (plan §1.1). It matches the doc- // row's own compile_kwd — Python _do_build carries the doc row's compile_kwd- // verbatim onto the dataset row, it does NOT rewrite it to the template kind.+ // CompileKwd is the canonical knowledge compilation kind for this bucket.CompileKwd string// TemplateID is the compilation template id (compilation_template_ids[0]) that// produced this bucket. Stamped on the dataset row so read/delete paths can// filter per template (mirror Python get_dataset_structure).TemplateID string- // TemplateKind is the authoritative template kind (compilation_template_kind_kwd,- // e.g. "knowledge_graph", "mind_map", "timeline"). This — NOT compile_kwd — is- // the field read/delete paths match on for the dataset-structure kind- // (mirror Python get_dataset_structure._discover_scope_templates, which- // matches _resolve_dataset_structure_kind against compilation_template_kind_kwd).+ // TemplateKind identifies the producing template and determines CompileKwd.TemplateKind string// RelationType is the relation type (Python default "related"), persisted as// relation_type_kwd and part of the relation bucket/ID identity.@@ -154,7 +146,7 @@ func (w engineWriter) DeleteMergedWikiPages(ctx context.Context, tenant, kb stri}baseName := fmt.Sprintf("ragflow_%s", tenant)_, err := eng.DeleteChunks(ctx, map[string]interface{}{- "id": ids, "kb_id": kb, "available_int": 1, "scope_kwd": "dataset", "compile_kwd": compileKwdWikiPage,+ "id": ids, "kb_id": kb, "available_int": 1, "scope_kwd": "dataset", "type_kwd": compileKwdWikiPage,}, baseName, kb)return err}@@ -341,10 +333,10 @@ func (w engineWriter) loadWikiPageContent(ctx context.Context, tenant, kb stringOffset: offset,Limit: pageBatchSize,SelectFields: []string{- "slug_kwd", "page_type_kwd", "md_with_weight", "content_with_weight",+ "slug_kwd", "page_type_kwd", "compile_kwd", "entity_type_kwd", "type_kwd", "md_with_weight", "content_with_weight",},Filter: map[string]interface{}{- "compile_kwd": compileKwdWikiPage,+ "type_kwd": compileKwdWikiPage,"available_int": 1,"kb_id": kb,"slug_kwd": slugs,@@ -359,7 +351,7 @@ func (w engineWriter) loadWikiPageContent(ctx context.Context, tenant, kb string}for _, row := range res.Chunks {target, ok := wikiPageCommitTargetForRow(- pageEngineString(row["page_type_kwd"]),+ types.WikiPageCategory(row),pageEngineString(row["slug_kwd"]),)if !ok {@@ -540,7 +532,7 @@ func (w engineWriter) WriteMergedStructure(ctx context.Context, tenant, kb strin"doc_id": kb,"kb_id": kb,"available_int": 1,- "compile_kwd": ckwd,+ "compile_kwd": types.CanonicalCompilationKind(kccommon.FirstNonEmpty(b.TemplateKind, ckwd)),"scope_kwd": "dataset","content_with_weight": payload,"source_doc_ids": b.SourceDocIDs,@@ -548,12 +540,12 @@ func (w engineWriter) WriteMergedStructure(ctx context.Context, tenant, kb strin"create_time": now.Format("2006-01-02 15:04:05"),"create_timestamp_flt": float64(now.Unix()),}- // Stamp the authoritative template identity so read/delete paths can match- // the dataset-structure kind by compilation_template_kind_kwd (mirror Python- // get_dataset_structure._discover_scope_templates), independent of the- // autotype-valued compile_kwd above.- if b.TemplateKind != "" {- row["compilation_template_kind_kwd"] = b.TemplateKind+ if b.CompileType != "" {+ extra, err := json.Marshal(map[string]string{"compile_type": b.CompileType})+ if err != nil {+ return err+ }+ row["extra"] = string(extra)}if b.TemplateID != "" {row["compilation_template_ids"] = []string{b.TemplateID}@@ -599,7 +591,7 @@ func (w engineWriter) WriteMergedStructure(ctx context.Context, tenant, kb strin"doc_id": kb,"kb_id": kb,"available_int": 0,- "compile_kwd": ckwd,+ "compile_kwd": types.CanonicalCompilationKind(kccommon.FirstNonEmpty(b.TemplateKind, ckwd)),"scope_kwd": "dataset","type_kwd": "kg_build_meta","create_time": now.Format("2006-01-02 15:04:05"),@@ -707,9 +699,9 @@ func mergeExistingStructureBuckets(ctx context.Context, eng engine.DocEngine, bafunc structureCompileKind(compileKwd string) string {if compileKwd = strings.TrimSpace(compileKwd); compileKwd != "" {- return compileKwd+ return types.CanonicalCompilationKind(compileKwd)}- return compileKwdStructure+ return "graph"}func structureRelationType(value string) string {@@ -738,8 +730,8 @@ func structureRowIdentity(row map[string]interface{}) string {if ids := firstStringSlice(row["compilation_template_ids"]); len(ids) > 0 {templateID = ids[0]}- template := structureTemplateIdentity(templateID, structureString(row["compilation_template_kind_kwd"]))- compileKwd := structureCompileKind(structureString(row["compile_kwd"]))+ template := structureTemplateIdentity(templateID, types.CompilationKind(row))+ compileKwd := structureCompileKind(types.CompilationKind(row))payload := structureRowPayload(row)if strings.EqualFold(structureRowType(row), "relation") {from := structureString(row["from_entity_kwd"])@@ -966,7 +958,7 @@ func mergedChunkMap(tenant, kb, runID, inputHash string, now time.Time, p kccomm// carry scope_kwd="dataset" alongside available_int=1 so the consumer and// clean paths can filter by scope instead of (only) available_int."scope_kwd": "dataset",- "compile_kwd": compileKwdForVariant(p.Variant),+ "compile_kwd": types.CanonicalCompilationKind(kccommon.FirstNonEmpty(p.Kind, string(p.Variant))),"content_with_weight": p.Content,"source_doc_ids": srcDocIDs,"source_chunk_ids": srcChunkIDs,@@ -975,9 +967,9 @@ func mergedChunkMap(tenant, kb, runID, inputHash string, now time.Time, p kccomm"create_time": now.Format("2006-01-02 15:04:05"),"create_timestamp_flt": float64(now.Unix()),}- // The product kind (page vs section) is carried by compile_kwd- // (wiki_page / wiki_section), which is what Python stores and what- // productFromChunkMap derives Meta["kind"] from — there is no kind column.+ if p.Variant == kccommon.VariantWiki {+ m["type_kwd"] = "wiki_" + kccommon.FirstNonEmpty(metaString(p.Meta, "kind"), "page")+ }// wiki_incremental port: preserve the original creation timestamp across a// page merge. If the incoming merged product already carries// created_at_unix (restored by the Reader from create_timestamp_flt), reuse@@ -988,11 +980,8 @@ func mergedChunkMap(tenant, kb, runID, inputHash string, now time.Time, p kccomm// Rebuild the human-readable form from the preserved unix time.m["create_time"] = time.Unix(int64(v), 0).Format("2006-01-02 15:04:05")}- // Carry the wiki page metadata onto the merged row so the dataset-level- // products keep the fields the artifact API (ListArtifacts/ListWikiTopics)- // and page renderers read. Without this the merged rows lose page_type_kwd /- // topic_kwd / title_kwd and the compilation page would show no wiki pages- // even though per-document products carry them.+ // Carry wiki metadata onto the merged row so the dataset-level+ // products retain their category, topic, title, and rendered page body.//// slug_kwd follows the Python writer contract (api/db/db_models.py): it is// stored as the full "<page_type>/<slug>" form so GetWikiPage's filter@@ -1011,7 +1000,7 @@ func mergedChunkMap(tenant, kb, runID, inputHash string, now time.Time, p kccommm["title_kwd"] = v}if pageType != "" {- m["page_type_kwd"] = pageType+ m["entity_type_kwd"] = pageType}if v := kccommon.NormalizeWikiTopicPath(metaString(p.Meta, "topic")); v != "" {m["topic_kwd"] = v@@ -1291,14 +1280,7 @@ func metaFloat(m map[string]any, key string) (float64, bool) {return 0, false}-// KwdToVariant is the inverse of compileKwdForVariant: it maps a stored-// compile_kwd back to its compiler Variant. Both wiki_page and wiki_section-// map to VariantWiki (same product family); the page/section distinction is-// carried by compile_kwd itself, not the variant. Returns an error for an-// unknown kwd so callers can reject dirty/foreign rows. Structure products-// stamp the inferred compile kind verbatim (list/set/hypergraph), which are NOT-// in the KindToVariant whitelist, so they are mapped to VariantStructure-// explicitly before the whitelist lookup; unknown kinds hard-fail (O2a).+// KwdToVariant maps a stored compilation kind to its execution variant.func KwdToVariant(kwd string) (kccommon.Variant, error) {switch kwd {case compileKwdWikiPage, compileKwdWikiSection, compileKwdWikiEntity, compileKwdWikiRelation:@@ -1321,22 +1303,15 @@ func KwdToVariant(kwd string) (kccommon.Variant, error) {// --- Wiki page graph materialization (wiki_entity / wiki_relation) ----// compile_kwd values for the dataset-level products this package writes. The-// wiki variant compiles into "wiki_page" rows (per the Python writer contract-// that GetWikiAlteration / ListArtifacts / GetWikiGraph all filter on-// compile_kwd = "wiki_page"), while the page graph is materialized as the-// dedicated wiki_entity / wiki_relation buckets.+// Wiki row-role values used by readers and graph projection.const (compileKwdWikiPage = "wiki_page"compileKwdWikiSection = "wiki_section"compileKwdWikiEntity = "wiki_entity"compileKwdWikiRelation = "wiki_relation"compileKwdWikiPageGraph = "wiki_page_graph" // legacy Python blob, swept on drop- // compileKwdStructure tags structure dataset-level merged rows- // (scope_kwd="dataset"); compileKwdNav tags the dataset-navigation rows- // written by NavService. Both are targets of per-variant clean (B4).+ // Structure is the default execution family when no template kind is supplied.compileKwdStructure = "structure"- compileKwdNav = "dataset_nav")// wikiGraphBatchSize bounds how many graph rows each parallel InsertChunks call@@ -1347,14 +1322,12 @@ const wikiGraphBatchSize = 200// a heavily-shared page does not accumulate an unbounded id list.const wikiGraphSourceDocCap = 64-// compileKwdForVariant maps a compiler Variant to the compile_kwd stamped on the-// merged chunk document. The wiki variant is special-cased because all read-// paths filter on compile_kwd = "wiki_page", not the raw variant string "wiki".+// compileKwdForVariant returns the canonical storage kind of an execution variant.func compileKwdForVariant(v kccommon.Variant) string {if v == kccommon.VariantWiki {- return compileKwdWikiPage+ return "wiki"}- return string(v)+ return types.CanonicalCompilationKind(string(v))}// wikiGraphXXHash derives a stable 16-char hex id (matches Python@@ -1475,7 +1448,7 @@ func (w engineWriter) loadActiveDocumentWikiPages(ctx context.Context, tenant, kbySlug := make(map[string]wikiPageProjection)for offset := 0; ; offset += batchSize {filter := map[string]interface{}{- "compile_kwd": compileKwdWikiPage, "available_int": 0,+ "type_kwd": compileKwdWikiPage, "available_int": 0,"scope_kwd": "doc", "kb_id": kb,}if statusAvailable {@@ -1486,7 +1459,7 @@ func (w engineWriter) loadActiveDocumentWikiPages(ctx context.Context, tenant, kKbIDs: []string{kb},Filter: filter,SelectFields: []string{- "slug_kwd", "page_type_kwd", "title_kwd", "entity_names_kwd",+ "slug_kwd", "page_type_kwd", "compile_kwd", "entity_type_kwd", "type_kwd", "title_kwd", "entity_names_kwd","summary_with_weight", "outlinks_kwd", "source_doc_ids", "source_chunk_ids",},Limit: batchSize, Offset: offset,@@ -1506,7 +1479,7 @@ func (w engineWriter) loadActiveDocumentWikiPages(ctx context.Context, tenant, kpage := bySlug[slug]if page.Slug == "" {page.Slug = slug- page.PageType = metaString(row, "page_type_kwd")+ page.PageType = types.WikiPageCategory(row)page.Title = metaString(row, "title_kwd")page.Summary = metaString(row, "summary_with_weight")}@@ -1546,13 +1519,8 @@ func (w engineWriter) DeleteMerged(ctx context.Context, tenant, kb string) errorreturn w.DeleteMergedForVariant(ctx, tenant, kb, []kccommon.Variant{kccommon.VariantWiki})}-// DeleteMergedForVariant deletes the dataset-level merged rows for a KB across-// the given compile variants (B4). When variants is empty it clears the full set-// the consumer manages (B1b: a full rebuild always clears everything, so an-// empty set still means "clear all"). Row scope: structure dataset rows are-// tagged scope_kwd="dataset"; wiki merged rows carry available_int=1; nav rows-// carry compile_kwd="dataset_nav". The filter union is OR-ed across variants so-// one call clears every relevant row in a single engine delete.+// DeleteMergedForVariant clears the selected dataset outputs, preserving+// document-level compilation inputs. An empty variant set clears all outputs.func (w engineWriter) DeleteMergedForVariant(ctx context.Context, tenant, kb string, variants []kccommon.Variant) error {eng := w.engif eng == nil {@@ -1563,30 +1531,22 @@ func (w engineWriter) DeleteMergedForVariant(ctx context.Context, tenant, kb str}baseName := fmt.Sprintf("ragflow_%s", tenant)if len(variants) == 0 {- // Full-set clean (B1b): everything the consumer manages. Structure- // dataset rows are tagged scope_kwd="dataset" (not a fixed compile_kwd),- // so the full clean deletes by kb_id + compile_kwd IN (the fixed-kwd- // buckets) OR scope_kwd="dataset". The scope_kwd="dataset" sweep requires- // the dataset-structure filter keys, so it is guarded like the per-variant- // structure branch (review fix: this is the highest-impact destructive- // path and must fail loudly on unsupported engines).+ // Navigation roles and searchable wiki outputs are separate from the+ // dataset-scoped structure sweep.if !datasetStructureSupported() {return errDatasetStructureUnsupported()}_, err := eng.DeleteChunks(ctx, map[string]interface{}{"kb_id": kb,- "compile_kwd": []string{- compileKwdNav,- compileKwdWikiPage,- compileKwdWikiSection,- compileKwdStructure,+ "or": []map[string]interface{}{+ {"type_kwd": []string{"nav_doc", "nav_cluster"}},+ {"type_kwd": []string{compileKwdWikiPage, compileKwdWikiSection}, "available_int": 1},},}, baseName, kb)if err != nil {return fmt.Errorf("delete merged (all variants): %w", err)}- // structure dataset rows carry scope_kwd="dataset" + compile_kwd="structure";- // sweep them too (idempotent with the compile_kwd filter above).+ // Clear dataset-scoped outputs after preserving document inputs._, err = eng.DeleteChunks(ctx, map[string]interface{}{"kb_id": kb,"scope_kwd": "dataset",@@ -1596,12 +1556,7 @@ func (w engineWriter) DeleteMergedForVariant(ctx context.Context, tenant, kb str}return nil}- // Issue ONE delete per distinct variant bucket rather than one AND-ed filter:- // different variants target different columns (wiki: available_int=1 +- // compile_kwd; structure: scope_kwd="dataset"; nav: compile_kwd="dataset_nav"- // with available_int=0), and AND-ing them would exclude the others' rows.- // RebuildDataset (B1b) passes the full managed set, so this per-bucket sweep- // is correct for both full and per-variant rebuilds.+ // Each output family has its own role, scope, and visibility constraints.for _, v := range variants {switch v {case kccommon.VariantWiki:@@ -1610,7 +1565,7 @@ func (w engineWriter) DeleteMergedForVariant(ctx context.Context, tenant, kb strif _, err := eng.DeleteChunks(ctx, map[string]interface{}{"kb_id": kb,"available_int": 1,- "compile_kwd": []string{compileKwdWikiPage, compileKwdWikiSection},+ "type_kwd": []string{compileKwdWikiPage, compileKwdWikiSection},}, baseName, kb); err != nil {return fmt.Errorf("delete merged (wiki): %w", err)}@@ -1633,11 +1588,11 @@ func (w engineWriter) DeleteMergedForVariant(ctx context.Context, tenant, kb strreturn fmt.Errorf("delete merged (structure/mindmap): %w", err)}case kccommon.VariantTree:- // tree/nav rows are compile_kwd="dataset_nav" with available_int=0;+ // Navigation rows have available_int=0;// do NOT add available_int=1 (that would exclude them).if _, err := eng.DeleteChunks(ctx, map[string]interface{}{- "kb_id": kb,- "compile_kwd": []string{compileKwdNav},+ "kb_id": kb,+ "type_kwd": []string{"nav_doc", "nav_cluster"},}, baseName, kb); err != nil {return fmt.Errorf("delete merged (nav): %w", err)}@@ -1694,12 +1649,12 @@ func (w engineWriter) loadMergedWikiPages(ctx context.Context, tenant, kb stringIndexNames: []string{baseName},KbIDs: []string{kb},Filter: map[string]interface{}{- "compile_kwd": compileKwdWikiPage,+ "type_kwd": compileKwdWikiPage,"available_int": 1,"kb_id": kb,},SelectFields: []string{- "slug_kwd", "page_type_kwd", "title_kwd",+ "slug_kwd", "page_type_kwd", "entity_type_kwd", "type_kwd", "title_kwd","entity_names_kwd", "summary_with_weight", "outlinks_kwd","source_doc_ids", "source_chunk_ids",// compile_kwd is selected purely so the query-result telemetry@@ -1730,7 +1685,7 @@ func (w engineWriter) loadMergedWikiPages(ctx context.Context, tenant, kb stringfor _, c := range res.Chunks {p := wikiPageProjection{Slug: metaString(c, "slug_kwd"),- PageType: metaString(c, "page_type_kwd"),+ PageType: types.WikiPageCategory(c),Title: metaString(c, "title_kwd"),Aliases: metaStringSlice(c, "entity_names_kwd"),Summary: metaString(c, "summary_with_weight"),@@ -1835,9 +1790,9 @@ func (w engineWriter) projectWikiGraphRows(_ context.Context, tenant, kb string,"doc_id": kb,"kb_id": kb,"available_int": 1,- "compile_kwd": compileKwdWikiEntity,- "type_kwd": "wiki_" + p.PageType,- "entity_type_kwd": "wiki_" + p.PageType,+ "compile_kwd": "wiki",+ "type_kwd": compileKwdWikiEntity,+ "entity_type_kwd": p.PageType,"slug_kwd": p.Slug,"title_kwd": p.Title,"weight_int": weight,@@ -1891,7 +1846,7 @@ func (w engineWriter) projectWikiGraphRows(_ context.Context, tenant, kb string,"doc_id": kb,"kb_id": kb,"available_int": 1,- "compile_kwd": compileKwdWikiRelation,+ "compile_kwd": "wiki","type_kwd": compileKwdWikiRelation,"from_kwd": p.Slug,"to_kwd": tgt,@@ -1902,7 +1857,7 @@ func (w engineWriter) projectWikiGraphRows(_ context.Context, tenant, kb string,}var relCount, entCount intfor _, r := range rows {- switch r["compile_kwd"] {+ switch r["type_kwd"] {case compileKwdWikiRelation:relCount++case compileKwdWikiEntity:diff --git a/internal/ingestion/knowledge_compile/writer_test.go b/internal/ingestion/knowledge_compile/writer_test.goindex caab047ff4..cf85feac25 100644--- a/internal/ingestion/knowledge_compile/writer_test.go+++ b/internal/ingestion/knowledge_compile/writer_test.go@@ -88,7 +88,7 @@ func TestMergedChunkMapKeepsWikiFields(t *testing.T) {cases := map[string]string{"slug_kwd": "entity/alpha","title_kwd": "Alpha",- "page_type_kwd": "entity",+ "entity_type_kwd": "entity","topic_kwd": "Knowledge/Core/Alpha","summary_with_weight": "A page about Alpha",// The wiki page body goes to md_with_weight, the column Python writes@@ -284,9 +284,9 @@ func TestDeleteMergedScopesToMergedWikiRows(t *testing.T) {if cond["available_int"] != 1 {t.Errorf("DeleteMerged must scope to available_int=1, got %v", cond["available_int"])}- variants, ok := cond["compile_kwd"].([]string)+ variants, ok := cond["type_kwd"].([]string)if !ok {- t.Fatalf("DeleteMerged must pass a compile_kwd string slice, got %T", cond["compile_kwd"])+ t.Fatalf("DeleteMerged must pass a compile_kwd string slice, got %T", cond["type_kwd"])}if len(variants) != 2 || variants[0] != compileKwdWikiPage || variants[1] != compileKwdWikiSection {t.Errorf("DeleteMerged compile_kwd = %v, want [%q %q]", variants, compileKwdWikiPage, compileKwdWikiSection)@@ -302,26 +302,22 @@ func TestDeleteMergedForVariant_EmptySetClearsAll(t *testing.T) {if err := w.DeleteMergedForVariant(context.Background(), "t1", "kb1", nil); err != nil {t.Fatalf("DeleteMergedForVariant: %v", err)}- // The full clean issues two deletes: the compile_kwd bucket first, then the- // scope_kwd="dataset" sweep. Assert on the first (compile_kwd) delete.- if len(eng.deleteConds) < 1 {- t.Fatal("engine.DeleteChunks was not called")- }- kwds, ok := eng.deleteConds[0]["compile_kwd"].([]string)- if !ok {- t.Fatalf("compile_kwd filter not a string slice, got %T", eng.deleteConds[0]["compile_kwd"])- }- want := []string{compileKwdNav, compileKwdWikiPage, compileKwdWikiSection, compileKwdStructure}- if len(kwds) != len(want) {- t.Fatalf("full clean compile_kwd = %v, want %v", kwds, want)- }- got := map[string]bool{}- for _, k := range kwds {- got[k] = true- }- for _, k := range want {- if !got[k] {- t.Errorf("full clean missing compile_kwd %q", k)+ if len(eng.deleteConds) != 2 {+ t.Fatalf("full clean issued %d deletes, want 2", len(eng.deleteConds))+ }+ for _, test := range []struct {+ row map[string]interface{}+ want bool+ }{+ {map[string]interface{}{"kb_id": "kb1", "compile_kwd": "wiki", "type_kwd": "wiki_page", "available_int": 1}, true},+ {map[string]interface{}{"kb_id": "kb1", "compile_kwd": "wiki_page", "available_int": 1}, true},+ {map[string]interface{}{"kb_id": "kb1", "compile_kwd": "page_index", "type_kwd": "nav_doc", "available_int": 0}, true},+ {map[string]interface{}{"kb_id": "kb1", "compile_kwd": "graph", "type_kwd": "entity", "scope_kwd": "doc"}, false},+ {map[string]interface{}{"kb_id": "kb1", "compile_kwd": "wiki", "type_kwd": "wiki_page", "available_int": 0}, false},+ {map[string]interface{}{"kb_id": "kb2", "compile_kwd": "tree", "type_kwd": "nav_doc"}, false},+ } {+ if got := rowMatchesFilter(test.row, eng.deleteConds[0]); got != test.want {+ t.Errorf("full clean matched %v = %v, want %v", test.row, got, test.want)}}// The second delete sweeps structure scope_kwd="dataset".@@ -352,9 +348,9 @@ func TestDeleteMergedForVariant_WikiScopesToAvailableOne(t *testing.T) {if cond["available_int"] != 1 {t.Errorf("wiki clean must scope to available_int=1, got %v", cond["available_int"])}- kwds, ok := cond["compile_kwd"].([]string)+ kwds, ok := cond["type_kwd"].([]string)if !ok || len(kwds) != 2 {- t.Fatalf("wiki clean compile_kwd = %v, want [wiki_page wiki_section]", cond["compile_kwd"])+ t.Fatalf("wiki clean compile_kwd = %v, want [wiki_page wiki_section]", cond["type_kwd"])}}@@ -392,11 +388,8 @@ func TestDeleteMergedForVariant_WikiPlusNavCoversNavRows(t *testing.T) {}var wikiCond, navCond map[string]interface{}for _, cond := range eng.deleteConds {- kwds, _ := cond["compile_kwd"].([]string)- for _, k := range kwds {- if k == compileKwdNav {- navCond = cond- }+ if roles, ok := cond["type_kwd"].([]string); ok && len(roles) == 2 && roles[0] == "nav_doc" && roles[1] == "nav_cluster" {+ navCond = cond}if cond["available_int"] == 1 {wikiCond = cond@@ -406,7 +399,7 @@ func TestDeleteMergedForVariant_WikiPlusNavCoversNavRows(t *testing.T) {t.Error("expected a wiki delete scoped to available_int=1")}if navCond == nil {- t.Error("expected a nav delete (compile_kwd=dataset_nav)")+ t.Error("expected a nav delete scoped to nav_doc/nav_cluster")} else if _, hasAvail := navCond["available_int"]; hasAvail {t.Errorf("nav delete must not carry available_int (nav rows are 0), got %v", navCond["available_int"])}diff --git a/internal/ingestion/task/knowledge_compiler_wiring.go b/internal/ingestion/task/knowledge_compiler_wiring.goindex ddc72e59b2..aab3a92ba4 100644--- a/internal/ingestion/task/knowledge_compiler_wiring.go+++ b/internal/ingestion/task/knowledge_compiler_wiring.go@@ -258,7 +258,7 @@ func wikiCompiledRows(output map[string]any) []map[string]any {if !ok {continue}- compileKWD := strings.TrimSpace(anyString(row["compile_kwd"]))+ compileKWD := enginetypes.CompilationRowType(row)if compileKWD != "wiki_page" && compileKWD != "wiki_section" {continue}@@ -291,7 +291,7 @@ func replaceDirtyWikiProducts(ctx context.Context, request knowledge_compile.WikSelectFields: []string{"id", "compile_kwd", "slug_kwd", "parent_id"},Filter: map[string]any{"doc_id": []string{request.DocumentID},- "compile_kwd": []string{"wiki_page", "wiki_section"},+ "type_kwd": []string{"wiki_page", "wiki_section"},"available_int": 0,},})@@ -316,7 +316,7 @@ func replaceDirtyWikiProducts(ctx context.Context, request knowledge_compile.WikaffectedPageIDs := make(map[string]struct{})if existing != nil && !fullReplace {for _, row := range existing.Chunks {- if anyString(row["compile_kwd"]) != "wiki_page" {+ if enginetypes.CompilationRowType(row) != "wiki_page" {continue}if _, ok := affected[anyString(row["slug_kwd"])]; ok {@@ -326,7 +326,7 @@ func replaceDirtyWikiProducts(ctx context.Context, request knowledge_compile.Wik}if existing != nil && fullReplace {for _, row := range existing.Chunks {- if anyString(row["compile_kwd"]) == "wiki_page" {+ if enginetypes.CompilationRowType(row) == "wiki_page" {removedSlugs = append(removedSlugs, anyString(row["slug_kwd"]))}}@@ -369,7 +369,7 @@ func replaceDirtyWikiProducts(ctx context.Context, request knowledge_compile.Wikfunc clearWikiActiveStates(ctx context.Context, docEngine engine.DocEngine, request knowledge_compile.WikiDirtyRequest) error {_, err := docEngine.DeleteChunks(ctx, map[string]any{"kb_id": request.DatasetID,- "compile_kwd": "wiki_map_active",+ "type_kwd": "wiki_map_active","available_int": 0,"source_doc_ids": []string{request.DocumentID},}, fmt.Sprintf("ragflow_%s", request.TenantID), request.DatasetID)@@ -845,13 +845,13 @@ func (s *kcWikiPageStore) FindSimilarPages(ctx context.Context, tenantID, dataseIndexNames: []string{fmt.Sprintf("ragflow_%s", tenantID)},KbIDs: []string{datasetID},Limit: k,- SelectFields: []string{"id", "slug_kwd", "title_kwd", "page_type_kwd", "topic_kwd", "plan_group_kwd", "summary_with_weight", "content_with_weight", "md_with_weight", "entity_names_kwd", "related_kb_pages_kwd", "outlinks_kwd", "source_chunk_ids", "_score"},+ SelectFields: []string{"id", "slug_kwd", "title_kwd", "page_type_kwd", "compile_kwd", "type_kwd", "entity_type_kwd", "topic_kwd", "plan_group_kwd", "summary_with_weight", "content_with_weight", "md_with_weight", "entity_names_kwd", "related_kb_pages_kwd", "outlinks_kwd", "source_chunk_ids", "_score"},// compile_kwd="wiki_page" is the schema-backed discriminator for wiki// pages (sections carry compile_kwd="wiki_section"); there is no// "kc_kind" column in the chunk schema, so filtering on it would return// empty on Infinity.Filter: map[string]interface{}{- "compile_kwd": "wiki_page",+ "type_kwd": "wiki_page",},MatchExprs: []interface{}{&enginetypes.MatchDenseExpr{VectorColumnName: fmt.Sprintf("q_%d_vec", len(vec)),@@ -881,10 +881,10 @@ func (s *kcWikiPageStore) GetPageBySlug(ctx context.Context, tenantID, datasetIDIndexNames: []string{fmt.Sprintf("ragflow_%s", tenantID)},KbIDs: []string{datasetID},Limit: 1,- SelectFields: []string{"id", "slug_kwd", "title_kwd", "page_type_kwd", "topic_kwd", "plan_group_kwd", "summary_with_weight", "content_with_weight", "md_with_weight", "entity_names_kwd", "related_kb_pages_kwd", "outlinks_kwd", "source_chunk_ids", "_score"},+ SelectFields: []string{"id", "slug_kwd", "title_kwd", "page_type_kwd", "compile_kwd", "type_kwd", "entity_type_kwd", "topic_kwd", "plan_group_kwd", "summary_with_weight", "content_with_weight", "md_with_weight", "entity_names_kwd", "related_kb_pages_kwd", "outlinks_kwd", "source_chunk_ids", "_score"},Filter: map[string]interface{}{- "compile_kwd": "wiki_page",- "slug_kwd": slug,+ "type_kwd": "wiki_page",+ "slug_kwd": slug,},}res, err := s.docEngine.Search(ctx, req)@@ -903,9 +903,9 @@ func (s *kcWikiPageStore) FindPagesBySourceChunks(ctx context.Context, tenantID,IndexNames: []string{fmt.Sprintf("ragflow_%s", tenantID)},KbIDs: []string{datasetID},Limit: k,- SelectFields: []string{"id", "slug_kwd", "title_kwd", "page_type_kwd", "topic_kwd", "summary_with_weight", "content_with_weight", "md_with_weight", "entity_names_kwd", "related_kb_pages_kwd", "outlinks_kwd", "source_chunk_ids", "_score"},+ SelectFields: []string{"id", "slug_kwd", "title_kwd", "page_type_kwd", "compile_kwd", "type_kwd", "entity_type_kwd", "topic_kwd", "summary_with_weight", "content_with_weight", "md_with_weight", "entity_names_kwd", "related_kb_pages_kwd", "outlinks_kwd", "source_chunk_ids", "_score"},Filter: map[string]interface{}{- "compile_kwd": "wiki_page",+ "type_kwd": "wiki_page","source_chunk_ids": chunkIDs,},}@@ -929,7 +929,7 @@ func wikiPageCandidateFromRow(row map[string]interface{}) kc.WikiPageCandidate {ID: strings.TrimSpace(anyString(row["id"])),Slug: strings.TrimSpace(anyString(row["slug_kwd"])),Title: strings.TrimSpace(anyString(row["title_kwd"])),- PageType: strings.TrimSpace(anyString(row["page_type_kwd"])),+ PageType: enginetypes.WikiPageCategory(row),Topic: strings.TrimSpace(anyString(row["topic_kwd"])),PlanGroup: strings.TrimSpace(anyString(row["plan_group_kwd"])),Summary: strings.TrimSpace(anyString(row["summary_with_weight"])),diff --git a/internal/ingestion/task/pipeline_executor.go b/internal/ingestion/task/pipeline_executor.goindex a32797dd0a..236f05fdb1 100644--- a/internal/ingestion/task/pipeline_executor.go+++ b/internal/ingestion/task/pipeline_executor.go@@ -528,7 +528,7 @@ func (s *PipelineExecutor) loadDocumentCompiledState(ctx context.Context) ([]strKbIDs: []string{s.taskCtx.Doc.KbID},Offset: offset,Limit: pageSize,- SelectFields: []string{"id", "compile_kwd", "compilation_template_kind_kwd"},+ SelectFields: []string{"id", "compile_kwd", "compilation_template_kind_kwd", "type_kwd"},Filter: map[string]any{"doc_id": []string{s.taskCtx.Doc.ID}},})if err != nil {@@ -538,7 +538,7 @@ func (s *PipelineExecutor) loadDocumentCompiledState(ctx context.Context) ([]strbreak}for _, row := range result.Chunks {- if strings.TrimSpace(asCompiledKwd(row)) == "" {+ if strings.TrimSpace(asCompiledKwd(row)) == "" || enginetypes.IsNavigationRow(row) {continue}oldProducts = append(oldProducts, row)@@ -609,22 +609,16 @@ func applyDocumentAvailability(chunks []map[string]any, status *string) {}}-// compiledVariants returns the sorted, de-duplicated set of compile types a-// document's compiled products carry. It reads the authoritative-// `compilation_template_kind_kwd` the KnowledgeCompiler component stamps on each-// compiled product (the resolved template's kind) and maps it through-// common.KindToVariant (O2a whitelist: unknown kinds are skipped). Products-// without an authoritative kind fall back to their `compile_kwd`-derived variant.-// This surfaces the compiler's runtime variant inference to PublishCompleted so-// the consumer can route the dataset-level re-compile per compile type.+// compiledVariants returns the sorted, de-duplicated execution variants needed+// to route document completion to the dataset consumer.func compiledVariants(chunks []map[string]any) []string {seen := map[string]struct{}{}for _, ck := range chunks {- if _, ok := ck["compile_kwd"]; !ok {+ if _, ok := ck["compile_kwd"]; !ok || enginetypes.IsNavigationRow(ck) {continue}var v kccommon.Variant- if kind, ok := ck["compilation_template_kind_kwd"].(string); ok && kind != "" {+ if kind := enginetypes.CompilationKind(ck); kind != "" {mapped, err := kccommon.KindToVariant(kind)if err != nil {continue // unknown template kind (O2a): skip, do not mis-route@@ -662,11 +656,11 @@ func compiledVariants(chunks []map[string]any) []string {func compiledTaskTypes(chunks []map[string]any) []string {seen := map[string]struct{}{}for _, ck := range chunks {- if _, ok := ck["compile_kwd"]; !ok {+ if _, ok := ck["compile_kwd"]; !ok || enginetypes.IsNavigationRow(ck) {continue}taskType := ""- if kind, ok := ck["compilation_template_kind_kwd"].(string); ok && strings.TrimSpace(kind) != "" {+ if kind := enginetypes.CompilationKind(ck); kind != "" {if mapped, err := kccommon.KindToTaskType(kind); err == nil {taskType = mapped}diff --git a/internal/rag/agentic-rag/runtime/tool_compiled_expansion.go b/internal/rag/agentic-rag/runtime/tool_compiled_expansion.goindex a38a5f12c5..e81a47d488 100644--- a/internal/rag/agentic-rag/runtime/tool_compiled_expansion.go+++ b/internal/rag/agentic-rag/runtime/tool_compiled_expansion.go@@ -22,6 +22,7 @@ import ("encoding/json""fmt""math"+ enginetypes "ragflow/internal/engine/types""sort""strings"@@ -597,14 +598,14 @@ func asString(v any) string {}// seedFilters builds the entity/relation condition: {"knowledge_graph_kwd": [kind]} plus an-// optional compile_kwd ("tree") or compilation_template_kind_kwd selector.+// optional canonical compilation-kind selector.func seedFilters(kind, compileKwd, templateKwd string) map[string][]string {f := map[string][]string{"knowledge_graph_kwd": {kind}}if compileKwd != "" {f["compile_kwd"] = []string{compileKwd}}if templateKwd != "" {- f["compilation_template_kind_kwd"] = []string{templateKwd}+ f["compile_kwd"] = []string{enginetypes.CanonicalCompilationKind(templateKwd)}}return f}diff --git a/internal/rag/agentic-rag/runtime/tool_compiled_expansion_test.go b/internal/rag/agentic-rag/runtime/tool_compiled_expansion_test.goindex d81044f0bc..691654b663 100644--- a/internal/rag/agentic-rag/runtime/tool_compiled_expansion_test.go+++ b/internal/rag/agentic-rag/runtime/tool_compiled_expansion_test.go@@ -21,6 +21,7 @@ import ("context""errors""log"+ enginetypes "ragflow/internal/engine/types""strings""testing"@@ -248,6 +249,12 @@ func TestCompiledExpanderGroupsDocScopeByRealOwner(t *testing.T) {func rowMatchesFilters(r map[string]any, filters map[string][]string) bool {for k, vals := range filters {rv := asString(r[k])+ if k == "compile_kwd" {+ rv = enginetypes.CompilationKind(r)+ if len(vals) > 0 && strings.HasPrefix(vals[0], "wiki_") {+ rv = enginetypes.CompilationRowType(r)+ }+ }matched := falsefor _, v := range vals {if rv == v {diff --git a/internal/rag/agentic-rag/runtime/tool_navigation.go b/internal/rag/agentic-rag/runtime/tool_navigation.goindex 72dfc98424..3f1f933111 100644--- a/internal/rag/agentic-rag/runtime/tool_navigation.go+++ b/internal/rag/agentic-rag/runtime/tool_navigation.go@@ -117,7 +117,7 @@ func loadStructureGraph(ctx context.Context, indexName, docID string, kinds map[}sr := StructureRow{CompileKwd: fmt.Sprint(row["compile_kwd"]),- TemplateKind: fmt.Sprint(row["compilation_template_kind_kwd"]),+ TemplateKind: types.CompilationKind(row),KnowledgeGraphKwd: kg,Content: fmt.Sprint(row["content_with_weight"]),}@@ -213,14 +213,13 @@ func parseFloats(v any) ([]float64, bool) {}func normalizeKind(row map[string]interface{}) string {- kind, _ := row["compilation_template_kind_kwd"].(string)- if kind == "" {- kind, _ = row["compile_kwd"].(string)- }- kind = strings.ToLower(strings.TrimSpace(strings.ReplaceAll(kind, "-", "_")))- if kind == "pageindex" || kind == "page_index" || kind == "knowledge_graph" {+ kind := types.CompilationKind(row)+ if kind == "page_index" || kind == "graph" {return "timeline"}+ if kind == "mind_map" {+ return "mindmap"+ }return kind}diff --git a/internal/service/dataset_artifact_service.go b/internal/service/dataset_artifact_service.goindex dbc62a0e46..b013c60ce4 100644--- a/internal/service/dataset_artifact_service.go+++ b/internal/service/dataset_artifact_service.go@@ -139,7 +139,7 @@ func intValue(v interface{}) int {// HasWiki reports whether the dataset has any compiled wiki artifact.func (s *DatasetArtifactService) HasWiki(ctx context.Context, tenantID, datasetID string) (bool, error) {_, total, err := s.searchCompiled(ctx, tenantID, datasetID,- map[string]interface{}{"compile_kwd": []string{CompileKwdWikiPage}}, nil, 0, 1, nil)+ map[string]interface{}{"type_kwd": []string{CompileKwdWikiPage}}, nil, 0, 1, nil)if err != nil {return false, err}@@ -166,7 +166,7 @@ type WikiPageItem struct {}// wikiPageSelectFields are the engine fields ListWikiPages reads.-var wikiPageSelectFields = []string{"slug_kwd", "title_kwd", "page_type_kwd", "topic_kwd", "outlinks_int", "summary_with_weight"}+var wikiPageSelectFields = []string{"slug_kwd", "title_kwd", "page_type_kwd", "compile_kwd", "entity_type_kwd", "type_kwd", "topic_kwd", "outlinks_int", "summary_with_weight"}// wikiPageOrderBy ranks wiki pages by connectivity then title.func wikiPageOrderBy() *types.OrderByExpr {@@ -183,7 +183,7 @@ func (s *DatasetArtifactService) ListWikiPages(ctx context.Context, tenantID, da// list_wiki_pages has no such duplication because its writer emits one row// per page, so mirror that by selecting the merged rows (available_int=1).filter := map[string]interface{}{- "compile_kwd": []string{CompileKwdWikiPage},+ "type_kwd": []string{CompileKwdWikiPage},"available_int": 1, // merged dataset-level rows only (see engine available_int handling)}if pageType != "" {@@ -249,7 +249,7 @@ func wikiPageItemsFromChunks(chunks []map[string]interface{}) []WikiPageItem {}func wikiPageItemFromChunk(c map[string]interface{}) WikiPageItem {- pageType := firstStringValue(c["page_type_kwd"])+ pageType := types.WikiPageCategory(c)// slug_kwd is stored as the full "<page_type>/<slug>" form (Python// contract); expose the bare slug to the frontend so it can be placed in// a single URL path segment (gin :slug does not match '/').@@ -305,13 +305,13 @@ func (s *DatasetArtifactService) GetWikiPage(ctx context.Context, tenantID, data// filter matches the stored value exactly.slugKwd := pageType + "/" + slugfilter := map[string]interface{}{- "compile_kwd": []string{CompileKwdWikiPage},+ "type_kwd": []string{CompileKwdWikiPage},"page_type_kwd": []string{pageType},"slug_kwd": []string{slugKwd},"available_int": 1, // merged dataset-level page, not the per-doc source row}chunks, _, err := s.searchCompiled(ctx, tenantID, datasetID, filter,- []string{"slug_kwd", "title_kwd", "page_type_kwd", "topic_kwd", "md_with_weight",+ []string{"slug_kwd", "title_kwd", "page_type_kwd", "compile_kwd", "entity_type_kwd", "type_kwd", "topic_kwd", "md_with_weight","content_with_weight", "summary_with_weight", "entity_names_kwd", "outlinks_kwd","related_kb_pages_kwd", "source_chunk_ids", "source_doc_ids"},0, 1, nil)@@ -331,7 +331,7 @@ func (s *DatasetArtifactService) GetWikiPage(ctx context.Context, tenantID, data// slug_kwd is the full "<page_type>/<slug>" form; expose the bare slug so a// client can pass it straight back to GetWikiPage/UpdateWikiPage without the// "<page_type>/" prefix being doubled (matches ListWikiPages).- detailPageType := firstStringValue(c["page_type_kwd"])+ detailPageType := types.WikiPageCategory(c)detailSlug := firstStringValue(c["slug_kwd"])if detailPageType != "" {detailSlug = strings.TrimPrefix(detailSlug, detailPageType+"/")@@ -364,7 +364,7 @@ func (s *DatasetArtifactService) UpdateWikiPage(ctx context.Context, tenantID, d// deterministically from the bare slug (see GetWikiPage).slugKwd := pageType + "/" + slugfilter := map[string]interface{}{- "compile_kwd": []string{CompileKwdWikiPage},+ "type_kwd": []string{CompileKwdWikiPage},"page_type_kwd": []string{pageType},"slug_kwd": []string{slugKwd},"available_int": 1, // merged dataset-level page only@@ -417,7 +417,7 @@ type WikiTopicItem struct {// title/slug/summary, mirroring Python list_wiki_topics.func (s *DatasetArtifactService) ListWikiTopics(ctx context.Context, tenantID, datasetID, keywords string) ([]WikiTopicItem, int64, error) {filter := map[string]interface{}{- "compile_kwd": []string{CompileKwdWikiPage},+ "type_kwd": []string{CompileKwdWikiPage},"page_type_kwd": []string{"concept", "entity", "topic"},"available_int": 1, // count only merged pages, not per-doc source rows}@@ -461,7 +461,7 @@ func (s *DatasetArtifactService) ListWikiTopics(ctx context.Context, tenantID, d// contains the keyword.func (s *DatasetArtifactService) wikiTopicsWithKeywordPages(ctx context.Context, tenantID, datasetID, lowerKeyword string) (map[string]bool, error) {filter := map[string]interface{}{- "compile_kwd": []string{CompileKwdWikiPage},+ "type_kwd": []string{CompileKwdWikiPage},"page_type_kwd": []string{"concept", "entity"},}fields := []string{"topic_kwd", "title_kwd", "slug_kwd", "summary_with_weight"}@@ -579,8 +579,8 @@ type WikiGraphRelation struct {// GetWikiGraph returns an incremental wiki graph slice for a dataset. keywords// seeds the overview with BM25 entity matches; topN overrides the entity cap.func (s *DatasetArtifactService) GetWikiGraph(ctx context.Context, tenantID, datasetID, keywords string, topN *int) (*WikiGraph, error) {- entityFilter := map[string]interface{}{"compile_kwd": []string{CompileKwdWikiEntity}, "available_int": 1}- relationFilter := map[string]interface{}{"compile_kwd": []string{CompileKwdWikiRelation}, "available_int": 1}+ entityFilter := map[string]interface{}{"type_kwd": []string{CompileKwdWikiEntity}, "available_int": 1}+ relationFilter := map[string]interface{}{"type_kwd": []string{CompileKwdWikiRelation}, "available_int": 1}_, totalEntities, err := s.searchCompiled(ctx, tenantID, datasetID, entityFilter, []string{"id"}, 0, 1, nil)if err != nil {return nil, err@@ -716,7 +716,7 @@ func (s *DatasetArtifactService) GetWikiAlteration(ctx context.Context, tenantIDinvolved := map[string]struct{}{}for offset := 0; ; offset += 1000 {chunks, total, err := s.searchCompiled(ctx, tenantID, datasetID,- map[string]interface{}{"compile_kwd": []string{CompileKwdWikiPage}},+ map[string]interface{}{"type_kwd": []string{CompileKwdWikiPage}},[]string{"source_doc_ids"}, offset, 1000, nil)if err != nil {return nil, err@@ -990,7 +990,7 @@ func (s *DatasetArtifactService) GetSkillTree(ctx context.Context, tenantID, datfilter["kwd"] = []string{kwd}}chunks, total, err := s.searchCompiled(ctx, tenantID, datasetID, filter,- []string{"kwd", "title_kwd", "page_type_kwd", "outlinks_int", "inlinks_int"}, 0, 10000, nil)+ []string{"kwd", "title_kwd", "page_type_kwd", "compile_kwd", "entity_type_kwd", "type_kwd", "outlinks_int", "inlinks_int"}, 0, 10000, nil)if err != nil {return nil, 0, err}@@ -1000,7 +1000,7 @@ func (s *DatasetArtifactService) GetSkillTree(ctx context.Context, tenantID, datitems = append(items, SkillTreeItem{Kwd: firstStringValue(c["kwd"]),Title: firstStringValue(c["title_kwd"]),- PageType: firstStringValue(c["page_type_kwd"]),+ PageType: types.WikiPageCategory(c),Outlinks: outlinks,Inlinks: inlinks,})diff --git a/internal/service/dataset_structure_graph.go b/internal/service/dataset_structure_graph.goindex 8c0a657590..ffdf08689b 100644--- a/internal/service/dataset_structure_graph.go+++ b/internal/service/dataset_structure_graph.go@@ -664,7 +664,7 @@ func resolveDatasetStructureKind(kind string) string {return "knowledge_graph"case "mindmap", "mind_map":// Align with Python _DATASET_STRUCTURE_KIND_ALIASES: resolve the- // user-facing "mindmap" to the stored compilation_template_kind_kwd value+ // user-facing "mindmap" to the canonical compilation kind// "mind_map" (the template kind, which is what read/delete paths match on).return "mind_map"case "timeline":@@ -880,7 +880,7 @@ func (s *DatasetArtifactService) discoverDocumentGraphTemplateMeta(ctx context.C}for _, row := range rows {templateID := rowTemplateID(row)- kind := firstStringValue(row["compilation_template_kind_kwd"])+ kind := types.CompilationKind(row)if kind == "" {kind = firstStringValue(row["compile_kwd"])}@@ -947,17 +947,8 @@ type DatasetStructureGraphResponse struct {Templates []DocumentStructureGraphTemplate `json:"templates"`}-// GetDatasetStructure returns the dataset-scope structure graph for a resolved-// kind, mirroring Python get_dataset_structure (dataset_api_service.py). Discovery-// scans knowledge_graph_kwd=["entity"] dataset rows (scope_kwd="dataset") and-// matches the resolved kind against the stamped compilation_template_kind_kwd —-// NOT compile_kwd (compile_kwd holds the autotype "hypergraph"/"list"/"mindmap",-// which is never the kind discriminator; Python _discover_scope_templates matches-// _resolve_dataset_structure_kind against compilation_template_kind_kwd the same-// way). It collects distinct template ids, then reads each template's dataset-// entity/relation rows via buildBucket. It does NOT read kg_build_meta (write/-// delete-side only). When Keywords is set, it returns the keyword-matched-// entity subgraph using the name/BM25 and relation expansion path.+// GetDatasetStructure returns dataset-scope entity/relation graphs for the+// requested compilation kind, optionally narrowed by keyword matching.func (s *DatasetArtifactService) GetDatasetStructure(ctx context.Context, in DatasetStructureGraphInput) (*DatasetStructureGraphResponse, error) {resolved := resolveDatasetStructureKind(in.Kind)if resolved == "" {@@ -968,31 +959,27 @@ func (s *DatasetArtifactService) GetDatasetStructure(ctx context.Context, in Dat}resp := &DatasetStructureGraphResponse{Kind: resolved, Templates: []DocumentStructureGraphTemplate{}}- // Discover distinct template ids whose stamped template kind resolves to the- // requested kind, scanning dataset-scope entity rows only. scope_kwd="dataset"- // is required here (unlike the legacy doc_graph fallback) because dataset rows- // are the only ones carrying compilation_template_kind_kwd we can trust for the- // dataset-scope kind match.+ // Discover templates from dataset entity rows of the requested kind.templateIDs := map[string]struct{}{}// "id" must be projected: graphRowSearch keys its result map by the row id,// and Infinity only returns fields listed in SelectFields (it does not// synthesize id), so omitting it silently drops every row (review Major).// The resolved kind is pushed into the filter so the engine applies the// predicate instead of scanning all entity rows and discarding them in Go.- metaFields := []string{"id", "compilation_template_kind_kwd", "compilation_template_ids"}+ metaFields := []string{"id", "compile_kwd", "compilation_template_kind_kwd", "compilation_template_ids"}entityCountFilter := map[string]interface{}{- "knowledge_graph_kwd": []string{"entity"},- "scope_kwd": []string{"dataset"},- "compilation_template_kind_kwd": []string{resolved},+ "knowledge_graph_kwd": []string{"entity"},+ "scope_kwd": []string{"dataset"},+ "compile_kwd": []string{types.CanonicalCompilationKind(resolved)},}_, entityTotal, err := graphRowSearch(ctx, in.TenantID, in.DatasetID, []string{"id"}, entityCountFilter, nil, 0, 1, nil)if err != nil {return nil, err}relationCountFilter := map[string]interface{}{- "knowledge_graph_kwd": []string{"relation"},- "scope_kwd": []string{"dataset"},- "compilation_template_kind_kwd": []string{resolved},+ "knowledge_graph_kwd": []string{"relation"},+ "scope_kwd": []string{"dataset"},+ "compile_kwd": []string{types.CanonicalCompilationKind(resolved)},}_, relationTotal, err := graphRowSearch(ctx, in.TenantID, in.DatasetID, []string{"id"}, relationCountFilter, nil, 0, 1, nil)if err != nil {@@ -1006,7 +993,7 @@ func (s *DatasetArtifactService) GetDatasetStructure(ctx context.Context, in Datreturn nil, err}for _, row := range rows {- tkind := firstStringValue(row["compilation_template_kind_kwd"])+ tkind := types.CompilationKind(row)if tkind == "" || resolveDatasetStructureKind(tkind) != resolved {continue}@@ -1041,10 +1028,10 @@ func (s *DatasetArtifactService) GetDatasetStructure(ctx context.Context, in Dat}}bucketMeta, entities, relations, err := s.keywordSubgraph(ctx, in.TenantID, in.DatasetID, "", keywords, templateMeta, map[string]interface{}{- "scope_kwd": []string{"dataset"},- "compilation_template_ids": []string{tid},- "compilation_template_kind_kwd": []string{resolved},- "knowledge_graph_kwd": []string{"entity"},+ "scope_kwd": []string{"dataset"},+ "compilation_template_ids": []string{tid},+ "compile_kwd": []string{types.CanonicalCompilationKind(resolved)},+ "knowledge_graph_kwd": []string{"entity"},})if err != nil {return nil, err@@ -1068,9 +1055,9 @@ func (s *DatasetArtifactService) GetDatasetStructure(ctx context.Context, in Dat// Read each template's dataset entity/relation rows.for tid := range templateIDs {scope := map[string]interface{}{- "scope_kwd": []string{"dataset"},- "compilation_template_ids": []string{tid},- "compilation_template_kind_kwd": []string{resolved},+ "scope_kwd": []string{"dataset"},+ "compilation_template_ids": []string{tid},+ "compile_kwd": []string{types.CanonicalCompilationKind(resolved)},}entities, relations, err := s.buildBucket(ctx, in.TenantID, in.DatasetID, scope, nil)if err != nil {@@ -1096,7 +1083,7 @@ func (s *DatasetArtifactService) GetDatasetStructure(ctx context.Context, in Dat// resolveGraphBucket mirrors Python _resolve_bucket.func resolveGraphBucket(row map[string]interface{}, templateMeta map[string]map[string]interface{}, documentID string) (map[string]interface{}, map[string]interface{}) {compileKwd := firstStringValue(row["compile_kwd"])- kindVal := firstStringValue(row["compilation_template_kind_kwd"])+ kindVal := types.CompilationKind(row)if kindVal == "" {kindVal = compileKwd}@@ -1292,7 +1279,7 @@ func (s *DatasetArtifactService) keywordSubgraph(ctx context.Context, tenantID,name := strings.ToLower(strings.Join(strings.Fields(graphStr(candidate.node["name"])), " "))template := rowTemplateID(candidate.row)if template == "" {- template = firstStringValue(candidate.row["compilation_template_kind_kwd"])+ template = types.CompilationKind(candidate.row)}if template == "" {template = firstStringValue(candidate.row["compile_kwd"])diff --git a/internal/service/document/knowledge_compile_status.go b/internal/service/document/knowledge_compile_status.goindex 0683d78484..ad04de0df8 100644--- a/internal/service/document/knowledge_compile_status.go+++ b/internal/service/document/knowledge_compile_status.go@@ -69,7 +69,7 @@ func (s *DocumentService) documentKnowledgeCompileTypes(ctx context.Context, tenKbIDs: []string{datasetID},Offset: offset,Limit: 1000,- SelectFields: []string{"compile_kwd", "compilation_template_kind_kwd"},+ SelectFields: []string{"compile_kwd", "compilation_template_kind_kwd", "type_kwd"},Filter: map[string]any{"doc_id": []string{documentID}},IncludeUnavailable: true,})@@ -81,7 +81,10 @@ func (s *DocumentService) documentKnowledgeCompileTypes(ctx context.Context, tenbreak}for _, row := range result.Chunks {- kind := strings.TrimSpace(documentStoreString(row["compilation_template_kind_kwd"]))+ if types.IsNavigationRow(row) {+ continue+ }+ kind := types.CompilationKind(row)variant, variantErr := kccommon.KindToVariant(kind)if variantErr != nil {variant, variantErr = knowledge_compile.KwdToVariant(strings.TrimSpace(documentStoreString(row["compile_kwd"])))diff --git a/internal/service/document/wiki_refresh.go b/internal/service/document/wiki_refresh.goindex a1fa55f6c9..91ec114c8a 100644--- a/internal/service/document/wiki_refresh.go+++ b/internal/service/document/wiki_refresh.go@@ -74,7 +74,7 @@ func (s *DocumentService) loadAvailabilityToggleChunkIDs(ctx context.Context, inKbIDs: []string{datasetID},Offset: offset,Limit: sourceChunkAvailabilityBatchSize,- SelectFields: []string{"id", "compile_kwd"},+ SelectFields: []string{"id", "compile_kwd", "type_kwd"},Filter: map[string]any{"doc_id": []string{documentID}},})cancel()@@ -85,6 +85,9 @@ func (s *DocumentService) loadAvailabilityToggleChunkIDs(ctx context.Context, inbreak}for _, row := range result.Chunks {+ if enginetypes.IsNavigationRow(row) {+ continue+ }if kwd := strings.TrimSpace(documentStoreString(row["compile_kwd"])); kwd != "" {variant, variantErr := knowledge_compile.KwdToVariant(kwd)if variantErr != nil || variant == kccommon.VariantWiki {@@ -111,7 +114,7 @@ func (s *DocumentService) loadSourceChunkIDs(ctx context.Context, indexName, datKbIDs: []string{datasetID},Offset: offset,Limit: sourceChunkAvailabilityBatchSize,- SelectFields: []string{"id", "compile_kwd"},+ SelectFields: []string{"id", "compile_kwd", "type_kwd"},Filter: map[string]any{"doc_id": []string{documentID}},IncludeUnavailable: includeUnavailable,})@@ -123,6 +126,9 @@ func (s *DocumentService) loadSourceChunkIDs(ctx context.Context, indexName, datbreak}for _, row := range result.Chunks {+ if enginetypes.IsNavigationRow(row) {+ continue+ }if strings.TrimSpace(documentStoreString(row["compile_kwd"])) != "" {continue}diff --git a/internal/service/file/file_commit.go b/internal/service/file/file_commit.goindex 2c266575ec..81c805390a 100644--- a/internal/service/file/file_commit.go+++ b/internal/service/file/file_commit.go@@ -709,7 +709,7 @@ func (s *FileCommitService) readCurrentPageContent(ctx context.Context, tenantIDLimit: 1,SelectFields: []string{"md_with_weight", "content_with_weight"},Filter: map[string]interface{}{- "compile_kwd": []string{"wiki_page"},+ "type_kwd": []string{"wiki_page"},"page_type_kwd": []string{pageType},"slug_kwd": []string{slugKwd},"available_int": 1,diff --git a/internal/service/nav/nav.go b/internal/service/nav/nav.goindex c1d77d658f..ad411533bf 100644--- a/internal/service/nav/nav.go+++ b/internal/service/nav/nav.go@@ -64,11 +64,12 @@ type NavHit struct {// UpsertDocInput carries the per-document summary to place into the nav tree.type UpsertDocInput struct {- TenantID string- KbID string- DocID string- Summary string // document summary text (tree product or page_index summary)- Embedd []float32 // optional precomputed embedding+ CompileKind string // tree or page_index+ TenantID string+ KbID string+ DocID string+ Summary string // document summary text (tree product or page_index summary)+ Embedd []float32 // optional precomputed embedding}// NavMergeLLM summarizes or merges nav text via an LLM. It mirrors the Pythondiff --git a/internal/service/nlp/datasetnav.go b/internal/service/nlp/datasetnav.goindex 2cad92b644..06a5fb4e74 100644--- a/internal/service/nlp/datasetnav.go+++ b/internal/service/nlp/datasetnav.go@@ -33,11 +33,10 @@ import ()// Dataset-nav constants. These mirror Python's dataset_nav.py: nav rows live in-// the document index as compile_kwd="dataset_nav" rows with available_int=0+// the document index with type_kwd nav_doc/nav_cluster and available_int=0// (invisible to the default retriever, which filters available_int=1), and the// tree is threaded through parent_kwd ("root" for depth-0 clusters).const (- navCompileKwd = "dataset_nav"navRootParent = "root"navMergeThreshold = 0.80 // sim >= this -> merge doc into cluster@@ -128,9 +127,9 @@ func (s *NavService) navIndexName(tenantID string) string {return fmt.Sprintf("ragflow_%s", tenantID)}-// navFilter builds the common nav filter that pins compile_kwd.+// navFilter scopes navigation reads and writes to navigation row roles.func navFilter(extra map[string]interface{}) map[string]interface{} {- f := map[string]interface{}{"compile_kwd": []string{navCompileKwd}}+ f := map[string]interface{}{"type_kwd": []string{nav.TypeNavDoc, nav.TypeNavCluster}}for k, v := range extra {f[k] = v}@@ -648,7 +647,7 @@ func navIntersectScope(docIDs []string, allowed map[string]struct{}) []string {// SummariesByDocIDs returns the nav_doc description — the document's overall// summary — keyed by doc_id for the given documents. It mirrors Python// dataset_api_service._nav_doc_summaries: it reads the nav_doc rows-// (compile_kwd=dataset_nav, type_kwd=nav_doc) for the doc_ids and yields, per+// (type_kwd=nav_doc) for the doc_ids and yields, per// doc, the payload description. The description wins over the row's label on// purpose: callers render this value to the model as the routed document's// <summary>, and title_kwd holds only the summary's first line (length-capped).@@ -706,6 +705,13 @@ func (s *NavService) UpsertDoc(ctx context.Context, in nav.UpsertDocInput) errorif strings.TrimSpace(in.Summary) == "" {return nil}+ in.CompileKind = types.CanonicalCompilationKind(in.CompileKind)+ if in.CompileKind == "" {+ in.CompileKind = "tree"+ }+ if in.CompileKind != "tree" && in.CompileKind != "page_index" {+ return fmt.Errorf("datasetnav: invalid compilation kind %q", in.CompileKind)+ }de, err := s.docEngine()if err != nil {return err@@ -739,7 +745,7 @@ func (s *NavService) UpsertDoc(ctx context.Context, in nav.UpsertDocInput) error"type_kwd": []string{nav.TypeNavDoc},"doc_id": []string{in.DocID},}),- []string{"content_with_weight", "content_ltks", "content_sm_ltks", "title_kwd", "docnm_kwd"}, 0, 1, nil)+ []string{"content_with_weight", "content_ltks", "content_sm_ltks", "title_kwd", "docnm_kwd", "compile_kwd"}, 0, 1, nil)if err != nil {return err}@@ -751,16 +757,20 @@ func (s *NavService) UpsertDoc(ctx context.Context, in nav.UpsertDocInput) error// file name. The tokenized text is (re)built when it is missing.title := navDocTitle(in.Summary)titleStale := navRowLabel(existing[0]) != title+ kindStale := firstStringValue(existing[0]["compile_kwd"]) != in.CompileKindltksStale := firstStringValue(existing[0]["content_ltks"]) == "" ||firstStringValue(existing[0]["content_sm_ltks"]) == ""if payload, ok := existing[0]["content_with_weight"].(string); ok {var m map[string]interface{}if err := json.Unmarshal([]byte(payload), &m); err == nil {if d, _ := m["description"].(string); d == in.Summary {- if !titleStale && !ltksStale {+ if !titleStale && !ltksStale && !kindStale {return nil // unchanged}updates := map[string]interface{}{}+ if kindStale {+ updates["compile_kwd"] = in.CompileKind+ }if titleStale {updates["title_kwd"] = title}@@ -770,10 +780,9 @@ func (s *NavService) UpsertDoc(ctx context.Context, in nav.UpsertDocInput) error}return de.UpdateChunks(ctx,map[string]interface{}{- "compile_kwd": []string{navCompileKwd},- "type_kwd": []string{nav.TypeNavDoc},- "doc_id": []string{in.DocID},- "kb_id": in.KbID,+ "type_kwd": []string{nav.TypeNavDoc},+ "doc_id": []string{in.DocID},+ "kb_id": in.KbID,},updates, s.navIndexName(in.TenantID), in.KbID)}@@ -813,7 +822,7 @@ func (s *NavService) UpsertDoc(ctx context.Context, in nav.UpsertDocInput) errorreturn err}if err := s.appendDocToCluster(ctx, de, in.TenantID, in.KbID, bestName, in.DocID,- fmt.Sprintf("q_%d_vec", len(vec))); err != nil {+ fmt.Sprintf("q_%d_vec", len(vec)), in.CompileKind); err != nil {return err}// Explicit stable id (A5): the nav_doc row is addressable by a@@ -822,7 +831,7 @@ func (s *NavService) UpsertDoc(ctx context.Context, in nav.UpsertDocInput) error// instead of relying on a doc_id + type filter alone.row := map[string]interface{}{"id": navDocID(in.TenantID, in.KbID, in.DocID),- "compile_kwd": navCompileKwd,+ "compile_kwd": in.CompileKind,"available_int": 0,"type_kwd": "nav_doc",// The nav_doc's label is a readable title derived from the summary@@ -882,7 +891,7 @@ func (s *NavService) UpsertDoc(ctx context.Context, in nav.UpsertDocInput) errorclusterRow := map[string]interface{}{"id": navClusterID(in.TenantID, in.KbID, name),"doc_id": in.KbID, // cluster rows carry the kb as doc_id (Python _build_nav_cluster_row), so ES InsertChunks does not skip them- "compile_kwd": navCompileKwd,+ "compile_kwd": in.CompileKind,"available_int": 0,"type_kwd": "nav_cluster","title_kwd": name,@@ -906,7 +915,7 @@ func (s *NavService) UpsertDoc(ctx context.Context, in nav.UpsertDocInput) error// empty. The nav_doc carries a readable title + parent_kwd = the cluster name.docRow := map[string]interface{}{"id": navDocID(in.TenantID, in.KbID, in.DocID),- "compile_kwd": navCompileKwd,+ "compile_kwd": in.CompileKind,"available_int": 0,"type_kwd": "nav_doc","title_kwd": navDocTitle(in.Summary),@@ -964,8 +973,8 @@ func navChildRehomeCondition(child map[string]interface{}, kbID string) map[strireturn nil}cond := map[string]interface{}{- "compile_kwd": []string{navCompileKwd},- "kb_id": kbID,+ "type_kwd": []string{nav.TypeNavDoc, nav.TypeNavCluster},+ "kb_id": kbID,}if rowID := firstStringValue(child["id"]); rowID != "" {cond["id"] = []string{rowID}@@ -1148,7 +1157,7 @@ func (s *NavService) maybeSplitCluster(ctx context.Context, tenantID, kbID, clus// engine-assigned row id and inherit the cluster's directly-held documents.clusterChunks, _, err := s.navSearch(ctx, tenantID, kbID,navClusterCondition(tenantID, kbID, clusterName),- []string{"id", "doc_count_int", "doc_ids_kwd", "parent_kwd", "depth_int"}, 0, 1, nil)+ []string{"id", "doc_count_int", "doc_ids_kwd", "parent_kwd", "depth_int", "compile_kwd"}, 0, 1, nil)if err != nil {return err}@@ -1242,6 +1251,10 @@ func (s *NavService) maybeSplitCluster(ctx context.Context, tenantID, kbID, clusif err != nil {return err}+ splitCompileKind := "tree"+ if len(clusterChunks) > 0 && types.CompilationKind(clusterChunks[0]) == "page_index" {+ splitCompileKind = "page_index"+ }for _, spl := range []struct {name stringcount int@@ -1269,7 +1282,7 @@ func (s *NavService) maybeSplitCluster(ctx context.Context, tenantID, kbID, clus}row := map[string]interface{}{"id": navClusterID(tenantID, kbID, spl.name),- "compile_kwd": navCompileKwd,+ "compile_kwd": splitCompileKind,"available_int": 0,"type_kwd": "nav_cluster","title_kwd": spl.name,@@ -1507,10 +1520,10 @@ func (s *NavService) llmCreateSummary(ctx context.Context, tenantID, text string// doc_count_int. Implemented as a read-modify-write. vecCol names the engine// vector column (q_<dim>_vec) so an overfull-cluster split can inherit a// representative vector; pass "" when the caller has no known dimension.-func (s *NavService) appendDocToCluster(ctx context.Context, de engine.DocEngine, tenantID, kbID, clusterName, docID, vecCol string) error {+func (s *NavService) appendDocToCluster(ctx context.Context, de engine.DocEngine, tenantID, kbID, clusterName, docID, vecCol, compileKind string) error {chunks, _, err := s.navSearch(ctx, tenantID, kbID,navClusterCondition(tenantID, kbID, clusterName),- []string{"doc_ids_kwd", "doc_count_int"}, 0, 1, nil)+ []string{"doc_ids_kwd", "doc_count_int", "compile_kwd"}, 0, 1, nil)if err != nil {return err}@@ -1546,9 +1559,13 @@ func (s *NavService) appendDocToCluster(ctx context.Context, de engine.DocEngine// this one. The read-modify-write here is expected to run under a per-dataset// lock held by the UpsertDoc caller; without it, concurrent appends to the// same cluster can lose updates.+ updates := map[string]interface{}{"doc_ids_kwd": ids, "doc_count_int": count}+ if kind := types.CompilationKind(chunks[0]); kind != "tree" && kind != "page_index" {+ updates["compile_kwd"] = compileKind+ }if err := de.UpdateChunks(ctx,navClusterCondition(tenantID, kbID, clusterName),- map[string]interface{}{"doc_ids_kwd": ids, "doc_count_int": count},+ updates,s.navIndexName(tenantID), kbID); err != nil {return err}@@ -1649,7 +1666,7 @@ func (s *NavService) findClusterContainingDoc(ctx context.Context, tenantID, kbIIndexNames: []string{s.navIndexName(tenantID)},KbIDs: []string{kbID},SelectFields: []string{"title_kwd", "docnm_kwd", "doc_ids_kwd"},- Filter: map[string]interface{}{"type_kwd": "nav_cluster", "compile_kwd": []string{navCompileKwd}},+ Filter: map[string]interface{}{"type_kwd": "nav_cluster"},Offset: offset,Limit: pageSize,})diff --git a/internal/service/nlp/datasetnav_test.go b/internal/service/nlp/datasetnav_test.goindex 56e03bef5c..48a024b1eb 100644--- a/internal/service/nlp/datasetnav_test.go+++ b/internal/service/nlp/datasetnav_test.go@@ -188,6 +188,8 @@ func (m *memNavEngine) FilterDocIdsByMetaPushdown(context.Context, *gorm.DB, []sreturn nil}+const navCompileKwd = "dataset_nav"+// matchNavRow mirrors dataset_nav._matches_condition (dataset_nav.py:598-608):// every field must have one of the row's values equal to one of the condition's// values. A list condition therefore means intersection when the row's field is@@ -307,6 +309,49 @@ func newTestNav(eng *memNavEngine) *NavService {return ns}+func TestNavServicePageIndexRowsAndCleanup(t *testing.T) {+ eng := &memNavEngine{}+ ns := newTestNav(eng)+ if err := ns.UpsertDoc(t.Context(), nav.UpsertDocInput{TenantID: "t1", KbID: "kb1", DocID: "d1", CompileKind: "page_index", Summary: "Page index summary"}); err != nil {+ t.Fatal(err)+ }+ if len(eng.rows) != 2 {+ t.Fatalf("rows = %d, want document and cluster", len(eng.rows))+ }+ for _, row := range eng.rows {+ if row["compile_kwd"] != "page_index" {+ t.Fatalf("compile_kwd = %v, want page_index", row["compile_kwd"])+ }+ if row["type_kwd"] == "nav_cluster" {+ row["compile_kwd"] = "dataset_nav"+ }+ }+ if err := ns.UpsertDoc(t.Context(), nav.UpsertDocInput{TenantID: "t1", KbID: "kb1", DocID: "d2", CompileKind: "page_index", Summary: "Page index summary"}); err != nil {+ t.Fatal(err)+ }+ if len(eng.rows) != 3 {+ t.Fatalf("rows = %d, want two documents in one cluster", len(eng.rows))+ }+ for _, row := range eng.rows {+ if row["compile_kwd"] != "page_index" {+ t.Fatalf("updated compile_kwd = %v, want page_index", row["compile_kwd"])+ }+ }+ hits, err := ns.Search(t.Context(), "t1", "kb1", "Page index", nil, nil, 5)+ if err != nil || len(hits) == 0 {+ t.Fatalf("Search = %v, %v", hits, err)+ }+ if err := ns.RemoveDoc(t.Context(), "t1", "kb1", "d1"); err != nil {+ t.Fatal(err)+ }+ if err := ns.RemoveDoc(t.Context(), "t1", "kb1", "d2"); err != nil {+ t.Fatal(err)+ }+ if len(eng.rows) != 0 {+ t.Fatalf("rows after RemoveDoc = %d, want 0", len(eng.rows))+ }+}+// TestNavService_UpsertDoc_WritesNavRow asserts acceptance #1: after UpsertDoc// the row carries both compile_kwd=dataset_nav and available_int=0.func TestNavService_UpsertDoc_WritesNavRow(t *testing.T) {@@ -319,8 +364,8 @@ func TestNavService_UpsertDoc_WritesNavRow(t *testing.T) {t.Fatal("expected at least one nav row")}row := eng.rows[0]- if row["compile_kwd"] != "dataset_nav" {- t.Errorf("compile_kwd = %v, want dataset_nav", row["compile_kwd"])+ if row["compile_kwd"] != "tree" {+ t.Errorf("compile_kwd = %v, want tree", row["compile_kwd"])}if row["available_int"] != 0 {t.Errorf("available_int = %v, want 0", row["available_int"])diff --git a/internal/service/wikisearch/engine_service.go b/internal/service/wikisearch/engine_service.goindex 9eac92b799..a10e24a1a5 100644--- a/internal/service/wikisearch/engine_service.go+++ b/internal/service/wikisearch/engine_service.go@@ -9,10 +9,7 @@ import ("ragflow/internal/engine/types")-// compileKWDWikiPage is the canonical compile_kwd for compiled wiki pages. It-// must match Python's WIKI_PAGE_COMPILE_KWD ("wiki_page", wiki.py:1661) AND the-// Go compiler's variantCompileKWD[VariantWiki] (component.go), so both Python--// and Go-produced wiki pages are surfaced by this service.+// compileKWDWikiPage identifies the wiki page role.const compileKWDWikiPage = "wiki_page"// tenantIndexName returns the tenant-scoped chunk index name@@ -59,7 +56,7 @@ func (s *engineWikiService) AvailableFor(ctx context.Context, tenantID string, dKbIDs: datasetIDs,Limit: 1,Filter: map[string]interface{}{- "compile_kwd": compileKWDWikiPage,+ "type_kwd": compileKWDWikiPage,"available_int": 1,},})@@ -91,12 +88,9 @@ func (s *engineWikiService) QueryPages(ctx context.Context, tenantID string, dat"id", "kb_id", "doc_id", "docnm_kwd", "content_with_weight","slug_kwd", "source_chunk_ids",},- // Discriminate wiki pages by compile_kwd="wiki_page". There is NO- // "kc_kind" column in the chunk schema (infinity_mapping.json:47-56), so- // it must not be used as a filter. Sections (page.go kind:"section") are- // stamped compile_kwd="wiki_section", so this filter returns pages only.+ // Page roles exclude sections and internal wiki state.Filter: map[string]interface{}{- "compile_kwd": compileKWDWikiPage,+ "type_kwd": compileKWDWikiPage,"available_int": 1,},MatchExprs: []interface{}{diff --git a/internal/service/wikisearch/engine_service_test.go b/internal/service/wikisearch/engine_service_test.goindex fb71c681cc..700d4d5180 100644--- a/internal/service/wikisearch/engine_service_test.go+++ b/internal/service/wikisearch/engine_service_test.go@@ -44,10 +44,10 @@ func (f *fakeDocEngine) Search(_ context.Context, req *types.SearchRequest) (*tyrows := f.searchRows// Honor the compile_kwd filter so availability semantics are exercised// faithfully (only rows carrying the requested compile_kwd match).- if kwd, ok := req.Filter["compile_kwd"].(string); ok {+ if kwd, ok := req.Filter["type_kwd"].(string); ok {filtered := make([]map[string]interface{}, 0, len(rows))for _, r := range rows {- if r["compile_kwd"] == kwd {+ if types.CompilationRowType(r) == kwd {filtered = append(filtered, r)}}@@ -109,8 +109,8 @@ func TestEngineService_QueryPages_NormalizesEngineRowShape(t *testing.T) {if !reflect.DeepEqual(eng.searchReq.KbIDs, []string{"kb1"}) {t.Errorf("KbIDs = %v, want [kb1]", eng.searchReq.KbIDs)}- if f, ok := eng.searchReq.Filter["compile_kwd"].(string); !ok || f != "wiki_page" {- t.Errorf("compile_kwd filter = %v, want wiki_page", eng.searchReq.Filter["compile_kwd"])+ if f, ok := eng.searchReq.Filter["type_kwd"].(string); !ok || f != "wiki_page" {+ t.Errorf("compile_kwd filter = %v, want wiki_page", eng.searchReq.Filter["type_kwd"])}// SelectFields must project slug_kwd + source_chunk_ids (Infinity's default// projection omits them), otherwise the page slug and P7 provenance are lost.@@ -238,8 +238,8 @@ func TestEngineService_AvailableFor_BoundedExistenceSearch(t *testing.T) {if eng.searchReq.Limit != 1 {t.Errorf("existence Limit = %d, want 1 (bounded)", eng.searchReq.Limit)}- if f, ok := eng.searchReq.Filter["compile_kwd"].(string); !ok || f != "wiki_page" {- t.Errorf("existence compile_kwd filter = %v, want wiki_page", eng.searchReq.Filter["compile_kwd"])+ if f, ok := eng.searchReq.Filter["type_kwd"].(string); !ok || f != "wiki_page" {+ t.Errorf("existence compile_kwd filter = %v, want wiki_page", eng.searchReq.Filter["type_kwd"])}if !reflect.DeepEqual(eng.searchReq.KbIDs, []string{"kb1"}) {t.Errorf("existence KbIDs = %v, want [kb1]", eng.searchReq.KbIDs)